257 lines
7.6 KiB
Go
257 lines
7.6 KiB
Go
|
//go:build generate
|
||
|
|
|
||
|
|
// This program generates a property file in Go file from Unicode Character
|
||
|
|
// Database auxiliary data files. The command line arguments are as follows:
|
||
|
|
//
|
||
|
|
// 1. The name of the Unicode data file (just the filename, without extension).
|
||
|
|
// Can be "-" (to skip) if the emoji flag is included.
|
||
|
|
// 2. The name of the locally generated Go file.
|
||
|
|
// 3. The name of the slice mapping code points to properties.
|
||
|
|
// 4. The name of the generator, for logging purposes.
|
||
|
|
// 5. (Optional) Flags, comma-separated. The following flags are available:
|
||
|
|
// - "emojis=<property>": include the specified emoji properties (e.g.
|
||
|
|
// "Extended_Pictographic").
|
||
|
|
// - "gencat": include general category properties.
|
||
|
|
//
|
||
|
|
//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic
|
||
|
|
//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic
|
||
|
|
//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences
|
||
|
|
//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat
|
||
|
|
//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth
|
||
|
|
//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation
|
||
|
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bufio"
|
||
|
|
"bytes"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"go/format"
|
||
|
|
"io/ioutil"
|
||
|
|
"log"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
"regexp"
|
||
|
|
"sort"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
// We want to test against a specific version rather than the latest. When the
|
||
|
|
// package is upgraded to a new version, change these to generate new tests.
|
||
|
|
const (
|
||
|
|
propertyURL = `https://www.unicode.org/Public/14.0.0/ucd/%s.txt`
|
||
|
|
emojiURL = `https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt`
|
||
|
|
)
|
||
|
|
|
||
|
|
// The regular expression for a line containing a code point range property.
|
||
|
|
var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`)
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
if len(os.Args) < 5 {
|
||
|
|
fmt.Println("Not enough arguments, see code for details")
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
|
||
|
|
log.SetPrefix("gen_properties (" + os.Args[4] + "): ")
|
||
|
|
log.SetFlags(0)
|
||
|
|
|
||
|
|
// Parse flags.
|
||
|
|
flags := make(map[string]string)
|
||
|
|
if len(os.Args) >= 6 {
|
||
|
|
for _, flag := range strings.Split(os.Args[5], ",") {
|
||
|
|
flagFields := strings.Split(flag, "=")
|
||
|
|
if len(flagFields) == 1 {
|
||
|
|
flags[flagFields[0]] = "yes"
|
||
|
|
} else {
|
||
|
|
flags[flagFields[0]] = flagFields[1]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Parse the text file and generate Go source code from it.
|
||
|
|
_, includeGeneralCategory := flags["gencat"]
|
||
|
|
var mainURL string
|
||
|
|
if os.Args[1] != "-" {
|
||
|
|
mainURL = fmt.Sprintf(propertyURL, os.Args[1])
|
||
|
|
}
|
||
|
|
src, err := parse(mainURL, flags["emojis"], includeGeneralCategory)
|
||
|
|
if err != nil {
|
||
|
|
log.Fatal(err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Format the Go code.
|
||
|
|
formatted, err := format.Source([]byte(src))
|
||
|
|
if err != nil {
|
||
|
|
log.Fatal("gofmt:", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Save it to the (local) target file.
|
||
|
|
log.Print("Writing to ", os.Args[2])
|
||
|
|
if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil {
|
||
|
|
log.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// parse parses the Unicode Properties text files located at the given URLs and
|
||
|
|
// returns their equivalent Go source code to be used in the uniseg package. If
|
||
|
|
// "emojiProperty" is not an empty string, emoji code points for that emoji
|
||
|
|
// property (e.g. "Extended_Pictographic") will be included. In those cases, you
|
||
|
|
// may pass an empty "propertyURL" to skip parsing the main properties file. If
|
||
|
|
// "includeGeneralCategory" is true, the Unicode General Category property will
|
||
|
|
// be extracted from the comments and included in the output.
|
||
|
|
func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) {
|
||
|
|
if propertyURL == "" && emojiProperty == "" {
|
||
|
|
return "", errors.New("no properties to parse")
|
||
|
|
}
|
||
|
|
|
||
|
|
// Temporary buffer to hold properties.
|
||
|
|
var properties [][4]string
|
||
|
|
|
||
|
|
// Open the first URL.
|
||
|
|
if propertyURL != "" {
|
||
|
|
log.Printf("Parsing %s", propertyURL)
|
||
|
|
res, err := http.Get(propertyURL)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
in1 := res.Body
|
||
|
|
defer in1.Close()
|
||
|
|
|
||
|
|
// Parse it.
|
||
|
|
scanner := bufio.NewScanner(in1)
|
||
|
|
num := 0
|
||
|
|
for scanner.Scan() {
|
||
|
|
num++
|
||
|
|
line := strings.TrimSpace(scanner.Text())
|
||
|
|
|
||
|
|
// Skip comments and empty lines.
|
||
|
|
if strings.HasPrefix(line, "#") || line == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
// Everything else must be a code point range, a property and a comment.
|
||
|
|
from, to, property, comment, err := parseProperty(line)
|
||
|
|
if err != nil {
|
||
|
|
return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err)
|
||
|
|
}
|
||
|
|
properties = append(properties, [4]string{from, to, property, comment})
|
||
|
|
}
|
||
|
|
if err := scanner.Err(); err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Open the second URL.
|
||
|
|
if emojiProperty != "" {
|
||
|
|
log.Printf("Parsing %s", emojiURL)
|
||
|
|
res, err := http.Get(emojiURL)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
in2 := res.Body
|
||
|
|
defer in2.Close()
|
||
|
|
|
||
|
|
// Parse it.
|
||
|
|
scanner := bufio.NewScanner(in2)
|
||
|
|
num := 0
|
||
|
|
for scanner.Scan() {
|
||
|
|
num++
|
||
|
|
line := scanner.Text()
|
||
|
|
|
||
|
|
// Skip comments, empty lines, and everything not containing
|
||
|
|
// "Extended_Pictographic".
|
||
|
|
if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
// Everything else must be a code point range, a property and a comment.
|
||
|
|
from, to, property, comment, err := parseProperty(line)
|
||
|
|
if err != nil {
|
||
|
|
return "", fmt.Errorf("emojis line %d: %v", num, err)
|
||
|
|
}
|
||
|
|
properties = append(properties, [4]string{from, to, property, comment})
|
||
|
|
}
|
||
|
|
if err := scanner.Err(); err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Sort properties.
|
||
|
|
sort.Slice(properties, func(i, j int) bool {
|
||
|
|
left, _ := strconv.ParseUint(properties[i][0], 16, 64)
|
||
|
|
right, _ := strconv.ParseUint(properties[j][0], 16, 64)
|
||
|
|
return left < right
|
||
|
|
})
|
||
|
|
|
||
|
|
// Header.
|
||
|
|
var (
|
||
|
|
buf bytes.Buffer
|
||
|
|
emojiComment string
|
||
|
|
)
|
||
|
|
columns := 3
|
||
|
|
if includeGeneralCategory {
|
||
|
|
columns = 4
|
||
|
|
}
|
||
|
|
if emojiURL != "" {
|
||
|
|
emojiComment = `
|
||
|
|
// and
|
||
|
|
// ` + emojiURL + `
|
||
|
|
// ("Extended_Pictographic" only)`
|
||
|
|
}
|
||
|
|
buf.WriteString(`package uniseg
|
||
|
|
|
||
|
|
// Code generated via go generate from gen_properties.go. DO NOT EDIT.
|
||
|
|
|
||
|
|
// ` + os.Args[3] + ` are taken from
|
||
|
|
// ` + propertyURL + emojiComment + `
|
||
|
|
// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode
|
||
|
|
// license agreement.
|
||
|
|
var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{
|
||
|
|
`)
|
||
|
|
|
||
|
|
// Properties.
|
||
|
|
for _, prop := range properties {
|
||
|
|
if includeGeneralCategory {
|
||
|
|
generalCategory := "gc" + prop[3][:2]
|
||
|
|
if generalCategory == "gcL&" {
|
||
|
|
generalCategory = "gcLC"
|
||
|
|
}
|
||
|
|
prop[3] = prop[3][3:]
|
||
|
|
fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3])
|
||
|
|
} else {
|
||
|
|
fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Tail.
|
||
|
|
buf.WriteString("}")
|
||
|
|
|
||
|
|
return buf.String(), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// parseProperty parses a line of the Unicode properties text file containing a
|
||
|
|
// property for a code point range and returns it along with its comment.
|
||
|
|
func parseProperty(line string) (from, to, property, comment string, err error) {
|
||
|
|
fields := propertyPattern.FindStringSubmatch(line)
|
||
|
|
if fields == nil {
|
||
|
|
err = errors.New("no property found")
|
||
|
|
return
|
||
|
|
}
|
||
|
|
from = fields[1]
|
||
|
|
to = fields[3]
|
||
|
|
if to == "" {
|
||
|
|
to = from
|
||
|
|
}
|
||
|
|
property = fields[4]
|
||
|
|
comment = fields[5]
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// translateProperty translates a property name as used in the Unicode data file
|
||
|
|
// to a variable used in the Go code.
|
||
|
|
func translateProperty(prefix, property string) string {
|
||
|
|
return prefix + strings.ReplaceAll(property, "_", "")
|
||
|
|
}
|