package main import ( "crypto/sha1" "encoding/json" "flag" "fmt" "io" "os" "github.com/BurntSushi/toml" "github.com/nicksnyder/go-i18n/v2/i18n" "github.com/nicksnyder/go-i18n/v2/internal" "github.com/nicksnyder/go-i18n/v2/internal/plural" "golang.org/x/text/language" yaml "gopkg.in/yaml.v3" ) func usageMerge() { fmt.Fprintf(os.Stderr, `usage: goi18n merge [options] [message files] Merge reads all messages in the message files and produces two files per language. xx-yy.active.format This file contains messages that should be loaded at runtime. xx-yy.translate.format This file contains messages that are empty and should be translated. Message file names must have a suffix of a supported format (e.g. ".json") and contain a valid language tag as defined by RFC 5646 (e.g. "en-us", "fr", "zh-hant", etc.). To add support for a new language, create an empty translation file with the appropriate name and pass it in to goi18n merge. Flags: -sourceLanguage tag Translate messages from this language (e.g. en, en-US, zh-Hant-CN) Default: en -outdir directory Write message files to this directory. Default: . -format format Output message files in this format. Supported formats: json, toml, yaml Default: toml `) } type mergeCommand struct { messageFiles []string sourceLanguage languageTag outdir string format string } func (mc *mergeCommand) name() string { return "merge" } func (mc *mergeCommand) parse(args []string) error { flags := flag.NewFlagSet("merge", flag.ExitOnError) flags.Usage = usageMerge flags.Var(&mc.sourceLanguage, "sourceLanguage", "en") flags.StringVar(&mc.outdir, "outdir", ".", "") flags.StringVar(&mc.format, "format", "toml", "") if err := flags.Parse(args); err != nil { return err } mc.messageFiles = flags.Args() return nil } func (mc *mergeCommand) execute() error { if len(mc.messageFiles) < 1 { return fmt.Errorf("need at least one message file to parse") } inFiles := make(map[string][]byte) for _, path := range mc.messageFiles { content, err := os.ReadFile(path) if err != nil { return err } inFiles[path] = content } ops, err := merge(inFiles, mc.sourceLanguage.Tag(), mc.outdir, mc.format) if err != nil { return err } for path, content := range ops.writeFiles { if err := os.WriteFile(path, content, 0666); err != nil { return err } } for _, path := range ops.deleteFiles { // Ignore error since it isn't guaranteed to exist. _ = os.Remove(path) } return nil } type fileSystemOp struct { writeFiles map[string][]byte deleteFiles []string } func merge(messageFiles map[string][]byte, sourceLanguageTag language.Tag, outdir, outputFormat string) (*fileSystemOp, error) { unmerged := make(map[language.Tag][]map[string]*i18n.MessageTemplate) sourceMessageTemplates := make(map[string]*i18n.MessageTemplate) unmarshalFuncs := map[string]i18n.UnmarshalFunc{ "json": json.Unmarshal, "toml": toml.Unmarshal, "yaml": yaml.Unmarshal, } for path, content := range messageFiles { mf, err := i18n.ParseMessageFileBytes(content, path, unmarshalFuncs) if err != nil { return nil, fmt.Errorf("failed to load message file %s: %s", path, err) } templates := map[string]*i18n.MessageTemplate{} for _, m := range mf.Messages { template := i18n.NewMessageTemplate(m) if template == nil { continue } templates[m.ID] = template } if mf.Tag == sourceLanguageTag { for _, template := range templates { if sourceMessageTemplates[template.ID] != nil { return nil, fmt.Errorf("multiple source translations for id %q", template.ID) } template.Hash = hash(template) sourceMessageTemplates[template.ID] = template } } unmerged[mf.Tag] = append(unmerged[mf.Tag], templates) } if len(sourceMessageTemplates) == 0 { return nil, fmt.Errorf("no messages found for source locale %s", sourceLanguageTag) } pluralRules := plural.DefaultRules() all := make(map[language.Tag]map[string]*i18n.MessageTemplate) all[sourceLanguageTag] = sourceMessageTemplates for _, srcTemplate := range sourceMessageTemplates { for dstLangTag, messageTemplates := range unmerged { if dstLangTag == sourceLanguageTag { continue } pluralRule := pluralRules.Rule(dstLangTag) if pluralRule == nil { // Non-standard languages not supported because // we don't know if translations are complete or not. continue } if all[dstLangTag] == nil { all[dstLangTag] = make(map[string]*i18n.MessageTemplate) } dstMessageTemplate := all[dstLangTag][srcTemplate.ID] if dstMessageTemplate == nil { dstMessageTemplate = &i18n.MessageTemplate{ Message: &i18n.Message{ ID: srcTemplate.ID, Description: srcTemplate.Description, Hash: srcTemplate.Hash, }, PluralTemplates: make(map[plural.Form]*internal.Template), } all[dstLangTag][srcTemplate.ID] = dstMessageTemplate } // Check all unmerged message templates for this message id. for _, messageTemplates := range messageTemplates { unmergedTemplate := messageTemplates[srcTemplate.ID] if unmergedTemplate == nil { continue } // Ignore empty hashes for v1 backward compatibility. if unmergedTemplate.Hash != "" && unmergedTemplate.Hash != srcTemplate.Hash { // This was translated from different content so discard. continue } // Merge in the translated messages. for pluralForm := range pluralRule.PluralForms { dt := unmergedTemplate.PluralTemplates[pluralForm] if dt != nil && dt.Src != "" { dstMessageTemplate.PluralTemplates[pluralForm] = dt } } } } } translate := make(map[language.Tag]map[string]*i18n.MessageTemplate) active := make(map[language.Tag]map[string]*i18n.MessageTemplate) for langTag, messageTemplates := range all { active[langTag] = make(map[string]*i18n.MessageTemplate) if langTag == sourceLanguageTag { active[langTag] = messageTemplates continue } pluralRule := pluralRules.Rule(langTag) if pluralRule == nil { // Non-standard languages not supported because // we don't know if translations are complete or not. continue } for _, messageTemplate := range messageTemplates { srcMessageTemplate := sourceMessageTemplates[messageTemplate.ID] activeMessageTemplate, translateMessageTemplate := activeDst(srcMessageTemplate, messageTemplate, pluralRule) if translateMessageTemplate != nil { if translate[langTag] == nil { translate[langTag] = make(map[string]*i18n.MessageTemplate) } translate[langTag][messageTemplate.ID] = translateMessageTemplate } if activeMessageTemplate != nil { active[langTag][messageTemplate.ID] = activeMessageTemplate } } } writeFiles := make(map[string][]byte, len(translate)+len(active)) for langTag, messageTemplates := range translate { path, content, err := writeFile(outdir, "translate", langTag, outputFormat, messageTemplates, false) if err != nil { return nil, err } writeFiles[path] = content } deleteFiles := []string{} for langTag, messageTemplates := range active { path, content, err := writeFile(outdir, "active", langTag, outputFormat, messageTemplates, langTag == sourceLanguageTag) if err != nil { return nil, err } if len(content) > 0 { writeFiles[path] = content } else { deleteFiles = append(deleteFiles, path) } } return &fileSystemOp{writeFiles: writeFiles, deleteFiles: deleteFiles}, nil } // activeDst returns the active part of the dst and whether dst is a complete translation of src. func activeDst(src, dst *i18n.MessageTemplate, pluralRule *plural.Rule) (active *i18n.MessageTemplate, translateMessageTemplate *i18n.MessageTemplate) { pluralForms := pluralRule.PluralForms if len(src.PluralTemplates) == 1 { pluralForms = map[plural.Form]struct{}{ plural.Other: {}, } } for pluralForm := range pluralForms { dt := dst.PluralTemplates[pluralForm] if dt == nil || dt.Src == "" { if translateMessageTemplate == nil { translateMessageTemplate = &i18n.MessageTemplate{ Message: &i18n.Message{ ID: src.ID, Description: src.Description, Hash: src.Hash, }, PluralTemplates: make(map[plural.Form]*internal.Template), } } srcPlural := src.PluralTemplates[pluralForm] if srcPlural == nil { srcPlural = src.PluralTemplates[plural.Other] } translateMessageTemplate.PluralTemplates[pluralForm] = srcPlural continue } if active == nil { active = &i18n.MessageTemplate{ Message: &i18n.Message{ ID: src.ID, Description: src.Description, Hash: src.Hash, }, PluralTemplates: make(map[plural.Form]*internal.Template), } } active.PluralTemplates[pluralForm] = dt } return } func hash(t *i18n.MessageTemplate) string { h := sha1.New() _, _ = io.WriteString(h, t.Description) _, _ = io.WriteString(h, t.PluralTemplates[plural.Other].Src) return fmt.Sprintf("sha1-%x", h.Sum(nil)) }