From 41664677583a835b15b626d3b522e5d4245442c7 Mon Sep 17 00:00:00 2001 From: Konstantinos Bairaktaris Date: Tue, 4 Jul 2023 15:29:16 +0300 Subject: [PATCH 1/3] Add '--replace-edited-strings' flag/config option --- README.md | 5 ++ cmd/tx/main.go | 32 ++++++----- internal/txlib/config/local.go | 64 ++++++++++++++------- internal/txlib/push.go | 44 +++++++------- pkg/jsonapi/resource.go | 6 ++ pkg/txapi/resource_strings_async_uploads.go | 8 ++- 6 files changed, 102 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index b19c88a..ca8e87f 100644 --- a/README.md +++ b/README.md @@ -590,6 +590,11 @@ fall back to taking the filesystem timestamp into account. any time. - `--silent`: Reduce verbosity of the output. +- `--replace-edited-strings`: If present, source strings that have been edited + (in the editor UI or via the API) will not be protected from this source + file push and will instead be replaced. This can also be set on a + per-resource level in the configuration file. + ### Pulling Files from Transifex `tx pull` is used to pull language files (usually translation language files) from diff --git a/cmd/tx/main.go b/cmd/tx/main.go index 2674ccd..855907a 100644 --- a/cmd/tx/main.go +++ b/cmd/tx/main.go @@ -259,6 +259,11 @@ func Main() { Name: "silent", Usage: "Whether to reduce verbosity of the output", }, + &cli.BoolFlag{ + Name: "replace-edited-strings", + Usage: "Whether to replace source strings that have been edited in the " + + "meantime", + }, }, Action: func(c *cli.Context) error { cfg, err := config.LoadFromPaths( @@ -327,19 +332,20 @@ func Main() { } args := txlib.PushCommandArguments{ - Source: c.Bool("source"), - Translation: c.Bool("translation"), - Force: c.Bool("force"), - Skip: c.Bool("skip"), - Xliff: c.Bool("xliff"), - Languages: languages, - ResourceIds: resourceIds, - UseGitTimestamps: c.Bool("use-git-timestamps"), - Branch: c.String("branch"), - Base: c.String("base"), - All: c.Bool("all"), - Workers: workers, - Silent: c.Bool("silent"), + Source: c.Bool("source"), + Translation: c.Bool("translation"), + Force: c.Bool("force"), + Skip: c.Bool("skip"), + Xliff: c.Bool("xliff"), + Languages: languages, + ResourceIds: resourceIds, + UseGitTimestamps: c.Bool("use-git-timestamps"), + Branch: c.String("branch"), + Base: c.String("base"), + All: c.Bool("all"), + Workers: workers, + Silent: c.Bool("silent"), + ReplaceEditedStrings: c.Bool("replace-edited-strings"), } if args.All && len(args.Languages) > 0 { diff --git a/internal/txlib/config/local.go b/internal/txlib/config/local.go index 3a2d7c9..1f97495 100644 --- a/internal/txlib/config/local.go +++ b/internal/txlib/config/local.go @@ -21,17 +21,18 @@ type LocalConfig struct { } type Resource struct { - OrganizationSlug string - ProjectSlug string - ResourceSlug string - FileFilter string - SourceFile string - SourceLanguage string - Type string - LanguageMappings map[string]string - Overrides map[string]string - MinimumPercentage int - ResourceName string + OrganizationSlug string + ProjectSlug string + ResourceSlug string + FileFilter string + SourceFile string + SourceLanguage string + Type string + LanguageMappings map[string]string + Overrides map[string]string + MinimumPercentage int + ResourceName string + ReplaceEditedStrings bool } func loadLocalConfig() (*LocalConfig, error) { @@ -121,18 +122,29 @@ func loadLocalConfigFromBytes(data []byte) (*LocalConfig, error) { return nil, err } + replaceEditedStrings := false + if section.HasKey("replace_edited_strings") { + replaceEditedStrings, err = section.Key("replace_edited_strings").Bool() + if err != nil { + return nil, fmt.Errorf( + "'replace_edited_strings' needs to be 'true' or 'false': %s", err, + ) + } + } + resource := Resource{ - OrganizationSlug: organizationSlug, - ProjectSlug: projectSlug, - ResourceSlug: resourceSlug, - FileFilter: section.Key("file_filter").String(), - SourceFile: section.Key("source_file").String(), - SourceLanguage: section.Key("source_lang").String(), - Type: section.Key("type").String(), - LanguageMappings: make(map[string]string), - Overrides: make(map[string]string), - MinimumPercentage: -1, - ResourceName: section.Key("resource_name").String(), + OrganizationSlug: organizationSlug, + ProjectSlug: projectSlug, + ResourceSlug: resourceSlug, + FileFilter: section.Key("file_filter").String(), + SourceFile: section.Key("source_file").String(), + SourceLanguage: section.Key("source_lang").String(), + Type: section.Key("type").String(), + LanguageMappings: make(map[string]string), + Overrides: make(map[string]string), + MinimumPercentage: -1, + ResourceName: section.Key("resource_name").String(), + ReplaceEditedStrings: replaceEditedStrings, } // Get first the perc in string to check if exists because .Key returns @@ -282,6 +294,10 @@ func (localCfg LocalConfig) saveToWriter(file io.Writer) error { return err } } + + section.NewKey( + "replace_edited_strings", strconv.FormatBool(resource.ReplaceEditedStrings), + ) } _, err = cfg.WriteTo(file) @@ -366,6 +382,10 @@ func localConfigsEqual(left, right *LocalConfig) bool { return false } } + + if leftResource.ReplaceEditedStrings != rightResource.ReplaceEditedStrings { + return false + } } return true diff --git a/internal/txlib/push.go b/internal/txlib/push.go index 93ca82a..9d8421b 100644 --- a/internal/txlib/push.go +++ b/internal/txlib/push.go @@ -17,19 +17,20 @@ import ( ) type PushCommandArguments struct { - Source bool - Translation bool - Force bool - Skip bool - Xliff bool - Languages []string - ResourceIds []string - UseGitTimestamps bool - Branch string - Base string - All bool - Workers int - Silent bool + Source bool + Translation bool + Force bool + Skip bool + Xliff bool + Languages []string + ResourceIds []string + UseGitTimestamps bool + Branch string + Base string + All bool + Workers int + Silent bool + ReplaceEditedStrings bool } func PushCommand( @@ -445,6 +446,7 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) { remoteStats[sourceLanguage.Id], args, resourceIsNew, + args.ReplaceEditedStrings || cfgResource.ReplaceEditedStrings, } } if args.Translation { // -t flag is set @@ -565,12 +567,13 @@ func (task *LanguagePushTask) Run(send func(string), abort func()) { } type SourceFilePushTask struct { - api *jsonapi.Connection - resource *jsonapi.Resource - sourceFile string - remoteStats *jsonapi.Resource - args PushCommandArguments - resourceIsNew bool + api *jsonapi.Connection + resource *jsonapi.Resource + sourceFile string + remoteStats *jsonapi.Resource + args PushCommandArguments + resourceIsNew bool + replaceEditedStrings bool } func (task *SourceFilePushTask) Run(send func(string), abort func()) { @@ -580,6 +583,7 @@ func (task *SourceFilePushTask) Run(send func(string), abort func()) { remoteStats := task.remoteStats args := task.args resourceIsNew := task.resourceIsNew + replaceEditedStrings := task.replaceEditedStrings parts := strings.Split(resource.Id, ":") sendMessage := func(body string, force bool) { @@ -624,7 +628,7 @@ func (task *SourceFilePushTask) Run(send func(string), abort func()) { err = handleThrottling( func() error { var err error - sourceUpload, err = txapi.UploadSource(api, resource, file) + sourceUpload, err = txapi.UploadSource(api, resource, file, replaceEditedStrings) return err }, "Uploading file", diff --git a/pkg/jsonapi/resource.go b/pkg/jsonapi/resource.go index 84dc0c5..66e0e27 100644 --- a/pkg/jsonapi/resource.go +++ b/pkg/jsonapi/resource.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "mime/multipart" + "strconv" ) type Resource struct { @@ -191,6 +192,11 @@ func (r *Resource) SaveAsMultipart(fields []string) error { if err != nil { return err } + case bool: + err := writer.WriteField(field, strconv.FormatBool(data)) + if err != nil { + return err + } case []byte: w, err := writer.CreateFormFile(field, fmt.Sprintf("%s.txt", field)) diff --git a/pkg/txapi/resource_strings_async_uploads.go b/pkg/txapi/resource_strings_async_uploads.go index 1787699..5e1157d 100644 --- a/pkg/txapi/resource_strings_async_uploads.go +++ b/pkg/txapi/resource_strings_async_uploads.go @@ -36,7 +36,10 @@ func (err *ResourceStringAsyncUploadAttributes) Error() string { } func UploadSource( - api *jsonapi.Connection, resource *jsonapi.Resource, file io.Reader, + api *jsonapi.Connection, + resource *jsonapi.Resource, + file io.Reader, + replaceEditedStrings bool, ) (*jsonapi.Resource, error) { data, err := io.ReadAll(file) if err != nil { @@ -49,7 +52,8 @@ func UploadSource( // Setting attributes directly here because POST and GET attributes are // different Attributes: map[string]interface{}{ - "content": data, + "content": data, + "replace_edited_strings": replaceEditedStrings, }, } upload.SetRelated("resource", resource) From 6d68ab014de57d3f10574c5c4a90035276b3299c Mon Sep 17 00:00:00 2001 From: Konstantinos Bairaktaris Date: Tue, 4 Jul 2023 15:49:58 +0300 Subject: [PATCH 2/3] Add '--keep-translations' flag/config option --- README.md | 4 ++++ cmd/tx/main.go | 6 ++++++ internal/txlib/config/local.go | 16 ++++++++++++++++ internal/txlib/push.go | 8 +++++++- pkg/txapi/resource_strings_async_uploads.go | 2 ++ 5 files changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ca8e87f..b2ff2d0 100644 --- a/README.md +++ b/README.md @@ -595,6 +595,10 @@ fall back to taking the filesystem timestamp into account. file push and will instead be replaced. This can also be set on a per-resource level in the configuration file. +- `--keep-transations`: If present, translations of source strings with the + same key whose content changes will not be discarded. This can also be set on + a per-resource level in the configuration file. + ### Pulling Files from Transifex `tx pull` is used to pull language files (usually translation language files) from diff --git a/cmd/tx/main.go b/cmd/tx/main.go index 855907a..8198b3b 100644 --- a/cmd/tx/main.go +++ b/cmd/tx/main.go @@ -264,6 +264,11 @@ func Main() { Usage: "Whether to replace source strings that have been edited in the " + "meantime", }, + &cli.BoolFlag{ + Name: "keep-translations", + Usage: "Whether to not discard translations if a source string with a " + + "pre-existing key changes", + }, }, Action: func(c *cli.Context) error { cfg, err := config.LoadFromPaths( @@ -346,6 +351,7 @@ func Main() { Workers: workers, Silent: c.Bool("silent"), ReplaceEditedStrings: c.Bool("replace-edited-strings"), + KeepTranslations: c.Bool("keep-translations"), } if args.All && len(args.Languages) > 0 { diff --git a/internal/txlib/config/local.go b/internal/txlib/config/local.go index 1f97495..0d72dbb 100644 --- a/internal/txlib/config/local.go +++ b/internal/txlib/config/local.go @@ -33,6 +33,7 @@ type Resource struct { MinimumPercentage int ResourceName string ReplaceEditedStrings bool + KeepTranslations bool } func loadLocalConfig() (*LocalConfig, error) { @@ -132,6 +133,16 @@ func loadLocalConfigFromBytes(data []byte) (*LocalConfig, error) { } } + keepTranslations := false + if section.HasKey("keep_translations") { + keepTranslations, err = section.Key("keep_translations").Bool() + if err != nil { + return nil, fmt.Errorf( + "'keep_translations' needs to be 'true' or 'false': %s", err, + ) + } + } + resource := Resource{ OrganizationSlug: organizationSlug, ProjectSlug: projectSlug, @@ -145,6 +156,7 @@ func loadLocalConfigFromBytes(data []byte) (*LocalConfig, error) { MinimumPercentage: -1, ResourceName: section.Key("resource_name").String(), ReplaceEditedStrings: replaceEditedStrings, + KeepTranslations: keepTranslations, } // Get first the perc in string to check if exists because .Key returns @@ -298,6 +310,10 @@ func (localCfg LocalConfig) saveToWriter(file io.Writer) error { section.NewKey( "replace_edited_strings", strconv.FormatBool(resource.ReplaceEditedStrings), ) + + section.NewKey( + "keep_translations", strconv.FormatBool(resource.KeepTranslations), + ) } _, err = cfg.WriteTo(file) diff --git a/internal/txlib/push.go b/internal/txlib/push.go index 9d8421b..4799a11 100644 --- a/internal/txlib/push.go +++ b/internal/txlib/push.go @@ -31,6 +31,7 @@ type PushCommandArguments struct { Workers int Silent bool ReplaceEditedStrings bool + KeepTranslations bool } func PushCommand( @@ -447,6 +448,7 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) { args, resourceIsNew, args.ReplaceEditedStrings || cfgResource.ReplaceEditedStrings, + args.KeepTranslations || cfgResource.KeepTranslations, } } if args.Translation { // -t flag is set @@ -574,6 +576,7 @@ type SourceFilePushTask struct { args PushCommandArguments resourceIsNew bool replaceEditedStrings bool + keepTranslations bool } func (task *SourceFilePushTask) Run(send func(string), abort func()) { @@ -584,6 +587,7 @@ func (task *SourceFilePushTask) Run(send func(string), abort func()) { args := task.args resourceIsNew := task.resourceIsNew replaceEditedStrings := task.replaceEditedStrings + keepTranslations := task.keepTranslations parts := strings.Split(resource.Id, ":") sendMessage := func(body string, force bool) { @@ -628,7 +632,9 @@ func (task *SourceFilePushTask) Run(send func(string), abort func()) { err = handleThrottling( func() error { var err error - sourceUpload, err = txapi.UploadSource(api, resource, file, replaceEditedStrings) + sourceUpload, err = txapi.UploadSource( + api, resource, file, replaceEditedStrings, keepTranslations, + ) return err }, "Uploading file", diff --git a/pkg/txapi/resource_strings_async_uploads.go b/pkg/txapi/resource_strings_async_uploads.go index 5e1157d..ccdeb03 100644 --- a/pkg/txapi/resource_strings_async_uploads.go +++ b/pkg/txapi/resource_strings_async_uploads.go @@ -40,6 +40,7 @@ func UploadSource( resource *jsonapi.Resource, file io.Reader, replaceEditedStrings bool, + keepTranslations bool, ) (*jsonapi.Resource, error) { data, err := io.ReadAll(file) if err != nil { @@ -54,6 +55,7 @@ func UploadSource( Attributes: map[string]interface{}{ "content": data, "replace_edited_strings": replaceEditedStrings, + "keep_translations": keepTranslations, }, } upload.SetRelated("resource", resource) From 26736f88d27d64cab549b76088fe87ddfd9e9890 Mon Sep 17 00:00:00 2001 From: Konstantinos Bairaktaris Date: Wed, 5 Jul 2023 13:04:03 +0300 Subject: [PATCH 3/3] Fix tests --- internal/txlib/migrate_test.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/txlib/migrate_test.go b/internal/txlib/migrate_test.go index 293f966..0a87ae0 100644 --- a/internal/txlib/migrate_test.go +++ b/internal/txlib/migrate_test.go @@ -6,6 +6,7 @@ import ( "log" "os" "path/filepath" + "regexp" "strings" "testing" @@ -680,10 +681,16 @@ func TestResourceMigrationFailed(t *testing.T) { string(content), "projslug1.ares")) assert.True(t, strings.Contains( string(content), "o:org:p:projslug2:r:ares2")) - assert.True(t, strings.Contains( - string(content), "minimum_perc = 10")) - assert.True(t, strings.Contains( - string(content), "minimum_perc = 0")) + matched, err := regexp.MatchString(`minimum_perc\s*=\s*10`, string(content)) + if err != nil { + t.Error(err) + } + assert.True(t, matched) + matched, err = regexp.MatchString(`minimum_perc\s*=\s*0`, string(content)) + if err != nil { + t.Error(err) + } + assert.True(t, matched) } func TestBackUpFileCreated(t *testing.T) {