Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Clarify Gitea/Crowdin locale behaviors, add tests for LocaleStore, fix some strings with semicolons #23819

Merged
merged 3 commits into from
Mar 31, 2023
Merged

Clarify Gitea/Crowdin locale behaviors, add tests for LocaleStore, fix some strings with semicolons #23819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
538bf8e
fix
wxiaoguang Mar 30, 2023
649bca8
fine tune tests
wxiaoguang Mar 30, 2023
2a29311
Merge branch 'main' into improve-locales
lunny Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions build/update-locales.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,10 @@ fi

mv ./options/locale/locale_en-US.ini ./options/

# the "ini" library for locale has many quirks
# * `a="xx"` gets `xx` (no quote)
# * `a=x\"y` gets `x\"y` (no unescaping)
# * `a="x\"y"` gets `"x\"y"` (no unescaping, the quotes are still there)
# * `a='x\"y'` gets `x\"y` (no unescaping, no quote)
# * `a="foo` gets `"foo` (although the quote is not closed)
# * 'a=`foo`' works like single-quote
# crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
# crowdin always outputs quoted strings if there are quotes in the strings.

# this script helps to unquote the crowdin outputs for the quirky ini library
# the "ini" library for locale has many quirks, its behavior is different from Crowdin.
# see i18n_test.go for more details

# this script helps to unquote the Crowdin outputs for the quirky ini library
# * find all `key="...\"..."` lines
# * remove the leading quote
# * remove the trailing quote
Expand Down
54 changes: 54 additions & 0 deletions modules/translation/i18n/i18n_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package i18n

import (
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -75,3 +76,56 @@ c=22
assert.Equal(t, "21", ls.Tr("lang1", "b"))
assert.Equal(t, "22", ls.Tr("lang1", "c"))
}

func TestLocaleStoreQuirks(t *testing.T) {
const nl = "\n"
q := func(q1, s string, q2 ...string) string {
return q1 + s + strings.Join(q2, "")
}
testDataList := []struct {
in string
out string
hint string
}{
{` xx`, `xx`, "simple, no quote"},
{`" xx"`, ` xx`, "simple, double-quote"},
{`' xx'`, ` xx`, "simple, single-quote"},
{"` xx`", ` xx`, "simple, back-quote"},

{`x\"y`, `x\"y`, "no unescape, simple"},
{q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"},
{q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"},
{q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"},

{q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"},
{q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"},
{q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"},

{`x ; y`, `x ; y`, "inline comment (;)"},
{`x # y`, `x # y`, "inline comment (#)"},
{`x \; y`, `x ; y`, `inline comment (\;)`},
{`x \# y`, `x # y`, `inline comment (\#)`},
}

for _, testData := range testDataList {
ls := NewLocaleStore()
err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
assert.NoError(t, err, testData.hint)
assert.Equal(t, testData.out, ls.Tr("lang1", "a"), testData.hint)
assert.NoError(t, ls.Close())
}

// TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
// and Crowdin always outputs quoted strings if there are quotes in the strings.
// So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly,
// it should be converted to `key="\"quoted\" unquoted"` first.
// TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini,
// then Crowdin will output:
// > key = "`x \" y`"
// Then Gitea will read a string with back-quotes, which is incorrect.
// TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore
// LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin.
// TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote
// so, the following line will be parsed as: value="`first", comment="second`" on Crowdin
// > a = `first; second`
}
8 changes: 4 additions & 4 deletions options/locale/locale_en-US.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2140,10 +2140,10 @@ settings.dismiss_stale_approvals_desc = When new commits that change the content
settings.require_signed_commits = Require Signed Commits
settings.require_signed_commits_desc = Reject pushes to this branch if they are unsigned or unverifiable.
settings.protect_branch_name_pattern = Protected Branch Name Pattern
settings.protect_protected_file_patterns = `Protected file patterns (separated using semicolon ';'):`
settings.protect_protected_file_patterns_desc = `Protected files are not allowed to be changed directly even if user has rights to add, edit, or delete files in this branch. Multiple patterns can be separated using semicolon (';'). See <a href="https://pkg.go.dev/github.com/gobwas/glob#Compile">github.com/gobwas/glob</a> documentation for pattern syntax. Examples: <code>.drone.yml</code>, <code>/docs/**/*.txt</code>.`
settings.protect_unprotected_file_patterns = `Unprotected file patterns (separated using semicolon ';'):`
settings.protect_unprotected_file_patterns_desc = `Unprotected files that are allowed to be changed directly if user has write access, bypassing push restriction. Multiple patterns can be separated using semicolon (';'). See <a href="https://pkg.go.dev/github.com/gobwas/glob#Compile">github.com/gobwas/glob</a> documentation for pattern syntax. Examples: <code>.drone.yml</code>, <code>/docs/**/*.txt</code>.`
settings.protect_protected_file_patterns = "Protected file patterns (separated using semicolon ';'):"
settings.protect_protected_file_patterns_desc = "Protected files are not allowed to be changed directly even if user has rights to add, edit, or delete files in this branch. Multiple patterns can be separated using semicolon (';'). See <a href='https://pkg.go.dev/github.com/gobwas/glob#Compile'>github.com/gobwas/glob</a> documentation for pattern syntax. Examples: <code>.drone.yml</code>, <code>/docs/**/*.txt</code>."
settings.protect_unprotected_file_patterns = "Unprotected file patterns (separated using semicolon ';'):"
settings.protect_unprotected_file_patterns_desc = "Unprotected files that are allowed to be changed directly if user has write access, bypassing push restriction. Multiple patterns can be separated using semicolon (';'). See <a href='https://pkg.go.dev/github.com/gobwas/glob#Compile'>github.com/gobwas/glob</a> documentation for pattern syntax. Examples: <code>.drone.yml</code>, <code>/docs/**/*.txt</code>."
settings.add_protected_branch = Enable protection
settings.delete_protected_branch = Disable protection
settings.update_protect_branch_success = Branch protection for rule '%s' has been updated.
Expand Down