diff --git a/package.json b/package.json index a85921a742..29f4823da5 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "@atom/source-map-support": "^0.3.4", "@babel/core": "7.18.6", + "@formatjs/icu-messageformat-parser": "^2.6.1", "about": "file:packages/about", "archive-view": "file:packages/archive-view", "async": "3.2.4", @@ -83,6 +84,7 @@ "grim": "2.0.3", "image-view": "file:packages/image-view", "incompatible-packages": "file:packages/incompatible-packages", + "intl-messageformat": "^10.5.1", "jasmine-json": "~0.0", "jasmine-reporters": "1.1.0", "jasmine-tagged": "^1.1.4", diff --git a/spec/package-manager-spec.js b/spec/package-manager-spec.js index 6948e10eb1..aca17056a9 100644 --- a/spec/package-manager-spec.js +++ b/spec/package-manager-spec.js @@ -1019,7 +1019,7 @@ describe('PackageManager', () => { spyOn(pack.mainModule, 'activate').andCallThrough(); await atom.packages.activatePackage('package-with-serialization'); - expect(pack.mainModule.activate).toHaveBeenCalledWith({ someNumber: 77 }); + expect(pack.mainModule.activate).toHaveBeenCalledWith({ someNumber: 77 }, { t: pack.t }); }); it('invokes ::onDidActivatePackage listeners with the activated package', async () => { diff --git a/spec/package-spec.js b/spec/package-spec.js index a235c4e919..899a4c9b02 100644 --- a/spec/package-spec.js +++ b/spec/package-spec.js @@ -18,7 +18,8 @@ describe('Package', function() { menuManager: atom.menu, contextMenuManager: atom.contextMenu, deserializerManager: atom.deserializers, - viewRegistry: atom.views + viewRegistry: atom.views, + i18n: atom.i18n }); const buildPackage = packagePath => build(Package, packagePath); diff --git a/src/atom-environment.js b/src/atom-environment.js index 13682f3e76..e9c389f7c4 100644 --- a/src/atom-environment.js +++ b/src/atom-environment.js @@ -45,6 +45,7 @@ const TextEditor = require('./text-editor'); const TextBuffer = require('text-buffer'); const TextEditorRegistry = require('./text-editor-registry'); const StartupTime = require('./startup-time'); +const I18n = require("./i18n"); const { getReleaseChannel } = require('./get-app-details.js'); const packagejson = require("../package.json"); @@ -103,6 +104,10 @@ class AtomEnvironment { type: 'object', properties: _.clone(ConfigSchema) }); + /** @type {I18n} */ + this.i18n = new I18n({ + config: this.config + }); /** @type {KeymapManager} */ this.keymaps = new KeymapManager({ @@ -135,7 +140,8 @@ class AtomEnvironment { grammarRegistry: this.grammars, deserializerManager: this.deserializers, viewRegistry: this.views, - uriHandlerRegistry: this.uriHandlerRegistry + uriHandlerRegistry: this.uriHandlerRegistry, + i18n: this.i18n }); /** @type {ThemeManager} */ @@ -266,6 +272,8 @@ class AtomEnvironment { }); this.config.resetUserSettings(userSettings); + this.i18n.initialise({ resourcePath }); + if (projectSpecification != null && projectSpecification.config != null) { this.project.replace(projectSpecification); } diff --git a/src/i18n.js b/src/i18n.js new file mode 100644 index 0000000000..950730c393 --- /dev/null +++ b/src/i18n.js @@ -0,0 +1,415 @@ +const fs = require("fs-plus"); +const path = require("path"); +const season = require("season"); +const { default: IntlMessageFormat } = require("intl-messageformat"); +const { parse: parseToAST } = require("@formatjs/icu-messageformat-parser"); + +/** + * @type {Array<{ + * ext: string; + * parse: (s: string) => any + * }>} + */ +const supportedFileExts = [ + { + ext: "cson", + parse: str => season.parse(str) + }, + { + ext: "json", + parse: str => JSON.parse(str) + } +]; + +module.exports = class I18n { + /** + * @param {object} opts + * @param {import("./config")} opts.config + */ + constructor({ config }) { + this.config = config; + /** @type {Array} */ + this.locales = []; + this.localisations = new Localisations(); + this.resourcePath = ""; + + this.config.setSchema("core.languageSettings", { + type: "object", + description: "These settings currently require a full restart to take effect", + properties: { + primaryLanguage: { + type: "string", + order: 1, + default: "en", + // TODO get available languages for enum purposes + }, + fallbackLanguages: { + type: "array", + order: 2, + description: "List of fallback languages in case something is not translated in your primary language. Note: `en` is always the last fallback language, to ensure that things at least show up.", + default: [], + items: { + type: "string" + // TODO consider array enum when enum array options are improved in settings UI? + } + } + } + }); + } + + /** + * @param {object} opts + * @param {string} opts.resourcePath + */ + initialise({ resourcePath }) { + this.locales = [ + this.config.get("core.languageSettings.primaryLanguage"), + ...this.config.get("core.languageSettings.fallbackLanguages"), + "en" + ].map(l => l.toLowerCase()); + this.resourcePath = resourcePath; + + this.localisations.initialise({ locales: this.locales }); // TODO ast cache + + this._loadStringsForCore(); + } + + /** + * @param {Key} keystr + * @param {Opts} opts + */ + t(keystr, opts = {}) { + return this.localisations.t(keystr, opts); + } + + /** + * @param {string} ns + */ + getT(ns) { + /** + * @param {Key} keystr + * @param {Opts} opts + */ + return (keystr, opts = {}) => this.t(`${ns}.${keystr}`, opts); + } + + _loadStringsForCore() { + this._loadStringsAt("core", path.join(this.resourcePath, "i18n")); + } + + /** + * @param {object} obj + * @param {string} obj.pkgName + * @param {string} obj.pkgPath + */ + loadStringsForPackage({ pkgName, pkgPath }) { + this._loadStringsAt(pkgName, path.join(pkgPath, "i18n")); + } + + /** + * @param {string} pkgName + * @param {string} i18nDirPath path to the i18n dir with the files in it + */ + _loadStringsAt(pkgName, i18nDirPath) { + if (!fs.existsSync(i18nDirPath)) return; + + /** @type {Array} */ + const filesArray = fs.readdirSync(i18nDirPath); + // set search performance is supposed to be better than array + const files = new Set(filesArray.map(f => f.toLowerCase())); + + /** @type {PackageStrings} */ + const packageStrings = {}; + this.locales.forEach(locale => { + const ext = supportedFileExts.find(({ ext }) => files.has(`${locale}.${ext}`)); + if (!ext) return; + + const filename = `${locale}.${ext.ext}`; + const filepath = path.join(i18nDirPath, filename); + + const strings = fs.readFileSync(filepath, "utf8"); + packageStrings[locale] = ext.parse(strings); + }); + this.localisations.addPackage({ + pkgName, + strings: packageStrings + }); + } +} + +class Localisations { + constructor() { + /** @type {Array} */ + this.locales = []; + /** @type {{ [k: string]: PackageLocalisations }} */ + this.packages = {}; + } + + /** + * @param {object} opts + * @param {Array} opts.locales + * @param {AllAstCache} [opts.asts] + */ + initialise({ locales, asts }) { + this.locales = locales; + this.asts = asts; + } + + /** + * @param {Key} keystr + * @param {Opts} opts + */ + t(keystr, opts = {}) { + let key = keystr.split("."); + if (key.length < 2) return fallback(keystr, opts); + + guardPrototypePollution(key); + + const pkgName = key[0]; + key = key.slice(1); + return this.packages[pkgName]?.t(key, opts) ?? fallback(keystr, opts); + } + + /** + * @param {object} opts + * @param {string} opts.pkgName + * @param {PackageStrings} opts.strings + */ + addPackage({ pkgName, strings }) { + this.packages[pkgName] = new PackageLocalisations({ + locales: this.locales, + strings, + asts: this.asts?.[pkgName] + }); + } +} + +/** + * manages strings of all languages for a single package + * (in other words, manages `PackageLocalisations` instances) + */ +class PackageLocalisations { + /** + * @param {object} opts + * @param {Array} opts.locales + * @param {PackageStrings} opts.strings + * @param {PackageASTCache} [opts.asts] + */ + constructor({ locales, strings: _strings, asts }) { + this.locales = locales; + /** @type {PackageASTCache} */ + this.asts = asts ?? {}; + /** @type {{ [k: string]: SingleLanguageLocalisations }} */ + this.localeObjs = {}; + + for (const [locale, strings] of Object.entries(_strings)) { + this.localeObjs[locale] = new SingleLanguageLocalisations({ + locale, + strings, + asts: (this.asts[locale] = this.asts[locale] ?? { items: {} }) + }); + } + } + + /** + * @param {SplitKey} key + * @param {Opts} opts + */ + t(key, opts = {}) { + for (const locale of this.locales) { + const localised = this.localeObjs[locale]?.t(key, opts); + if (localised) return localised; + } + } +} + +/** + * manages strings for a single locale of a single package + */ +// TODO someone please help me find a better name ~meadowsys +class SingleLanguageLocalisations { + /** + * @param {object} opts + * @param {string} opts.locale + * @param {Strings} opts.strings + * @param {ASTCache} [opts.asts] + */ + constructor({ locale, strings, asts }) { + this.locale = locale; + this.strings = strings; + /** @type {ASTCache} */ + this.asts = asts ?? { items: {} }; + /** @type {FormatterCache} */ + this.formatters = {}; + } + + /** + * @param {SplitKey} key + * @param {Opts} opts + */ + t(key, opts = {}) { + const formatter = this._getFormatter(key); + if (formatter) { + const formatted = /** @type {string} */ (formatter.format(opts)); + return formatted; + } + } + + /** + * @param {SplitKey} key + */ + _getFormatter(key) { + let value = this.formatters; + + const lastKeyPos = key.length - 1; + for (let i = 0; i < lastKeyPos; i++) { + const k = key[i]; + const v = value[k]; + + if (!v) { + /** @type {FormatterCache} */ + const cache = {}; + value[k] = cache; + value = cache; + continue; + } + if (v instanceof IntlMessageFormat) return; + value = v; + } + + const k = key[lastKeyPos]; + const v = value[k]; + + if (!v) { + const ast = this._getAST(key); + if (!ast) return; + + const formatter = new IntlMessageFormat(ast.ast, this.locale); + value[k] = formatter; + return formatter; + } + + if (v instanceof IntlMessageFormat) return v; + } + + /** + * @param {SplitKey} key + */ + _getAST(key) { + let value = this.asts; + + const lastKeyPos = key.length - 1; + for (let i = 0; i < lastKeyPos; i++) { + const k = key[i]; + const v = value.items[k]; + + if (!v) { + /** @type {ManyASTs} */ + const cache = { items: {} }; + value[k] = cache; + value = cache; + continue; + } + if (isAST(v)) return; + value = v; + } + + const k = key[lastKeyPos]; + const v = value.items[k]; + + if (!v) { + const string = this._getString(key); + if (!string) return; + + /** @type {AST} */ + const ast = { + ast: parseToAST(string) + }; + + value.items[k] = ast; + return ast; + } + + if (isAST(v)) return v; + } + + /** + * @param {SplitKey} key + */ + _getString(key) { + let value = this.strings; + + const lastKeyPos = key.length - 1; + for (let i = 0; i < lastKeyPos; i++) { + const k = key[i]; + const v = value[k]; + + if (!v || typeof v === "string") return; + value = v; + } + + const k = key[lastKeyPos]; + const v = value[k]; + if (typeof v === "string") return v; + } +} + +/** + * @param {OneOrManyASTs} obj + * @return {obj is AST} + */ +function isAST(obj) { + return "ast" in obj; +} + +/** + * @param {SplitKey} key + */ +function guardPrototypePollution(key) { + if (key.includes("__proto__")) { + throw new Error(`prototype pollution in key "${key.join(".")}" was detected and prevented`); + } +} + +/** + * @param {Key} keystr + * @param {Opts} opts + */ +function fallback(keystr, opts) { + const optsArray = Object.entries(opts); + if (optsArray.length === 0) return keystr; + + return `${keystr}: { ${ + optsArray + .map(([k, v]) => `${JSON.stringify(k)}: ${JSON.stringify(v)}`) + .join(", ") + } }`; +} + +/** + * "basic" types + * @typedef {import("@formatjs/icu-messageformat-parser").MessageFormatElement} MessageFormatElement + * @typedef {{ ast: Array }} AST + * @typedef {{ items: { [k: string]: OneOrManyASTs } }} ManyASTs + * @typedef {AST | ManyASTs} OneOrManyASTs + * + * @typedef {string} Key + * @typedef {Array} SplitKey + * @typedef {{ [k: string]: string }} Opts + */ +/** + * types for `Localisations` + * @typedef {{ [k: string]: PackageStrings }} AllStrings + * @typedef {{ [k: string]: PackageASTCache }} AllAstCache + */ +/** + * types for `PackageLocalisations` + * @typedef {{ [k: string]: Strings }} PackageStrings + * @typedef {{ [k: string]: ASTCache }} PackageASTCache + */ +/** + * used in `SingleLanguageLocalisations` + * @typedef {{ [k: string]: string | Strings }} Strings + * @typedef {ManyASTs} ASTCache + * @typedef {{ [k: string]: IntlMessageFormat | FormatterCache }} FormatterCache + */ diff --git a/src/package-manager.js b/src/package-manager.js index 23a8e904dc..372276357c 100644 --- a/src/package-manager.js +++ b/src/package-manager.js @@ -38,7 +38,8 @@ module.exports = class PackageManager { grammarRegistry: this.grammarRegistry, deserializerManager: this.deserializerManager, viewRegistry: this.viewRegistry, - uriHandlerRegistry: this.uriHandlerRegistry + uriHandlerRegistry: this.uriHandlerRegistry, + i18n: this.i18n } = params); this.emitter = new Emitter(); @@ -587,7 +588,8 @@ module.exports = class PackageManager { menuManager: this.menuManager, contextMenuManager: this.contextMenuManager, deserializerManager: this.deserializerManager, - viewRegistry: this.viewRegistry + viewRegistry: this.viewRegistry, + i18n: this.i18n }; pack = metadata.theme ? new ThemePackage(options) : new Package(options); @@ -692,7 +694,8 @@ module.exports = class PackageManager { menuManager: this.menuManager, contextMenuManager: this.contextMenuManager, deserializerManager: this.deserializerManager, - viewRegistry: this.viewRegistry + viewRegistry: this.viewRegistry, + i18n: this.i18n }; const pack = metadata.theme diff --git a/src/package.js b/src/package.js index 0071472dc2..db9534c6ad 100644 --- a/src/package.js +++ b/src/package.js @@ -30,6 +30,7 @@ module.exports = class Package { this.contextMenuManager = params.contextMenuManager; this.deserializerManager = params.deserializerManager; this.viewRegistry = params.viewRegistry; + this.i18n = params.i18n; this.emitter = new Emitter(); this.mainModule = null; @@ -45,6 +46,7 @@ module.exports = class Package { (this.metadata && this.metadata.name) || params.name || path.basename(this.path); + this.t = this.i18n.getT(this.name); this.reset(); } @@ -240,7 +242,10 @@ module.exports = class Package { } if (typeof this.mainModule.activate === 'function') { this.mainModule.activate( - this.packageManager.getPackageState(this.name) || {} + this.packageManager.getPackageState(this.name) || {}, + { + t: this.t + } ); } this.mainActivated = true; diff --git a/yarn.lock b/yarn.lock index 845064f393..9368b62924 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1431,6 +1431,45 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@formatjs/ecma402-abstract@1.17.1": + version "1.17.1" + resolved "https://registry.yarnpkg.com/@formatjs/ecma402-abstract/-/ecma402-abstract-1.17.1.tgz#6ac7d6a1d1c9c8eff76ab6ed949f2a5cbe424030" + integrity sha512-N2sjSUrmsEoynG8Q61pkrKlJ9PxcUGxJke1x3301aGyprGgl58wHWhgGUnzTfS4OHNNNQDxzjcXVp1t5fGW6yQ== + dependencies: + "@formatjs/intl-localematcher" "0.4.1" + tslib "^2.4.0" + +"@formatjs/fast-memoize@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@formatjs/fast-memoize/-/fast-memoize-2.2.0.tgz#33bd616d2e486c3e8ef4e68c99648c196887802b" + integrity sha512-hnk/nY8FyrL5YxwP9e4r9dqeM6cAbo8PeU9UjyXojZMNvVad2Z06FAVHyR3Ecw6fza+0GH7vdJgiKIVXTMbSBA== + dependencies: + tslib "^2.4.0" + +"@formatjs/icu-messageformat-parser@2.6.1", "@formatjs/icu-messageformat-parser@^2.6.1": + version "2.6.1" + resolved "https://registry.yarnpkg.com/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.6.1.tgz#ca497d5a2bff641dc0978bd9b64d1d02597980cb" + integrity sha512-dTDNupwdovxT1xDXC96zzPUua/XrxTQTOulJZSvaJP0pt3rr/cGR/tq4d7BnxY9oqPZpc4fjWBmrRlhcUyBSiw== + dependencies: + "@formatjs/ecma402-abstract" "1.17.1" + "@formatjs/icu-skeleton-parser" "1.6.1" + tslib "^2.4.0" + +"@formatjs/icu-skeleton-parser@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.6.1.tgz#3647f41b82e362c08bb80bd9b653c7eb6ff31118" + integrity sha512-/LQ6ovxYd8FQjVLmbV+WmuFy86o+JTc0cIQuWixuLuUMfRRif8eUQw3vPK5hx7C/g1UVmKAaOcYRTEsvyEGz9g== + dependencies: + "@formatjs/ecma402-abstract" "1.17.1" + tslib "^2.4.0" + +"@formatjs/intl-localematcher@0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@formatjs/intl-localematcher/-/intl-localematcher-0.4.1.tgz#af63e2c065731a33f6fed36dc85058009a7f8062" + integrity sha512-Fs4MhhHlLC0RrspX2u2KP7zlwL9eHrBZsOBxaPOeqrCZYLaOUK4cYXQ1ErpAB0HnGV/GUXNa5smzV/7jCuRzxg== + dependencies: + tslib "^2.4.0" + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -1895,7 +1934,6 @@ abbrev@1: version "1.9.1" dependencies: etch "^0.14.1" - semver "^7.3.8" acorn-jsx@^5.3.2: version "5.3.2" @@ -5484,6 +5522,16 @@ interpret@^1.0.0: resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== +intl-messageformat@^10.5.1: + version "10.5.1" + resolved "https://registry.yarnpkg.com/intl-messageformat/-/intl-messageformat-10.5.1.tgz#40304cbde01c8cb2236e11ac8c827642bed474d0" + integrity sha512-irEmjxHq0f1MHviQr3Q4ToF9EgYbnXDq2/R9MRTTveGKHgy6VZ29hQxswu4trqWaX7T6njKxSoKVG92OSz0U5Q== + dependencies: + "@formatjs/ecma402-abstract" "1.17.1" + "@formatjs/fast-memoize" "2.2.0" + "@formatjs/icu-messageformat-parser" "2.6.1" + tslib "^2.4.0" + invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" @@ -9497,6 +9545,11 @@ tslib@^2.3.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== +tslib@^2.4.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"