From 86eed66cb5a5dfff0f635b859fb510f3dd4fc2f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Feb 2021 06:02:41 +0000 Subject: [PATCH 001/164] build(deps-dev): bump eslint-plugin-jest from 24.1.3 to 24.1.5 (#2961) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index af9ecc6fd32c..88233dd22851 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "downshift": "^6.1.0", "eslint": "^7.18.0", "eslint-gitignore": "^0.1.0", - "eslint-plugin-jest": "24.1.3", + "eslint-plugin-jest": "24.1.5", "eslint-plugin-node": "11.1.0", "extend": "^3.0.2", "flexsearch": "0.6.32", diff --git a/yarn.lock b/yarn.lock index 0dddb01108a0..96cdb5393fd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7414,10 +7414,10 @@ eslint-plugin-import@^2.22.1: resolve "^1.17.0" tsconfig-paths "^3.9.0" -eslint-plugin-jest@24.1.3, eslint-plugin-jest@^24.1.0: - version "24.1.3" - resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-24.1.3.tgz#fa3db864f06c5623ff43485ca6c0e8fc5fe8ba0c" - integrity sha512-dNGGjzuEzCE3d5EPZQ/QGtmlMotqnYWD/QpCZ1UuZlrMAdhG5rldh0N0haCvhGnUkSeuORS5VNROwF9Hrgn3Lg== +eslint-plugin-jest@24.1.5, eslint-plugin-jest@^24.1.0: + version "24.1.5" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-24.1.5.tgz#1e866a9f0deac587d0a3d5d7cefe99815a580de2" + integrity sha512-FIP3lwC8EzEG+rOs1y96cOJmMVpdFNreoDJv29B5vIupVssRi8zrSY3QadogT0K3h1Y8TMxJ6ZSAzYUmFCp2hg== dependencies: "@typescript-eslint/experimental-utils" "^4.0.1" From 64175878e9ac2981fc170f881889cecc269e6c75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Feb 2021 06:57:18 +0000 Subject: [PATCH 002/164] build(deps): bump boto3 from 1.17.8 to 1.17.9 in /deployer (#2962) --- deployer/poetry.lock | 16 ++++++++-------- deployer/pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deployer/poetry.lock b/deployer/poetry.lock index 17c73aa9f556..181076d3108f 100644 --- a/deployer/poetry.lock +++ b/deployer/poetry.lock @@ -52,20 +52,20 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "boto3" -version = "1.17.8" +version = "1.17.9" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] -botocore = ">=1.20.8,<1.21.0" +botocore = ">=1.20.9,<1.21.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "botocore" -version = "1.20.8" +version = "1.20.9" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -479,7 +479,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "c347503855fa42ec5beafa34bc377f27718be50bd95b9e3ed0586a4ab5e71d5d" +content-hash = "d74d4ab60769ad0f7d1f7c1b6ec8327199bb0b75d97808232c0b02bfc2b0a992" [metadata.files] appdirs = [ @@ -498,12 +498,12 @@ black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] boto3 = [ - {file = "boto3-1.17.8-py2.py3-none-any.whl", hash = "sha256:c294eb103db0ab6c55a53f68cc87761ff7e564733e7b23e51dac475a18c0a12b"}, - {file = "boto3-1.17.8.tar.gz", hash = "sha256:819890e92268d730bdef1d8bac08fb069b148bec21f2172a1a99380798224e1b"}, + {file = "boto3-1.17.9-py2.py3-none-any.whl", hash = "sha256:3a8412020a59509e783755b5c9b910a4fc7f6b6f2b9473e7cd1e07b67672e0d1"}, + {file = "boto3-1.17.9.tar.gz", hash = "sha256:877f204dabe1bfa21aa9cfaacc72bd4b70a897d0fdcea799afa5c4743b6fc7ac"}, ] botocore = [ - {file = "botocore-1.20.8-py2.py3-none-any.whl", hash = "sha256:65e2fb81f941dfefa8cc3ef72d46a62d4b6219b8eb6550062b644b36935f6e4f"}, - {file = "botocore-1.20.8.tar.gz", hash = "sha256:cd621cdd14a81d2c3c5276516066d9e6ea20a515cf3113a80ad74f3f8b04a093"}, + {file = "botocore-1.20.9-py2.py3-none-any.whl", hash = "sha256:d725840b881be62fc52e8e24a6ada651128cf7f1ed1639b87322a7a213ffdbad"}, + {file = "botocore-1.20.9.tar.gz", hash = "sha256:c8614c230e7a8e042a8c07d47caea50ad21cb51415289bd34fa6d0382beddad7"}, ] certifi = [ {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, diff --git a/deployer/pyproject.toml b/deployer/pyproject.toml index c309dc4d86bb..e304faaf0f2f 100644 --- a/deployer/pyproject.toml +++ b/deployer/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.7" click = "^7.1.2" -boto3 = "^1.17.8" +boto3 = "^1.17.9" python-decouple = "^3.4" requests = {extras = ["security"], version = "^2.25.0"} elasticsearch-dsl = "^7.3.0" From 63ad3bd5f8732381a29d9d7734621d4d6cdc0666 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Wed, 17 Feb 2021 05:54:41 -0500 Subject: [PATCH 003/164] remove ga.send() debugging (#2959) --- client/src/ga-context.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/src/ga-context.tsx b/client/src/ga-context.tsx index 2780d1a176b9..3c2bacc4b7cb 100644 --- a/client/src/ga-context.tsx +++ b/client/src/ga-context.tsx @@ -45,8 +45,6 @@ declare global { function ga(...args) { if (typeof window === "object" && typeof window.ga === "function") { window.ga(...args); - } else { - console.debug("analytics (not sent)", ...args); } } From 1c61310721f8efbfdf504b8642ad204de21b88bb Mon Sep 17 00:00:00 2001 From: Florian Dieminger Date: Wed, 17 Feb 2021 15:51:59 +0100 Subject: [PATCH 004/164] fix/enhance redirect validation (#2807) * fix/enhance redirect validation * fix tests * add verify-redirects command * fix * Update content/redirect.js Co-authored-by: Peter Bengtsson * feedback * fix to url validation disable casting in tool * more feedback * add --strict to validate-redirects Co-authored-by: Peter Bengtsson --- content/redirect.js | 265 ++++++++++++++++++++++----------------- content/redirect.test.js | 33 ++--- tool/cli.js | 54 ++++++-- 3 files changed, 214 insertions(+), 138 deletions(-) diff --git a/content/redirect.js b/content/redirect.js index 93c4b7f91668..aaca25e8bbed 100644 --- a/content/redirect.js +++ b/content/redirect.js @@ -11,6 +11,7 @@ const { const { isArchivedFilePath } = require("./archive"); const FORBIDDEN_URL_SYMBOLS = ["\n", "\t"]; +const VALID_LOCALES_SET = new Set([...VALID_LOCALES.values()]); function checkURLInvalidSymbols(url) { for (const character of FORBIDDEN_URL_SYMBOLS) { @@ -20,9 +21,14 @@ function checkURLInvalidSymbols(url) { } } +function isVanityRedirectURL(url) { + const localeUrls = new Set([...VALID_LOCALES.values()].map((l) => `/${l}/`)); + return localeUrls.has(url); +} + function resolveDocumentPath(url) { - // Let's keep vanity urls to /en-US/ - if (url === "/en-US/") { + // Let's keep vanity urls to /en-US/ ... + if (isVanityRedirectURL(url)) { return url; } const [bareURL] = url.split("#"); @@ -41,6 +47,12 @@ function resolveDocumentPath(url) { const root = locale === "en-us" ? CONTENT_ROOT : CONTENT_TRANSLATED_ROOT; + if (!root) { + console.log( + `Trying to resolve a non-en-us path for ${url} without CONTENT_TRANSLATED_ROOT set.` + ); + return `$TRANSLATED/${relativeFilePath}`; + } const filePath = path.join(root, relativeFilePath); if (fs.existsSync(filePath)) { return filePath; @@ -49,7 +61,16 @@ function resolveDocumentPath(url) { } // Throw if this can't be a redirect from-URL. -function validateFromURL(url) { +function validateFromURL(url, checkResolve = true) { + if (!url.startsWith("/")) { + throw new Error(`From-URL must start with a / was ${url}`); + } + if (!url.includes("/docs/")) { + throw new Error(`From-URL must contain '/docs/' was ${url}`); + } + if (!VALID_LOCALES_SET.has(url.split("/")[1])) { + throw new Error(`The locale prefix is not valid or wrong case was ${url}`); + } checkURLInvalidSymbols(url); // This is a circular dependency we should solve that in another way. validateURLLocale(url); @@ -57,16 +78,22 @@ function validateFromURL(url) { if (path) { throw new Error(`From-URL resolves to a file (${path})`); } - const resolved = resolve(url); - if (resolved !== url) { - throw new Error( - `${url} is already matched as a redirect (to: '${resolved}')` - ); + if (checkResolve) { + const resolved = resolve(url); + if (resolved !== url) { + throw new Error( + `${url} is already matched as a redirect (to: '${resolved}')` + ); + } } } // Throw if this can't be a redirect to-URL. -function validateToURL(url) { +function validateToURL(url, checkResolve = true, checkPath = true) { + // Let's keep vanity urls to /en-US/ ... + if (isVanityRedirectURL(url)) { + return url; + } // If it's not external, it has to go to a valid document if (url.includes("://")) { // If this throws, conveniently the validator will do its job. @@ -74,29 +101,35 @@ function validateToURL(url) { if (parsedURL.protocol !== "https:") { throw new Error("We only redirect to https://"); } - } else { + } else if (url.startsWith("/")) { checkURLInvalidSymbols(url); validateURLLocale(url); - // Can't point to something that redirects to something - const resolved = resolve(url); - if (resolved !== url) { - throw new Error( - `${url} is already matched as a redirect (to: '${resolved}')` - ); + if (checkResolve) { + // Can't point to something that redirects to something + const resolved = resolve(url); + if (resolved !== url) { + throw new Error( + `${url} is already matched as a redirect (to: '${resolved}')` + ); + } } - const path = resolveDocumentPath(url); - if (!path) { - throw new Error(`To-URL has to resolve to a file (${path})`); + if (checkPath) { + const path = resolveDocumentPath(url); + if (!path) { + throw new Error(`To-URL has to resolve to a file (${url})`); + } } + } else { + throw new Error(`To-URL has to be external or start with / (${url})`); } } function validateURLLocale(url) { // Check that it's a valid document URL - const locale = url.split("/")[1]; - if (!locale || url.split("/")[2] !== "docs") { - throw new Error("The URL is expected to be /$locale/docs/"); + const [nothing, locale, docs] = url.split("/"); + if (nothing || !locale || docs !== "docs") { + throw new Error(`The URL is expected to start with /$locale/docs/: ${url}`); } const validValues = [...VALID_LOCALES.values()]; if (!validValues.includes(locale)) { @@ -104,6 +137,18 @@ function validateURLLocale(url) { } } +function errorOnEncoded(paris) { + for (const [from, to] of paris) { + const [decodedFrom, decodedTo] = decodePair([from, to]); + if (decodedFrom !== from) { + throw new Error(`From URL must be decoded: ${from}`); + } + if (decodedTo !== to) { + throw new Error(`To URL must be decoded: ${to}`); + } + } +} + function errorOnDuplicated(pairs) { const seen = new Set(); for (const [from] of pairs) { @@ -146,7 +191,28 @@ function removeOrphanedRedirects(pairs) { }); } -function add(locale, updatePairs, { fix = false } = {}) { +function loadPairsFromFile(filePath, strict = true) { + const content = fs.readFileSync(filePath, "utf-8"); + const pairs = content + .trim() + .split("\n") + // Skip the header line. + .slice(1) + .map((line) => line.trim().split(/\t+/)); + + if (strict) { + errorOnEncoded(pairs); + errorOnDuplicated(pairs); + } + validatePairs(pairs, strict); + return pairs; +} + +function loadLocaleAndAdd(locale, updatePairs, { fix = false } = {}) { + errorOnEncoded(updatePairs); + errorOnDuplicated(updatePairs); + validatePairs(updatePairs); + locale = locale.toLowerCase(); let root = CONTENT_ROOT; if (locale !== "en-us") { @@ -161,100 +227,69 @@ function add(locale, updatePairs, { fix = false } = {}) { const redirectsFilePath = path.join(root, locale, "_redirects.txt"); const pairs = []; if (fs.existsSync(redirectsFilePath)) { - const content = fs.readFileSync(redirectsFilePath, "utf-8"); - pairs.push( - ...content - .trim() - .split("\n") - // Skip the header line. - .slice(1) - .map((line) => line.trim().split(/\t+/)) - ); + // If we wanna fix we load relaxed, hence the !fix. + pairs.push(...loadPairsFromFile(redirectsFilePath, !fix)); } - const decodedUpdatePairs = decodePairs(updatePairs); - const decodedPairs = decodePairs(pairs); - - errorOnDuplicated(decodedPairs); - errorOnDuplicated(decodedUpdatePairs); - - const cleanPairs = removeConflictingOldRedirects( - decodedPairs, - decodedUpdatePairs - ); - cleanPairs.push(...decodedUpdatePairs); + const cleanPairs = removeConflictingOldRedirects(pairs, updatePairs); + cleanPairs.push(...updatePairs); let simplifiedPairs = shortCuts(cleanPairs); if (fix) { simplifiedPairs = removeOrphanedRedirects(simplifiedPairs); } - save(path.join(root, locale), simplifiedPairs); + validatePairs(simplifiedPairs); + + return { pairs: simplifiedPairs, root, changed: simplifiedPairs == pairs }; } -// The module level cache -const redirects = new Map(); +function add(locale, updatePairs, { fix = false } = {}) { + const { pairs, root } = loadLocaleAndAdd(locale, updatePairs, { fix }); + save(path.join(root, locale), pairs); +} -function load(files = null, verbose = false) { - if (!files) { - const localeFolders = fs - .readdirSync(CONTENT_ROOT) - .map((n) => path.join(CONTENT_ROOT, n)) - .filter((filepath) => fs.statSync(filepath).isDirectory()); - if (CONTENT_TRANSLATED_ROOT) { - const translatedLocaleFolders = fs - .readdirSync(CONTENT_TRANSLATED_ROOT) - .map((n) => path.join(CONTENT_TRANSLATED_ROOT, n)) - .filter((filepath) => fs.statSync(filepath).isDirectory()); - localeFolders.push(...translatedLocaleFolders); - } +function validateLocale(locale, strict = false) { + // To validate strict we check if there is something to fix. + const { changed } = loadLocaleAndAdd(locale, [], { fix: strict }); + if (changed) { + throw new Error(` _redirects.txt for ${locale} is flawed`); + } +} - files = localeFolders - .map((folder) => path.join(folder, "_redirects.txt")) - .filter((filePath) => fs.existsSync(filePath)); +function redirectFilePathForLocale(locale, throws = false) { + const makeFilePath = (root) => + path.join(root, locale.toLowerCase(), "_redirects.txt"); + + const filePath = makeFilePath(CONTENT_ROOT); + if (fs.existsSync(filePath)) { + return filePath; } + if (CONTENT_TRANSLATED_ROOT) { + const translatedFilePath = makeFilePath(CONTENT_TRANSLATED_ROOT); - function throwError(message, lineNumber, line) { - throw new Error(`Invalid line: ${message} (line ${lineNumber}) '${line}'`); + if (fs.existsSync(translatedFilePath)) { + return translatedFilePath; + } + } + if (throws) { + throw new Error(`no _redirects file for ${locale}`); } + return null; +} + +// The module level cache +const redirects = new Map(); - const validLocales = new Set([...VALID_LOCALES.values()]); +function load(locales = [...VALID_LOCALES.keys()], verbose = false) { + const files = locales + .map((locale) => redirectFilePathForLocale(locale)) + .filter((f) => f !== null); for (const redirectsFilePath of files) { if (verbose) { console.log(`Checking ${redirectsFilePath}`); } - const content = fs.readFileSync(redirectsFilePath, "utf-8"); - if (!content.endsWith("\n")) { - throw new Error( - `${redirectsFilePath} must have a trailing newline character.` - ); - } - const pairs = new Map(); - // Parse and collect all and throw errors on bad lines - content.split("\n").forEach((line, i) => { - if (!line.trim() || line.startsWith("#")) return; - const split = line.trim().split(/\t/); - if (split.length !== 2) { - throwError("Not two strings split by tab", i + 1, line); - } - const [from, to] = split; - if (!from.startsWith("/")) { - throwError("From-URL must start with a /", i + 1, line); - } - if (!from.includes("/docs/")) { - throwError("From-URL must contain '/docs/'", i + 1, line); - } - if (!validLocales.has(from.split("/")[1])) { - throwError( - `The locale prefix is not valid or wrong case '${ - from.split("/")[1] - }'.`, - i + 1, - line - ); - } - pairs.set(from.toLowerCase(), to); - }); + const pairs = loadPairsFromFile(redirectsFilePath, false); // Now that all have been collected, transfer them to the `redirects` map // but also do invariance checking. for (const [from, to] of pairs) { @@ -347,23 +382,26 @@ function shortCuts(pairs, throws = false) { return mappedPairs; } +function decodePair([from, to]) { + const fromDecoded = decodePath(from); + let toDecoded; + if (to.startsWith("/")) { + toDecoded = decodePath(to); + } else { + toDecoded = decodeURI(to); + } + return [fromDecoded, toDecoded]; +} + function decodePairs(pairs) { - return pairs.map(([from, to]) => { - const fromDecoded = decodePath(from); - let toDecoded; - if (to.startsWith("/")) { - toDecoded = decodePath(to); - } else { - toDecoded = decodeURI(to); - } - if ( - checkURLInvalidSymbols(from) || - (to.startsWith("/") && checkURLInvalidSymbols(to)) - ) { - throw new Error(`${from}\t${to} contains invalid symbols`); - } - return [fromDecoded, toDecoded]; - }); + return pairs.map((pair) => decodePair(pair)); +} + +function validatePairs(pairs, checkExists = true) { + for (const [from, to] of pairs) { + validateFromURL(from, false); + validateToURL(to, false, checkExists); + } } function save(localeFolder, pairs) { @@ -382,6 +420,7 @@ module.exports = { load, validateFromURL, validateToURL, + validateLocale, testing: { shortCuts, diff --git a/content/redirect.test.js b/content/redirect.test.js index 016cf945eb17..d061a4b4b5ad 100644 --- a/content/redirect.test.js +++ b/content/redirect.test.js @@ -3,25 +3,25 @@ const Redirect = require("./redirect"); describe("short cuts", () => { it("simple chain", () => { const r = Redirect.testing.shortCuts([ - ["A", "B"], - ["B", "C"], + ["/en-US/docs/A", "/en-US/docs/B"], + ["/en-US/docs/B", "/en-US/docs/C"], ]); expect(r).toEqual([ - ["A", "C"], - ["B", "C"], + ["/en-US/docs/A", "/en-US/docs/C"], + ["/en-US/docs/B", "/en-US/docs/C"], ]); }); it("a = a", () => { const r = Redirect.testing.shortCuts([ - ["A", "A"], - ["b", "B"], + ["/en-US/docs/A", "/en-US/docs/A"], + ["/en-US/docs/b", "/en-US/docs/B"], ]); expect(r).toEqual([]); }); it("simple cycle", () => { const r = Redirect.testing.shortCuts([ - ["A", "B"], - ["B", "A"], + ["/en-US/docs/A", "/en-US/docs/B"], + ["/en-US/docs/B", "/en-US/docs/A"], ]); expect(r).toEqual([]); }); @@ -30,18 +30,23 @@ describe("short cuts", () => { describe("decode", () => { it("decode internal", () => { const r = Redirect.testing.decodePairs([ - ["/%40/%20/", "/%3Cfoo%3E"], - ["B", "C"], + ["/en-US/docs/%40/%20/", "/en-US/docs/%3Cfoo%3E"], + ["/en-US/docs/B", "/en-US/docs/C"], ]); expect(r).toEqual([ - ["/@/ /", "/"], - ["B", "C"], + ["/en-US/docs/@/ /", "/en-US/docs/"], + ["/en-US/docs/B", "/en-US/docs/C"], ]); }); it("decode internal", () => { const r = Redirect.testing.decodePairs([ - ["/some", "https://foo%40bar.com:foobar@mdn/%20%3A/%F0%9F%94%A5"], + [ + "/en-US/docs/some", + "https://foo%40bar.com:foobar@mdn/%20%3A/%F0%9F%94%A5", + ], + ]); + expect(r).toEqual([ + ["/en-US/docs/some", "https://foo%40bar.com:foobar@mdn/ %3A/🔥"], ]); - expect(r).toEqual([["/some", "https://foo%40bar.com:foobar@mdn/ %3A/🔥"]]); }); }); diff --git a/tool/cli.js b/tool/cli.js index 566472f93fad..926a51466de0 100644 --- a/tool/cli.js +++ b/tool/cli.js @@ -46,11 +46,43 @@ program .name("tool") .version("0.0.0") .disableGlobalOption("--silent") - .command("validate-redirects", "Check the _redirects.txt file(s)") + .cast(false) + .command("validate-redirects", "Try loading the _redirects.txt file(s)") + .argument("[locales...]", "Locale", { + default: [...VALID_LOCALES.keys()], + validator: [...VALID_LOCALES.keys()], + }) + .option("--strict", "Strict validation") .action( - tryOrExit(({ logger }) => { - Redirect.load(null, true); - logger.info(chalk.green("🍾 All is well in the world of redirects 🥂")); + tryOrExit(({ args, options, logger }) => { + const { locales } = args; + const { strict } = options; + let fine = true; + if (strict) { + for (const locale of locales) { + try { + Redirect.validateLocale(locale); + logger.info(chalk.green(`✓ redirects for ${locale} looking good!`)); + } catch (e) { + logger.info( + chalk.red(`_redirects.txt for ${locale} is causing issues: ${e}`) + ); + fine = false; + } + } + } else { + try { + Redirect.load(locales, true); + } catch (e) { + logger.info(chalk.red(`Unable to load redirects: ${e}`)); + fine = false; + } + } + if (fine) { + logger.info(chalk.green("🍾 All is well in the world of redirects 🥂")); + } else { + throw new Error("🔥 Errors loading redirects 🔥"); + } }) ) @@ -72,13 +104,13 @@ program .command("add-redirect", "Add a new redirect") .argument("", "From-URL", { validator: (value) => { - Redirect.validateFromURL(value); + Redirect.validateFromURL(value, false); return value; }, }) .argument("", "To-URL", { validator: (value) => { - Redirect.validateToURL(value); + Redirect.validateToURL(value, false); return value; }, }) @@ -92,16 +124,16 @@ program ) .command("fix-redirects", "Consolidate/fix redirects") - .argument("", "Locale", { + .argument("", "Locale", { default: [DEFAULT_LOCALE], validator: [...VALID_LOCALES.values(), ...VALID_LOCALES.keys()], }) .action( tryOrExit(({ args, logger }) => { - const { locale } = args; - for (const l of locale) { - Redirect.add(l.toLowerCase(), [], { fix: true }); - logger.info(chalk.green(`Fixed ${l}`)); + const { locales } = args; + for (const locale of locales) { + Redirect.add(locale.toLowerCase(), [], { fix: true }); + logger.info(chalk.green(`Fixed ${locale}`)); } }) ) From c0a4c08e03ea73c5d81867d3f7684b89d7366bfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Feb 2021 06:57:09 +0000 Subject: [PATCH 005/164] build(deps): bump boto3 from 1.17.9 to 1.17.10 in /deployer (#2970) --- deployer/poetry.lock | 16 ++++++++-------- deployer/pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deployer/poetry.lock b/deployer/poetry.lock index 181076d3108f..ff97d35fa9bb 100644 --- a/deployer/poetry.lock +++ b/deployer/poetry.lock @@ -52,20 +52,20 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "boto3" -version = "1.17.9" +version = "1.17.10" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] -botocore = ">=1.20.9,<1.21.0" +botocore = ">=1.20.10,<1.21.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] name = "botocore" -version = "1.20.9" +version = "1.20.10" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -479,7 +479,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "d74d4ab60769ad0f7d1f7c1b6ec8327199bb0b75d97808232c0b02bfc2b0a992" +content-hash = "2cc0da4b3be19f937d038ceb7e3668f6aaa82b754e525394b8936958f016cfe9" [metadata.files] appdirs = [ @@ -498,12 +498,12 @@ black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] boto3 = [ - {file = "boto3-1.17.9-py2.py3-none-any.whl", hash = "sha256:3a8412020a59509e783755b5c9b910a4fc7f6b6f2b9473e7cd1e07b67672e0d1"}, - {file = "boto3-1.17.9.tar.gz", hash = "sha256:877f204dabe1bfa21aa9cfaacc72bd4b70a897d0fdcea799afa5c4743b6fc7ac"}, + {file = "boto3-1.17.10-py2.py3-none-any.whl", hash = "sha256:1709ff5feb363fee7fcaa2330e659fcbc2b4c03a14f75a884ed682ee66011fc4"}, + {file = "boto3-1.17.10.tar.gz", hash = "sha256:80a761eff3b1cb0798d7e1a41b7c8e6d85c9647a8f7b6105335201a69404caa2"}, ] botocore = [ - {file = "botocore-1.20.9-py2.py3-none-any.whl", hash = "sha256:d725840b881be62fc52e8e24a6ada651128cf7f1ed1639b87322a7a213ffdbad"}, - {file = "botocore-1.20.9.tar.gz", hash = "sha256:c8614c230e7a8e042a8c07d47caea50ad21cb51415289bd34fa6d0382beddad7"}, + {file = "botocore-1.20.10-py2.py3-none-any.whl", hash = "sha256:a601ee5a4ae66832f328ca362b5404d22b75f1c181f6cc0934f3cfca749eb27d"}, + {file = "botocore-1.20.10.tar.gz", hash = "sha256:8c84eac6daf38890714e005623083106d68e9b2088e62132fdbf7d2b1228ecbd"}, ] certifi = [ {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, diff --git a/deployer/pyproject.toml b/deployer/pyproject.toml index e304faaf0f2f..8346e7db21ef 100644 --- a/deployer/pyproject.toml +++ b/deployer/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.7" click = "^7.1.2" -boto3 = "^1.17.9" +boto3 = "^1.17.10" python-decouple = "^3.4" requests = {extras = ["security"], version = "^2.25.0"} elasticsearch-dsl = "^7.3.0" From 752df37dbe75ef4ff0ad1d72f0ad1ee5ce9779a4 Mon Sep 17 00:00:00 2001 From: Florian Dieminger Date: Thu, 18 Feb 2021 16:22:27 +0100 Subject: [PATCH 006/164] fix handling of hashes in redirects (#2965) --- content/redirect.js | 41 +++++++++++++++++++++++++++++++++------- content/redirect.test.js | 10 ++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/content/redirect.js b/content/redirect.js index aaca25e8bbed..0cb19fbec786 100644 --- a/content/redirect.js +++ b/content/redirect.js @@ -105,19 +105,20 @@ function validateToURL(url, checkResolve = true, checkPath = true) { checkURLInvalidSymbols(url); validateURLLocale(url); + const [bareURL] = url.split("#"); if (checkResolve) { // Can't point to something that redirects to something - const resolved = resolve(url); - if (resolved !== url) { + const resolved = resolve(bareURL); + if (resolved !== bareURL) { throw new Error( - `${url} is already matched as a redirect (to: '${resolved}')` + `${bareURL} is already matched as a redirect (to: '${resolved}')` ); } } if (checkPath) { - const path = resolveDocumentPath(url); + const path = resolveDocumentPath(bareURL); if (!path) { - throw new Error(`To-URL has to resolve to a file (${url})`); + throw new Error(`To-URL has to resolve to a file (${bareURL})`); } } } else { @@ -322,6 +323,7 @@ function shortCuts(pairs, throws = false) { from.toLowerCase(), to.toLowerCase(), ]); + const hashPairs = pairs.filter(([, to]) => to.includes("#")); // Directed graph of all redirects. const dg = new Map(lowerCasePairs); @@ -371,12 +373,37 @@ function shortCuts(pairs, throws = false) { transitiveDag.set(from, to); } } + + // We want to shortcut + // /en-US/docs/foo/bar /en-US/docs/foo#bar + // /en-US/docs/foo /en-US/docs/Web/something + // to + // /en-US/docs/foo/bar /en-US/docs/something#bar + // /en-US/docs/foo /en-US/docs/Web/something + for (const [from, to] of hashPairs) { + const [bareTo, ...hashes] = to.split("#"); + const bareToLC = bareTo.toLowerCase(); + if (transitiveDag.has(bareToLC)) { + const redirectedTo = transitiveDag.get(bareToLC); + const newTo = `${redirectedTo}#${hashes.join("#").toLowerCase()}`; + + // Add casing for the hashed new URL. + const redirectedToCased = casing.get(redirectedTo); + const newToCased = `${redirectedToCased}#${hashes.join("#")}`; + casing.set(newTo, newToCased); + + // Log something since this is opportunistic! + console.log(`Short cutting hashed redirect: ${from} -> ${newTo}`); + transitiveDag.set(from.toLowerCase(), newTo); + } + } + const transitivePairs = [...transitiveDag.entries()]; // Restore cases! const mappedPairs = transitivePairs.map(([from, to]) => [ - casing.get(from), - casing.get(to), + casing.get(from) || from, + casing.get(to) || to, ]); mappedPairs.sort(sortTuples); return mappedPairs; diff --git a/content/redirect.test.js b/content/redirect.test.js index d061a4b4b5ad..7ca301aa7644 100644 --- a/content/redirect.test.js +++ b/content/redirect.test.js @@ -25,6 +25,16 @@ describe("short cuts", () => { ]); expect(r).toEqual([]); }); + it("hashes", () => { + const r = Redirect.testing.shortCuts([ + ["/en-US/docs/A", "/en-US/docs/B#Foo"], + ["/en-US/docs/B", "/en-US/docs/C"], + ]); + expect(r).toEqual([ + ["/en-US/docs/A", "/en-US/docs/C#Foo"], + ["/en-US/docs/B", "/en-US/docs/C"], + ]); + }); }); describe("decode", () => { From 5b2ced538d0b01d448db3187be9ca9bb1a50e94f Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Thu, 18 Feb 2021 12:53:29 -0500 Subject: [PATCH 007/164] push google analytics to run later (#2891) * Push Google Analytics to run later Fixes #2809 * tidying up * finishing * remove dead code * fix test * fix eslint error --- client/src/app.tsx | 3 -- client/src/constants.ts | 4 -- client/src/document/index.tsx | 9 ++-- client/src/ga-context.tsx | 23 ----------- client/src/site-search/index.tsx | 8 ++-- docs/envvars.md | 10 ----- docs/google-analytics.md | 70 ++++++++++++++++++++++++++++++++ package.json | 2 +- ssr/mozilla.dnthelper.min.js | 1 - ssr/render.js | 64 +++++++++++++++-------------- testing/tests/index.test.js | 4 +- tool/cli.js | 69 +++++++++++++++++++++++++++++++ tool/mozilla.dnthelper.min.js | 33 +++++++++++++++ 13 files changed, 217 insertions(+), 83 deletions(-) create mode 100644 docs/google-analytics.md delete mode 100644 ssr/mozilla.dnthelper.min.js create mode 100644 tool/mozilla.dnthelper.min.js diff --git a/client/src/app.tsx b/client/src/app.tsx index c362f7649e34..638eda6488ac 100644 --- a/client/src/app.tsx +++ b/client/src/app.tsx @@ -15,7 +15,6 @@ import { SiteSearch } from "./site-search"; import { PageContentContainer } from "./ui/atoms/page-content"; import { PageNotFound } from "./page-not-found"; import { Banner } from "./banners"; -import { useDebugGA } from "./ga-context"; const AllFlaws = React.lazy(() => import("./flaws")); const DocumentEdit = React.lazy(() => import("./document/forms/edit")); @@ -91,8 +90,6 @@ function LoadingFallback({ message }: { message?: string }) { } export function App(appProps) { - useDebugGA(); - // When preparing a build for use in the NPM package, CRUD_MODE is always true. // But if the App is loaded from the code that builds the SPAs, then `isServer` // is true. So you have to have `isServer && CRUD_MODE` at the same time. diff --git a/client/src/constants.ts b/client/src/constants.ts index 25279fe96654..caf3cefad5b0 100644 --- a/client/src/constants.ts +++ b/client/src/constants.ts @@ -11,10 +11,6 @@ export const AUTOCOMPLETE_SEARCH_WIDGET = JSON.parse( process.env.REACT_APP_AUTOCOMPLETE_SEARCH_WIDGET || JSON.stringify(CRUD_MODE) ); -export const DEBUG_GOOGLE_ANALYTICS = JSON.parse( - process.env.REACT_APP_DEBUG_GOOGLE_ANALYTICS || "false" -); - // You can read more about this in `docs/debugging-sitesearch.md`. export const DEBUG_SEARCH_RESULTS = JSON.parse( process.env.REACT_APP_DEBUG_SEARCH_RESULTS || "false" diff --git a/client/src/document/index.tsx b/client/src/document/index.tsx index d741a46d7bd7..7ee09cda1e22 100644 --- a/client/src/document/index.tsx +++ b/client/src/document/index.tsx @@ -85,11 +85,12 @@ export function Document(props /* TODO: define a TS interface for this */) { // Note that in local development, where you use `localhost:3000` // this will always be true because it's always client-side navigation. ga("set", "dimension19", "Yes"); + ga("send", { + hitType: "pageview", + location: window.location.toString(), + }); } - ga("send", { - hitType: "pageview", - location: window.location.toString(), - }); + // By counting every time a document is mounted, we can use this to know if // a client-side navigation happened. mountCounter.current++; diff --git a/client/src/ga-context.tsx b/client/src/ga-context.tsx index 3c2bacc4b7cb..0e16d1bf7a37 100644 --- a/client/src/ga-context.tsx +++ b/client/src/ga-context.tsx @@ -1,8 +1,6 @@ import * as React from "react"; import { useContext, useEffect, useState } from "react"; -import { DEBUG_GOOGLE_ANALYTICS } from "./constants"; - export type GAFunction = (...any) => void; export const CATEGORY_LEARNING_SURVEY = "learning web development"; @@ -97,27 +95,6 @@ export function useClientId() { return clientId; } -// This only really exists so you can debug Google Analytics when running the -// debug server (localhost:3000) otherwise you won't get useful logging in -// the Web Console when your code does things like `ga("send", ...)`. -// See the REACT_APP_DEBUG_GOOGLE_ANALYTICS in docs/envvars.md for more info. -export function useDebugGA() { - useEffect(() => { - if (DEBUG_GOOGLE_ANALYTICS) { - const internalScript = document.createElement("script"); - internalScript.textContent = ` - window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date; - ga('create', 'UA-00000000-0', 'mozilla.org');`.trim(); - document.head.appendChild(internalScript); - const externalScript = document.createElement("script"); - externalScript.src = - "https://www.google-analytics.com/analytics_debug.js"; - externalScript.async = true; - document.head.appendChild(externalScript); - } - }, []); -} - export function useGA() { return useContext(GAContext); } diff --git a/client/src/site-search/index.tsx b/client/src/site-search/index.tsx index cdc6040c834b..2b8c885ac2ae 100644 --- a/client/src/site-search/index.tsx +++ b/client/src/site-search/index.tsx @@ -36,11 +36,11 @@ export function SiteSearch() { // Note that in local development, where you use `localhost:3000` // this will always be true because it's always client-side navigation. ga("set", "dimension19", "Yes"); + ga("send", { + hitType: "pageview", + location: window.location.toString(), + }); } - ga("send", { - hitType: "pageview", - location: window.location.toString(), - }); // By counting every time a document is mounted, we can use this to know if // a client-side navigation happened. mountCounter.current++; diff --git a/docs/envvars.md b/docs/envvars.md index d49382c10bee..934e24408219 100644 --- a/docs/envvars.md +++ b/docs/envvars.md @@ -283,13 +283,3 @@ Toolbar bar appears based on this. It defaults to `NODE_ENV==='development'` if not set which means that it's enable by default when doing development with the `localhost:3000` dev server. - -### `REACT_APP_DEBUG_GOOGLE_ANALYTICS` - -**Default: `false`** - -When you use the `create-react-app` server on `localhost:3000` it can't -inject the Google Analytics script like you can when you server-side -render (see `ssr/render.js`). By setting this to `true` it will forcibly -inject a `` -tag and the necessary code to activate it. diff --git a/docs/google-analytics.md b/docs/google-analytics.md new file mode 100644 index 000000000000..0e91f2ba393c --- /dev/null +++ b/docs/google-analytics.md @@ -0,0 +1,70 @@ +# Google Analytics + +We use Google Analytics for counting pageviews and for sending arbitrary events +from the client-side code. The way it works is that you have to send an environment +variable, like: + +```bash +BUILD_GOOGLE_ANALYTICS_ACCOUNT=UA-1234678-0 +``` + +and that gets included in the build by more or less code-generating the snippet +we use to set up Google Analytics. + +By default, it's not set and that means no Google Analytics JavaScript code inside +the rendered final HTML. By setting the environment variable, a build-step will +generate a `/static/js/ga.js` file that configures how we enable Google Analytics. +And the server-side rendering will inject a `` - ).appendTo($("head")); - } + // As part of the pre-build steps, in the build root, a `ga.js` file is generated. + // The SSR rendering needs to know if exists and if so, what it's URL pathname is. + // The script will do two things: + // 1. created a `window.ga` object + // 2. async inject the download of that remote + // https://www.google-analytics.com/analytics.js file. + // With this script appearing before any other (also deferred) JS bundles, + // the `window.ga` will be immediately available but the remote analytics.js + // can come in when it comes in and it will send. + const gaScriptPathName = getGAScriptPathName(); + if (gaScriptPathName) { + $(" - -`; - return ( - <> -

Result

- `; +str = ``; %> <%- str %> diff --git a/kumascript/macros/EmbedLiveSample.ejs b/kumascript/macros/EmbedLiveSample.ejs index 7c0081a4fda6..30d5b695cc07 100644 --- a/kumascript/macros/EmbedLiveSample.ejs +++ b/kumascript/macros/EmbedLiveSample.ejs @@ -73,7 +73,7 @@ if (hasScreenshot) { %><% %><% } // end hasScreenshot -%>" @@ -29,7 +29,7 @@ describeMacro("EmbedLiveSample", function () { macro.ctx.env.url = "/en-US/docs/Web/SVG/Element/switch"; return assert.eventually.equal( macro.call("SVG_<switch>_example"), - '" @@ -39,7 +39,7 @@ describeMacro("EmbedLiveSample", function () { macro.ctx.env.url = "/en-US/docs/Web/SVG/Element/switch"; return assert.eventually.equal( macro.call("SVG_%3Cswitch%3E_example"), - '" @@ -50,7 +50,7 @@ describeMacro("EmbedLiveSample", function () { "/fr/docs/Web/CSS/Utilisation_de_d%C3%A9grad%C3%A9s_CSS"; return assert.eventually.equal( macro.call("Dégradés_linéaires_simples"), - '" @@ -60,7 +60,7 @@ describeMacro("EmbedLiveSample", function () { macro.ctx.env.url = "/en-US/docs/Web/HTML/Element/figure"; return assert.eventually.equal( macro.call('">'), - '" @@ -70,7 +70,7 @@ describeMacro("EmbedLiveSample", function () { macro.ctx.env.url = "/en-US/docs/Web/CSS/border-top-width"; return assert.eventually.equal( macro.call("Example", "100%"), - '" @@ -255,7 +255,7 @@ describeMacro("EmbedLiveSample", function () { "", '">' ), - '" @@ -266,7 +266,7 @@ describeMacro("EmbedLiveSample", function () { macro.ctx.env.url = "/en-US/docs/Web/CSS/-moz-appearance"; return assert.eventually.equal( macro.call("sampleNone", 100, 50, "", "", "nobutton"), - ' + +

Here's a link that contains the string :JavaScript within the href +attribute:
+ + A beginner's guide to SpiderMonkey, Mozilla's JavaScript engine +

+ +
    +
  • I'm
    • +
  • sneaky
    • +
+ + diff --git a/testing/tests/index.test.js b/testing/tests/index.test.js index e2545ab2ed52..1ba331ca4dd9 100644 --- a/testing/tests/index.test.js +++ b/testing/tests/index.test.js @@ -1280,3 +1280,22 @@ test("'lang' attribute should match the article", () => { expect($("html").attr("lang")).toBe("en-US"); expect($("article").attr("lang")).toBe("en-US"); }); + +test("unsafe HTML gets flagged as flaws and replace with its raw HTML", () => { + const builtFolder = path.join( + buildRoot, + "en-us", + "docs", + "web", + "unsafe_html" + ); + + const jsonFile = path.join(builtFolder, "index.json"); + const { doc } = JSON.parse(fs.readFileSync(jsonFile)); + expect(doc.flaws.unsafe_html.length).toBe(5); + + const htmlFile = path.join(builtFolder, "index.html"); + const html = fs.readFileSync(htmlFile, "utf-8"); + const $ = cheerio.load(html); + expect($("code.unsafe-html").length).toBe(5); +}); From 83cf627b71f30f4c104fac546fe4b3f09aef0f88 Mon Sep 17 00:00:00 2001 From: Ryan Johnson Date: Thu, 11 Mar 2021 15:11:23 -0800 Subject: [PATCH 148/164] add tool command for rendering/removing macros (#2955) --- content/document.js | 12 ++-- kumascript/index.js | 24 +++++--- tool/cli.js | 139 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 12 deletions(-) diff --git a/content/document.js b/content/document.js index 5b2e19a7a8d8..472745b4a2ea 100644 --- a/content/document.js +++ b/content/document.js @@ -354,9 +354,11 @@ function findByURL(url, ...args) { return doc; } -function findAll( - { files, folderSearch } = { files: new Set(), folderSearch: null } -) { +function findAll({ + files = new Set(), + folderSearch = null, + quiet = false, +} = {}) { if (!(files instanceof Set)) throw new TypeError("'files' not a Set"); if (folderSearch && typeof folderSearch !== "string") throw new TypeError("'folderSearch' not a string"); @@ -373,7 +375,9 @@ function findAll( roots.push(CONTENT_TRANSLATED_ROOT); } roots.push(CONTENT_ROOT); - console.log("Building roots:", roots); + if (!quiet) { + console.log("Building roots:", roots); + } for (const root of roots) { filePaths.push( ...glob diff --git a/kumascript/index.js b/kumascript/index.js index c802328fa553..a36e06d9977d 100644 --- a/kumascript/index.js +++ b/kumascript/index.js @@ -20,10 +20,17 @@ const DEPENDENCY_LOOP_INTRO = const renderCache = new LRU({ max: 2000 }); -const renderFromURL = async (url, urlsSeen = null) => { +const renderFromURL = async ( + url, + { urlsSeen = null, selective_mode = false, invalidateCache = false } = {} +) => { const urlLC = url.toLowerCase(); if (renderCache.has(urlLC)) { - return renderCache.get(urlLC); + if (invalidateCache) { + renderCache.del(urlLC); + } else { + return renderCache.get(urlLC); + } } urlsSeen = urlsSeen || new Set([]); @@ -34,7 +41,9 @@ const renderFromURL = async (url, urlsSeen = null) => { } urlsSeen.add(urlLC); const prerequisiteErrorsByKey = new Map(); - const document = Document.findByURL(url); + const document = invalidateCache + ? Document.findByURL(url, Document.MEMOIZE_INVALIDATE) + : Document.findByURL(url); if (!document) { throw new Error( `From URL ${url} no folder on disk could be found. ` + @@ -51,7 +60,7 @@ const renderFromURL = async (url, urlsSeen = null) => { slug: metadata.slug, title: metadata.title, tags: metadata.tags || [], - selective_mode: false, + selective_mode, }, interactive_examples: { base_url: INTERACTIVE_EXAMPLES_BASE_URL, @@ -59,10 +68,9 @@ const renderFromURL = async (url, urlsSeen = null) => { live_samples: { base_url: LIVE_SAMPLES_BASE_URL || url }, }, async (url) => { - const [renderedHtml, errors] = await renderFromURL( - info.cleanURL(url), - urlsSeen - ); + const [renderedHtml, errors] = await renderFromURL(info.cleanURL(url), { + urlsSeen, + }); // Remove duplicate flaws. During the rendering process, it's possible for identical // flaws to be introduced when different dependency paths share common prerequisites. // For example, document A may have prerequisite documents B and C, and in turn, diff --git a/tool/cli.js b/tool/cli.js index 8624cc4a84a2..580001d9f596 100644 --- a/tool/cli.js +++ b/tool/cli.js @@ -10,6 +10,7 @@ const { syncAllTranslatedContent, } = require("../build/sync-translated-content"); const log = require("loglevel"); +const cheerio = require("cheerio"); const { DEFAULT_LOCALE, VALID_LOCALES } = require("../libs/constants"); const { @@ -28,6 +29,7 @@ const { GOOGLE_ANALYTICS_DEBUG, } = require("../build/constants"); const { runMakePopularitiesFile } = require("./popularities"); +const kumascript = require("../kumascript"); const PORT = parseInt(process.env.SERVER_PORT || "5000"); @@ -756,6 +758,143 @@ if (Mozilla && !Mozilla.dntEnabled()) { tryOrExit(async ({ options }) => { await buildSPAs(options); }) + ) + + .command( + "macros", + "Render and/or remove one or more macros from one or more documents" + ) + .option("-f, --force", "Render even if there are non-fixable flaws", { + default: false, + }) + .argument("", 'must be either "render" or "remove"') + .argument("", "folder of documents to target") + .argument("", "one or more macro names") + .action( + tryOrExit(async ({ args, options }) => { + if (!CONTENT_ROOT) { + throw new Error("CONTENT_ROOT not set"); + } + if (!CONTENT_TRANSLATED_ROOT) { + throw new Error("CONTENT_TRANSLATED_ROOT not set"); + } + const { force } = options; + const { cmd, foldersearch, macros } = args; + const cmdLC = cmd.toLowerCase(); + if (!["render", "remove"].includes(cmdLC)) { + throw new Error(`invalid macros command "${cmd}"`); + } + console.log( + `${cmdLC} the macro(s) ${macros + .map((m) => `"${m}"`) + .join(", ")} within content folder(s) matching "${foldersearch}"` + ); + const documents = Document.findAll({ + folderSearch: foldersearch, + quiet: true, + }); + if (!documents.count) { + throw new Error("no documents found"); + } + + async function renderOrRemoveMacros(document) { + try { + return await kumascript.render(document.url, { + invalidateCache: true, + selective_mode: [cmdLC, macros], + }); + } catch (error) { + if (error.name === "MacroInvocationError") { + error.updateFileInfo(document.fileInfo); + throw new Error( + `error trying to parse ${error.filepath}, line ${error.line} column ${error.column} (${error.error.message})` + ); + } + // Any other unexpected error re-thrown. + throw error; + } + } + + let countTotal = 0; + let countSkipped = 0; + let countModified = 0; + let countNoChange = 0; + for (const document of documents.iter()) { + countTotal++; + console.group(`${document.fileInfo.path}:`); + const originalRawHTML = document.rawHTML; + let [renderedHTML, flaws] = await renderOrRemoveMacros(document); + if (flaws.length) { + const fixableFlaws = flaws.filter((f) => f.redirectInfo); + const nonFixableFlaws = flaws.filter((f) => !f.redirectInfo); + const nonFixableFlawNames = [ + ...new Set(nonFixableFlaws.map((f) => f.name)).values(), + ].join(", "); + if (force || nonFixableFlaws.length === 0) { + // They're all fixable or we don't care if some or all are + // not, but let's at least fix any that we can. + if (nonFixableFlaws.length > 0) { + console.log( + `ignoring ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})` + ); + } + if (fixableFlaws.length) { + console.group( + `fixing ${fixableFlaws.length} fixable flaw(s) before proceeding:` + ); + // Let's start fresh so we don't keep the "data-flaw-src" + // attributes that may have been injected during the rendering. + document.rawHTML = originalRawHTML; + for (const flaw of fixableFlaws) { + const suggestion = flaw.macroSource.replace( + flaw.redirectInfo.current, + flaw.redirectInfo.suggested + ); + document.rawHTML = document.rawHTML.replace( + flaw.macroSource, + suggestion + ); + console.log(`${flaw.macroSource} --> ${suggestion}`); + } + console.groupEnd(); + Document.update( + document.url, + document.rawHTML, + document.metadata + ); + // Ok, we've fixed the fixable flaws, now let's render again. + [renderedHTML, flaws] = await renderOrRemoveMacros(document); + } + } else { + // There are one or more flaws that we can't fix, and we're not + // going to ignore them, so let's skip this document. + console.log( + `skipping, has ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})` + ); + console.groupEnd(); + countSkipped++; + continue; + } + } + // The Kumascript rendering wraps the result with a "body" tag + // (and more), so let's extract the HTML content of the "body" + // to get what we'll store in the document. + const $ = cheerio.load(renderedHTML); + const newRawHTML = $("body").html(); + if (newRawHTML !== originalRawHTML) { + Document.update(document.url, newRawHTML, document.metadata); + console.log(`modified`); + countModified++; + } else { + console.log(`no change`); + countNoChange++; + } + console.groupEnd(); + } + console.log( + `modified: ${countModified} | no-change: ${countNoChange} | skipped: ${countSkipped} | total: ${countTotal}` + ); + }) ); program.run(); From 9710df34f855b59fb0a0bd85341500a506a80a8d Mon Sep 17 00:00:00 2001 From: Florian Dieminger Date: Fri, 12 Mar 2021 09:22:30 +0100 Subject: [PATCH 149/164] add active locales (#3201) only show "onGithub" for active locales --- build/index.js | 1 + client/src/document/organisms/metadata/index.tsx | 2 +- content/constants.js | 3 ++- content/document.js | 4 ++++ libs/constants/index.js | 2 ++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/build/index.js b/build/index.js index a188d889fd53..7ebe064858ee 100644 --- a/build/index.js +++ b/build/index.js @@ -225,6 +225,7 @@ async function buildDocument(document, documentOptions = {}) { const doc = { isArchive: document.isArchive, isTranslated: document.isTranslated, + isActive: document.isActive, }; doc.flaws = {}; diff --git a/client/src/document/organisms/metadata/index.tsx b/client/src/document/organisms/metadata/index.tsx index 2b8e101260bf..8424b50aa1e9 100644 --- a/client/src/document/organisms/metadata/index.tsx +++ b/client/src/document/organisms/metadata/index.tsx @@ -30,7 +30,7 @@ export function Metadata({ doc, locale }) { return (