Skip to content

Commit

Permalink
add tool command for rendering/removing macros (#2955)
Browse files Browse the repository at this point in the history
  • Loading branch information
@escattone
escattone authored Mar 11, 2021
1 parent 7766fd2 commit 83cf627
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 12 deletions.
12 changes: 8 additions & 4 deletions content/document.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,11 @@ function findByURL(url, ...args) {
return doc;
}

function findAll(
{ files, folderSearch } = { files: new Set(), folderSearch: null }
) {
function findAll({
files = new Set(),
folderSearch = null,
quiet = false,
} = {}) {
if (!(files instanceof Set)) throw new TypeError("'files' not a Set");
if (folderSearch && typeof folderSearch !== "string")
throw new TypeError("'folderSearch' not a string");
Expand All @@ -373,7 +375,9 @@ function findAll(
roots.push(CONTENT_TRANSLATED_ROOT);
}
roots.push(CONTENT_ROOT);
console.log("Building roots:", roots);
if (!quiet) {
console.log("Building roots:", roots);
}
for (const root of roots) {
filePaths.push(
...glob
Expand Down
24 changes: 16 additions & 8 deletions kumascript/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,17 @@ const DEPENDENCY_LOOP_INTRO =

const renderCache = new LRU({ max: 2000 });

const renderFromURL = async (url, urlsSeen = null) => {
const renderFromURL = async (
url,
{ urlsSeen = null, selective_mode = false, invalidateCache = false } = {}
) => {
const urlLC = url.toLowerCase();
if (renderCache.has(urlLC)) {
return renderCache.get(urlLC);
if (invalidateCache) {
renderCache.del(urlLC);
} else {
return renderCache.get(urlLC);
}
}

urlsSeen = urlsSeen || new Set([]);
Expand All @@ -34,7 +41,9 @@ const renderFromURL = async (url, urlsSeen = null) => {
}
urlsSeen.add(urlLC);
const prerequisiteErrorsByKey = new Map();
const document = Document.findByURL(url);
const document = invalidateCache
? Document.findByURL(url, Document.MEMOIZE_INVALIDATE)
: Document.findByURL(url);
if (!document) {
throw new Error(
`From URL ${url} no folder on disk could be found. ` +
Expand All @@ -51,18 +60,17 @@ const renderFromURL = async (url, urlsSeen = null) => {
slug: metadata.slug,
title: metadata.title,
tags: metadata.tags || [],
selective_mode: false,
selective_mode,
},
interactive_examples: {
base_url: INTERACTIVE_EXAMPLES_BASE_URL,
},
live_samples: { base_url: LIVE_SAMPLES_BASE_URL || url },
},
async (url) => {
const [renderedHtml, errors] = await renderFromURL(
info.cleanURL(url),
urlsSeen
);
const [renderedHtml, errors] = await renderFromURL(info.cleanURL(url), {
urlsSeen,
});
// Remove duplicate flaws. During the rendering process, it's possible for identical
// flaws to be introduced when different dependency paths share common prerequisites.
// For example, document A may have prerequisite documents B and C, and in turn,
Expand Down
139 changes: 139 additions & 0 deletions tool/cli.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const {
syncAllTranslatedContent,
} = require("../build/sync-translated-content");
const log = require("loglevel");
const cheerio = require("cheerio");

const { DEFAULT_LOCALE, VALID_LOCALES } = require("../libs/constants");
const {
Expand All @@ -28,6 +29,7 @@ const {
GOOGLE_ANALYTICS_DEBUG,
} = require("../build/constants");
const { runMakePopularitiesFile } = require("./popularities");
const kumascript = require("../kumascript");

const PORT = parseInt(process.env.SERVER_PORT || "5000");

Expand Down Expand Up @@ -756,6 +758,143 @@ if (Mozilla && !Mozilla.dntEnabled()) {
tryOrExit(async ({ options }) => {
await buildSPAs(options);
})
)

.command(
"macros",
"Render and/or remove one or more macros from one or more documents"
)
.option("-f, --force", "Render even if there are non-fixable flaws", {
default: false,
})
.argument("<cmd>", 'must be either "render" or "remove"')
.argument("<foldersearch>", "folder of documents to target")
.argument("<macros...>", "one or more macro names")
.action(
tryOrExit(async ({ args, options }) => {
if (!CONTENT_ROOT) {
throw new Error("CONTENT_ROOT not set");
}
if (!CONTENT_TRANSLATED_ROOT) {
throw new Error("CONTENT_TRANSLATED_ROOT not set");
}
const { force } = options;
const { cmd, foldersearch, macros } = args;
const cmdLC = cmd.toLowerCase();
if (!["render", "remove"].includes(cmdLC)) {
throw new Error(`invalid macros command "${cmd}"`);
}
console.log(
`${cmdLC} the macro(s) ${macros
.map((m) => `"${m}"`)
.join(", ")} within content folder(s) matching "${foldersearch}"`
);
const documents = Document.findAll({
folderSearch: foldersearch,
quiet: true,
});
if (!documents.count) {
throw new Error("no documents found");
}

async function renderOrRemoveMacros(document) {
try {
return await kumascript.render(document.url, {
invalidateCache: true,
selective_mode: [cmdLC, macros],
});
} catch (error) {
if (error.name === "MacroInvocationError") {
error.updateFileInfo(document.fileInfo);
throw new Error(
`error trying to parse ${error.filepath}, line ${error.line} column ${error.column} (${error.error.message})`
);
}
// Any other unexpected error re-thrown.
throw error;
}
}

let countTotal = 0;
let countSkipped = 0;
let countModified = 0;
let countNoChange = 0;
for (const document of documents.iter()) {
countTotal++;
console.group(`${document.fileInfo.path}:`);
const originalRawHTML = document.rawHTML;
let [renderedHTML, flaws] = await renderOrRemoveMacros(document);
if (flaws.length) {
const fixableFlaws = flaws.filter((f) => f.redirectInfo);
const nonFixableFlaws = flaws.filter((f) => !f.redirectInfo);
const nonFixableFlawNames = [
...new Set(nonFixableFlaws.map((f) => f.name)).values(),
].join(", ");
if (force || nonFixableFlaws.length === 0) {
// They're all fixable or we don't care if some or all are
// not, but let's at least fix any that we can.
if (nonFixableFlaws.length > 0) {
console.log(
`ignoring ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})`
);
}
if (fixableFlaws.length) {
console.group(
`fixing ${fixableFlaws.length} fixable flaw(s) before proceeding:`
);
// Let's start fresh so we don't keep the "data-flaw-src"
// attributes that may have been injected during the rendering.
document.rawHTML = originalRawHTML;
for (const flaw of fixableFlaws) {
const suggestion = flaw.macroSource.replace(
flaw.redirectInfo.current,
flaw.redirectInfo.suggested
);
document.rawHTML = document.rawHTML.replace(
flaw.macroSource,
suggestion
);
console.log(`${flaw.macroSource} --> ${suggestion}`);
}
console.groupEnd();
Document.update(
document.url,
document.rawHTML,
document.metadata
);
// Ok, we've fixed the fixable flaws, now let's render again.
[renderedHTML, flaws] = await renderOrRemoveMacros(document);
}
} else {
// There are one or more flaws that we can't fix, and we're not
// going to ignore them, so let's skip this document.
console.log(
`skipping, has ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})`
);
console.groupEnd();
countSkipped++;
continue;
}
}
// The Kumascript rendering wraps the result with a "body" tag
// (and more), so let's extract the HTML content of the "body"
// to get what we'll store in the document.
const $ = cheerio.load(renderedHTML);
const newRawHTML = $("body").html();
if (newRawHTML !== originalRawHTML) {
Document.update(document.url, newRawHTML, document.metadata);
console.log(`modified`);
countModified++;
} else {
console.log(`no change`);
countNoChange++;
}
console.groupEnd();
}
console.log(
`modified: ${countModified} | no-change: ${countNoChange} | skipped: ${countSkipped} | total: ${countTotal}`
);
})
);

program.run();

0 comments on commit 83cf627

Please sign in to comment.