add tool command for rendering/removing macros

content/document.js

@@ -354,9 +354,11 @@ function findByURL(url, ...args) {
   return doc;
 }
 
-function findAll(
-  { files, folderSearch } = { files: new Set(), folderSearch: null }
-) {
+function findAll({
+  files = new Set(),
+  folderSearch = null,
+  quiet = false,
+} = {}) {
   if (!(files instanceof Set)) throw new TypeError("'files' not a Set");
   if (folderSearch && typeof folderSearch !== "string")
     throw new TypeError("'folderSearch' not a string");
@@ -373,7 +375,9 @@ function findAll(
     roots.push(CONTENT_TRANSLATED_ROOT);
   }
   roots.push(CONTENT_ROOT);
-  console.log("Building roots:", roots);
+  if (!quiet) {
+    console.log("Building roots:", roots);
+  }
   for (const root of roots) {
     filePaths.push(
       ...glob

kumascript/index.js

@@ -20,10 +20,17 @@ const DEPENDENCY_LOOP_INTRO =
 
 const renderCache = new LRU({ max: 2000 });
 
-const renderFromURL = async (url, urlsSeen = null) => {
+const renderFromURL = async (
+  url,
+  { urlsSeen = null, selective_mode = false, invalidateCache = false } = {}
+) => {
   const urlLC = url.toLowerCase();
   if (renderCache.has(urlLC)) {
-    return renderCache.get(urlLC);
+    if (invalidateCache) {
+      renderCache.del(urlLC);
+    } else {
+      return renderCache.get(urlLC);
+    }
   }
 
   urlsSeen = urlsSeen || new Set([]);
@@ -34,7 +41,9 @@ const renderFromURL = async (url, urlsSeen = null) => {
   }
   urlsSeen.add(urlLC);
   const prerequisiteErrorsByKey = new Map();
-  const document = Document.findByURL(url);
+  const document = invalidateCache
+    ? Document.findByURL(url, Document.MEMOIZE_INVALIDATE)
+    : Document.findByURL(url);
   if (!document) {
     throw new Error(
       `From URL ${url} no folder on disk could be found. ` +
@@ -51,18 +60,17 @@ const renderFromURL = async (url, urlsSeen = null) => {
         slug: metadata.slug,
         title: metadata.title,
         tags: metadata.tags || [],
-        selective_mode: false,
+        selective_mode,
       },
       interactive_examples: {
         base_url: INTERACTIVE_EXAMPLES_BASE_URL,
       },
       live_samples: { base_url: LIVE_SAMPLES_BASE_URL || url },
     },
     async (url) => {
-      const [renderedHtml, errors] = await renderFromURL(
-        info.cleanURL(url),
-        urlsSeen
-      );
+      const [renderedHtml, errors] = await renderFromURL(info.cleanURL(url), {
+        urlsSeen,
+      });
       // Remove duplicate flaws. During the rendering process, it's possible for identical
       // flaws to be introduced when different dependency paths share common prerequisites.
       // For example, document A may have prerequisite documents B and C, and in turn,

tool/cli.js

@@ -10,6 +10,7 @@ const {
   syncAllTranslatedContent,
 } = require("../build/sync-translated-content");
 const log = require("loglevel");
+const cheerio = require("cheerio");
 
 const { DEFAULT_LOCALE, VALID_LOCALES } = require("../libs/constants");
 const {
@@ -27,6 +28,7 @@ const {
   GOOGLE_ANALYTICS_DEBUG,
 } = require("../build/constants");
 const { runMakePopularitiesFile } = require("./popularities");
+const kumascript = require("../kumascript");
 
 const PORT = parseInt(process.env.SERVER_PORT || "5000");
 
@@ -748,6 +750,143 @@ if (Mozilla && !Mozilla.dntEnabled()) {
     tryOrExit(async ({ options }) => {
       await buildSPAs(options);
     })
+  )
+
+  .command(
+    "macros",
+    "Render and/or remove one or more macros from one or more documents"
+  )
+  .option("-f, --force", "Render even if there are non-fixable flaws", {
+    default: false,
+  })
+  .argument("<cmd>", 'must be either "render" or "remove"')
+  .argument("<foldersearch>", "folder of documents to target")
+  .argument("<macros...>", "one or more macro names")
+  .action(
+    tryOrExit(async ({ args, options }) => {
+      if (!CONTENT_ROOT) {
+        throw new Error("CONTENT_ROOT not set");
+      }
+      if (!CONTENT_TRANSLATED_ROOT) {
+        throw new Error("CONTENT_TRANSLATED_ROOT not set");
+      }
+      const { force } = options;
+      const { cmd, foldersearch, macros } = args;
+      const cmdLC = cmd.toLowerCase();
+      if (!["render", "remove"].includes(cmdLC)) {
+        throw new Error(`invalid macros command "${cmd}"`);
+      }
+      console.log(
+        `${cmdLC} the macro(s) ${macros
+          .map((m) => `"${m}"`)
+          .join(", ")} within content folder(s) matching "${foldersearch}"`
+      );
+      const documents = Document.findAll({
+        folderSearch: foldersearch,
+        quiet: true,
+      });
+      if (!documents.count) {
+        throw new Error("no documents found");
+      }
+
+      async function renderOrRemoveMacros(document) {
+        try {
+          return await kumascript.render(document.url, {
+            invalidateCache: true,
+            selective_mode: [cmdLC, macros],
+          });
+        } catch (error) {
+          if (error.name === "MacroInvocationError") {
+            error.updateFileInfo(document.fileInfo);
+            throw new Error(
+              `error trying to parse ${error.filepath}, line ${error.line} column ${error.column} (${error.error.message})`
+            );
+          }
+          // Any other unexpected error re-thrown.
+          throw error;
+        }
+      }
+
+      let countTotal = 0;
+      let countSkipped = 0;
+      let countModified = 0;
+      let countNoChange = 0;
+      for (const document of documents.iter()) {
+        countTotal++;
+        console.group(`${document.fileInfo.path}:`);
+        const originalRawHTML = document.rawHTML;
+        let [renderedHTML, flaws] = await renderOrRemoveMacros(document);
+        if (flaws.length) {
+          const fixableFlaws = flaws.filter((f) => f.redirectInfo);
+          const nonFixableFlaws = flaws.filter((f) => !f.redirectInfo);
+          const nonFixableFlawNames = [
+            ...new Set(nonFixableFlaws.map((f) => f.name)).values(),
+          ].join(", ");
+          if (force || nonFixableFlaws.length === 0) {
+            // They're all fixable or we don't care if some or all are
+            // not, but let's at least fix any that we can.
+            if (nonFixableFlaws.length > 0) {
+              console.log(
+                `ignoring ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})`
+              );
+            }
+            if (fixableFlaws.length) {
+              console.group(
+                `fixing ${fixableFlaws.length} fixable flaw(s) before proceeding:`
+              );
+              // Let's start fresh so we don't keep the "data-flaw-src"
+              // attributes that may have been injected during the rendering.
+              document.rawHTML = originalRawHTML;
+              for (const flaw of fixableFlaws) {
+                const suggestion = flaw.macroSource.replace(
+                  flaw.redirectInfo.current,
+                  flaw.redirectInfo.suggested
+                );
+                document.rawHTML = document.rawHTML.replace(
+                  flaw.macroSource,
+                  suggestion
+                );
+                console.log(`${flaw.macroSource} --> ${suggestion}`);
+              }
+              console.groupEnd();
+              Document.update(
+                document.url,
+                document.rawHTML,
+                document.metadata
+              );
+              // Ok, we've fixed the fixable flaws, now let's render again.
+              [renderedHTML, flaws] = await renderOrRemoveMacros(document);
+            }
+          } else {
+            // There are one or more flaws that we can't fix, and we're not
+            // going to ignore them, so let's skip this document.
+            console.log(
+              `skipping, has ${nonFixableFlaws.length} non-fixable flaw(s) (${nonFixableFlawNames})`
+            );
+            console.groupEnd();
+            countSkipped++;
+            continue;
+          }
+        }
+        // The Kumascript rendering wraps the result with a "body" tag
+        // (and more), so let's extract the HTML content of the "body"
+        // to get what we'll store in the document.
+        const $ = cheerio.load(renderedHTML);
+        const newRawHTML = $("body").html();
+        if (newRawHTML !== originalRawHTML) {
+          Document.update(document.url, newRawHTML, document.metadata);
+          console.log(`modified`);
+          countModified++;
+        } else {
+          console.log(`no change`);
+          countNoChange++;
+        }
+        console.groupEnd();
+      }
+      console.log(
+        `modified: ${countModified} | no-change: ${countNoChange} | skipped: ${countSkipped} | total: ${countTotal}`
+      );
+    })
   );
 
 program.run();