Create API v4 & rework API v3 (#8136)

.github/workflows/publish.yml

@@ -43,7 +43,7 @@ jobs:
           fi
 
       - name: Generate API files
-        run: node scripts/APIv3.js
+        run: node scripts/APIv*.js
 
       - name: Publish changes to Algolia
         if: steps.diff.outputs.entries

.prettierrc.json

@@ -0,0 +1,8 @@
+{
+  "trailingComma": "es5",
+  "tabWidth": 2,
+  "semi": true,
+  "singleQuote": false,
+  "editorconfig": true,
+  "bracketSpacing": true
+}

package.json

@@ -7,7 +7,10 @@
   "dependencies": {
     "@actions/core": "^1.10.1",
     "dotenv": "^16.4.5",
-    "glob": "^10.4.1"
+    "glob": "^10.4.1",
+    "ajv": "^8.16.0",
+    "ajv-errors": "^3.0.0",
+    "ajv-formats": "^3.0.1"
   },
   "optionalDependencies": {
     "algoliasearch": "^4.24.0"
@@ -16,9 +19,6 @@
     "@playwright/test": "^1.45.1",
     "@xmldom/xmldom": "^0.8.10",
     "abort-controller": "^3.0.0",
-    "ajv": "^8.16.0",
-    "ajv-errors": "^3.0.0",
-    "ajv-formats": "^3.0.1",
     "prettier": "^3.3.3",
     "xml2js": "^0.6.2",
     "xpath": "^0.0.34"

scripts/APIv3.js

@@ -1,77 +1,184 @@
 #!/usr/bin/env node
 
-const fs = require('fs');
-const path = require('path');
-
-let all = [];
-let tfa = {};
-let regions = {};
-
-// Read all JSON files from the 'entries' directory
-fs.readdirSync('entries').forEach((dir) => {
-  fs.readdirSync(path.join('entries', dir)).forEach((file) => {
-    if (file.endsWith('.json')) {
-      let data = JSON.parse(
-        fs.readFileSync(path.join('entries', dir, file), 'utf8'));
-      let key = Object.keys(data)[0];
-      all.push([key, data[key]]);
-    }
-  });
-});
+const fs = require("fs").promises;
+const path = require("path");
+const Ajv = require("ajv");
+const addFormats = require("ajv-formats");
+const { globSync } = require("glob");
+const { setFailed } = require("@actions/core");
+const core = require("@actions/core");
 
-// Process all entries
-all.sort((a, b) => a[0].localeCompare(b[0])).forEach(([entryName, entry]) => {
+const entriesDir = "entries";
+const apiDirectory = "api/v3";
+const jsonSchemaPath = "tests/schemas/APIv3.json";
 
-  // Process tfa methods
-  if ('tfa' in entry) {
-    entry['tfa'].forEach((method) => {
-      if (!tfa[method]) tfa[method] = [];
-      tfa[method].push([entryName, entry]);
-    });
-  }
+/**
+ * Read and parse a JSON file asynchronously.
+ *
+ * @param {string} filePath - The path to the JSON file.
+ * @returns {Promise<Object>} - The parsed JSON object.
+ */
+const readJSONFile = (filePath) =>
+  fs.readFile(filePath, "utf8").then(JSON.parse);
 
-  // Process regions
-  if ('regions' in entry) {
-    entry['regions'].forEach((region) => {
-      if (region[0] !== '-') {
-        if (!regions[region]) regions[region] = {count: 0};
-        regions[region]['count'] += 1;
-      }
-    });
-  }
+/**
+ * Write a JSON object to a file asynchronously.
+ *
+ * @param {string} filePath - The path to the output file.
+ * @param {Object} data - The JSON object to write.
+ * @returns {Promise<void>}
+ */
+const writeJSONFile = (filePath, data) =>
+  fs.writeFile(
+    filePath,
+    JSON.stringify(data, null, process.env.NODE_ENV !== "production" ? 2:0),
+  );
 
-  // Rename 'categories' to 'keywords'
-  if ('categories' in entry) {
-    entry['keywords'] = entry['categories'];
-    delete entry['categories'];
-  }
-});
+/**
+ * Ensure a directory exists, creating it if necessary.
+ *
+ * @param {string} dirPath - The path to the directory.
+ * @returns {Promise<void>}
+ */
+const ensureDir = (dirPath) =>
+  fs.mkdir(dirPath, { recursive: true }).catch((error) => {
+    if (error.code !== "EEXIST") throw error;
+  });
 
-// Write the all.json and tfa files
-const outputDir = 'api/v3';
-if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, {recursive: true});
+/**
+ * Process all entries by reading JSON files from the "entries" directory,
+ * sorting them, and processing each entry.
+ *
+ * @returns {Promise<Object>} - An object containing processed all, tfa, and regions data.
+ */
+const processEntries = async () => {
+  let allEntries = [];
+  let tfaMethods = {};
+  let regions = {};
 
-const writeJsonFile = (filename, data) => fs.writeFileSync(
-  path.join(outputDir, filename), JSON.stringify(data));
+  // Read all JSON files from the "entries" directory
+  const entryDirs = await fs.readdir(entriesDir);
+  const filePromises = entryDirs.map(async (dir) => {
+    const files = await fs.readdir(path.join(entriesDir, dir));
+    return files.filter((file) => file.endsWith(".json")).
+      map((file) => path.join(entriesDir, dir, file));
+  });
+  const allFiles = (await Promise.all(filePromises)).flat();
 
-writeJsonFile('all.json', all);
+  const all = await Promise.all(allFiles.map(async (file) => {
+    const data = await readJSONFile(file);
+    const key = Object.keys(data)[0];
+    return [key, data[key]];
+  }));
 
-Object.keys(tfa).
-  forEach((method) => writeJsonFile(`${method}.json`, tfa[method]));
+  await Promise.all(
+    all.sort((a, b) => a[0].localeCompare(b[0])).
+      map(async ([entryName, entry]) => {
+        await processEntry(entry, entryName, tfaMethods, regions);
+        allEntries.push([entryName, entry]);
+      }),
+  );
 
-// Add the 'int' region
-regions['int'] = {count: all.length, selection: true};
+  regions = Object.entries(regions).
+    sort(([, a], [, b]) => b.count - a.count).
+    reduce((acc, [k, v]) => (acc[k] = v, acc), {});
 
-// Write regions.json
-const sortedRegions = Object.entries(regions).
-  sort(([, a], [, b]) => b.count - a.count).
-  reduce((acc, [k, v]) => {
-    acc[k] = v;
+  const tfa = Object.entries(tfaMethods).reduce((acc, [method, entries]) => {
+    acc[method] = entries.sort(
+      ([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase()));
     return acc;
   }, {});
-writeJsonFile('regions.json', sortedRegions);
 
-// Write tfa.json
-const tfaEntries = all.filter(([, entry]) => entry.tfa).
-  sort(([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase()));
-writeJsonFile('tfa.json', tfaEntries);
+  return { allEntries, tfa, regions };
+};
+
+/**
+ * Process a single entry, updating the tfa and regions objects.
+ *
+ * @param {Object} entry - The entry data.
+ * @param {string} entryName - The name of the entry.
+ * @param {Object} tfaMethods - The tfaMethods object to update.
+ * @param {Object} regions - The regions object to update.
+ */
+const processEntry = (entry, entryName, tfaMethods, regions) => {
+  entry["tfa"]?.forEach((method) => {
+    tfaMethods[method] ||= [];
+    tfaMethods[method].push([entryName, entry]);
+  });
+
+  entry["regions"]?.forEach((region) => {
+    if (region[0] !== "-")
+      regions[region] ? regions[region].count++:regions[region] = { count: 0 };
+  });
+
+  entry.keywords = entry.categories;
+  delete entry.categories;
+};
+
+/**
+ * Generate JSON files from processed entries
+ *
+ * @param {Array} allEntries - The processed all entries.
+ * @param {Object} tfa - The processed tfa data.
+ * @param {Object} regions - The processed region data.
+ * @returns {Promise<void>}
+ */
+const generateAPI = async (allEntries, tfa, regions) => {
+  regions.int = { count: allEntries.length, selection: true };
+
+  await Promise.all([
+    writeJSONFile(path.join(apiDirectory, "all.json"), allEntries),
+    writeJSONFile(path.join(apiDirectory, "regions.json"), regions),
+    ...Object.keys(tfa).map((method) =>
+      writeJSONFile(path.join(apiDirectory, `${method}.json`), tfa[method]),
+    ),
+  ]);
+};
+
+/**
+ * Validate API files against JSON schema.
+ *
+ * @returns {Promise<void>}
+ */
+const validateSchema = async () => {
+  const ajv = new Ajv({ strict: false, allErrors: true });
+  addFormats(ajv);
+  require("ajv-errors")(ajv);
+
+  const schema = await readJSONFile(jsonSchemaPath);
+  const validate = ajv.compile(schema);
+  const files = globSync(`${apiDirectory}/*.json`, {
+    ignore: `${apiDirectory}/regions.json`,
+  });
+
+  await Promise.all(
+    files.map(async (file) => {
+      const data = await readJSONFile(file);
+      validate(data);
+      validate.errors?.forEach((err) => {
+        const { message } = err;
+        throw new Error(`${file} - ${message}`);
+      });
+    }),
+  );
+};
+
+/**
+ * Main function to process entries, ensure directories, serialize results, and validate schema.
+ *
+ * @returns {Promise<void>}
+ */
+const main = async () => {
+  try {
+    core.info("Generating API v3");
+    const { allEntries, tfa, regions } = await processEntries();
+    await ensureDir(apiDirectory);
+    await generateAPI(allEntries, tfa, regions);
+    await validateSchema();
+    core.info("API v3 generation completed successfully");
+  } catch (e) {
+    setFailed(e);
+  }
+};
+
+module.exports = main();