From a7242c5e086c9f06fa9aa314769b9176a78a3f2d Mon Sep 17 00:00:00 2001 From: Carl Date: Sun, 28 Jul 2024 12:43:15 +0200 Subject: [PATCH] Create API v4 & rework API v3 (#8136) --- .github/workflows/publish.yml | 2 +- .prettierrc.json | 8 + entries/3/34SP.com.json | 15 -- img/3/34SP.com.svg | 1 - package.json | 8 +- scripts/APIv3.js | 231 ++++++++++++++------ scripts/APIv4.js | 159 ++++++++++++++ tests/json.js | 7 +- tests/schemas/APIv3.json | 179 +++++++++++++++ tests/schemas/APIv4.json | 86 ++++++++ tests/{schema.json => schemas/entries.json} | 2 + 11 files changed, 612 insertions(+), 86 deletions(-) create mode 100644 .prettierrc.json delete mode 100644 entries/3/34SP.com.json delete mode 100644 img/3/34SP.com.svg create mode 100644 scripts/APIv4.js create mode 100644 tests/schemas/APIv3.json create mode 100644 tests/schemas/APIv4.json rename tests/{schema.json => schemas/entries.json} (97%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b3ecea10dff..f25569ef1f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,7 +43,7 @@ jobs: fi - name: Generate API files - run: node scripts/APIv3.js + run: node scripts/APIv*.js - name: Publish changes to Algolia if: steps.diff.outputs.entries diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 00000000000..7827137cf97 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,8 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": false, + "editorconfig": true, + "bracketSpacing": true +} diff --git a/entries/3/34SP.com.json b/entries/3/34SP.com.json deleted file mode 100644 index 72e01237562..00000000000 --- a/entries/3/34SP.com.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "34SP.com": { - "domain": "34SP.com", - "tfa": [ - "totp" - ], - "documentation": "https://www.34sp.com/kb/142/how-to-enable-two-factor-authentication-on-your-account", - "categories": [ - "hosting" - ], - "regions": [ - "us" - ] - } -} diff --git a/img/3/34SP.com.svg b/img/3/34SP.com.svg deleted file mode 100644 index c4dfca14c3f..00000000000 --- a/img/3/34SP.com.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/package.json b/package.json index 484b2ea25e3..1ed687b5588 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,10 @@ "dependencies": { "@actions/core": "^1.10.1", "dotenv": "^16.4.5", - "glob": "^10.4.1" + "glob": "^10.4.1", + "ajv": "^8.16.0", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1" }, "optionalDependencies": { "algoliasearch": "^4.24.0" @@ -16,9 +19,6 @@ "@playwright/test": "^1.45.1", "@xmldom/xmldom": "^0.8.10", "abort-controller": "^3.0.0", - "ajv": "^8.16.0", - "ajv-errors": "^3.0.0", - "ajv-formats": "^3.0.1", "prettier": "^3.3.3", "xml2js": "^0.6.2", "xpath": "^0.0.34" diff --git a/scripts/APIv3.js b/scripts/APIv3.js index 356f0a30cfa..8e941535de5 100644 --- a/scripts/APIv3.js +++ b/scripts/APIv3.js @@ -1,77 +1,184 @@ #!/usr/bin/env node -const fs = require('fs'); -const path = require('path'); - -let all = []; -let tfa = {}; -let regions = {}; - -// Read all JSON files from the 'entries' directory -fs.readdirSync('entries').forEach((dir) => { - fs.readdirSync(path.join('entries', dir)).forEach((file) => { - if (file.endsWith('.json')) { - let data = JSON.parse( - fs.readFileSync(path.join('entries', dir, file), 'utf8')); - let key = Object.keys(data)[0]; - all.push([key, data[key]]); - } - }); -}); +const fs = require("fs").promises; +const path = require("path"); +const Ajv = require("ajv"); +const addFormats = require("ajv-formats"); +const { globSync } = require("glob"); +const { setFailed } = require("@actions/core"); +const core = require("@actions/core"); -// Process all entries -all.sort((a, b) => a[0].localeCompare(b[0])).forEach(([entryName, entry]) => { +const entriesDir = "entries"; +const apiDirectory = "api/v3"; +const jsonSchemaPath = "tests/schemas/APIv3.json"; - // Process tfa methods - if ('tfa' in entry) { - entry['tfa'].forEach((method) => { - if (!tfa[method]) tfa[method] = []; - tfa[method].push([entryName, entry]); - }); - } +/** + * Read and parse a JSON file asynchronously. + * + * @param {string} filePath - The path to the JSON file. + * @returns {Promise} - The parsed JSON object. + */ +const readJSONFile = (filePath) => + fs.readFile(filePath, "utf8").then(JSON.parse); - // Process regions - if ('regions' in entry) { - entry['regions'].forEach((region) => { - if (region[0] !== '-') { - if (!regions[region]) regions[region] = {count: 0}; - regions[region]['count'] += 1; - } - }); - } +/** + * Write a JSON object to a file asynchronously. + * + * @param {string} filePath - The path to the output file. + * @param {Object} data - The JSON object to write. + * @returns {Promise} + */ +const writeJSONFile = (filePath, data) => + fs.writeFile( + filePath, + JSON.stringify(data, null, process.env.NODE_ENV !== "production" ? 2:0), + ); - // Rename 'categories' to 'keywords' - if ('categories' in entry) { - entry['keywords'] = entry['categories']; - delete entry['categories']; - } -}); +/** + * Ensure a directory exists, creating it if necessary. + * + * @param {string} dirPath - The path to the directory. + * @returns {Promise} + */ +const ensureDir = (dirPath) => + fs.mkdir(dirPath, { recursive: true }).catch((error) => { + if (error.code !== "EEXIST") throw error; + }); -// Write the all.json and tfa files -const outputDir = 'api/v3'; -if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, {recursive: true}); +/** + * Process all entries by reading JSON files from the "entries" directory, + * sorting them, and processing each entry. + * + * @returns {Promise} - An object containing processed all, tfa, and regions data. + */ +const processEntries = async () => { + let allEntries = []; + let tfaMethods = {}; + let regions = {}; -const writeJsonFile = (filename, data) => fs.writeFileSync( - path.join(outputDir, filename), JSON.stringify(data)); + // Read all JSON files from the "entries" directory + const entryDirs = await fs.readdir(entriesDir); + const filePromises = entryDirs.map(async (dir) => { + const files = await fs.readdir(path.join(entriesDir, dir)); + return files.filter((file) => file.endsWith(".json")). + map((file) => path.join(entriesDir, dir, file)); + }); + const allFiles = (await Promise.all(filePromises)).flat(); -writeJsonFile('all.json', all); + const all = await Promise.all(allFiles.map(async (file) => { + const data = await readJSONFile(file); + const key = Object.keys(data)[0]; + return [key, data[key]]; + })); -Object.keys(tfa). - forEach((method) => writeJsonFile(`${method}.json`, tfa[method])); + await Promise.all( + all.sort((a, b) => a[0].localeCompare(b[0])). + map(async ([entryName, entry]) => { + await processEntry(entry, entryName, tfaMethods, regions); + allEntries.push([entryName, entry]); + }), + ); -// Add the 'int' region -regions['int'] = {count: all.length, selection: true}; + regions = Object.entries(regions). + sort(([, a], [, b]) => b.count - a.count). + reduce((acc, [k, v]) => (acc[k] = v, acc), {}); -// Write regions.json -const sortedRegions = Object.entries(regions). - sort(([, a], [, b]) => b.count - a.count). - reduce((acc, [k, v]) => { - acc[k] = v; + const tfa = Object.entries(tfaMethods).reduce((acc, [method, entries]) => { + acc[method] = entries.sort( + ([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase())); return acc; }, {}); -writeJsonFile('regions.json', sortedRegions); -// Write tfa.json -const tfaEntries = all.filter(([, entry]) => entry.tfa). - sort(([a], [b]) => a.toLowerCase().localeCompare(b.toLowerCase())); -writeJsonFile('tfa.json', tfaEntries); + return { allEntries, tfa, regions }; +}; + +/** + * Process a single entry, updating the tfa and regions objects. + * + * @param {Object} entry - The entry data. + * @param {string} entryName - The name of the entry. + * @param {Object} tfaMethods - The tfaMethods object to update. + * @param {Object} regions - The regions object to update. + */ +const processEntry = (entry, entryName, tfaMethods, regions) => { + entry["tfa"]?.forEach((method) => { + tfaMethods[method] ||= []; + tfaMethods[method].push([entryName, entry]); + }); + + entry["regions"]?.forEach((region) => { + if (region[0] !== "-") + regions[region] ? regions[region].count++:regions[region] = { count: 0 }; + }); + + entry.keywords = entry.categories; + delete entry.categories; +}; + +/** + * Generate JSON files from processed entries + * + * @param {Array} allEntries - The processed all entries. + * @param {Object} tfa - The processed tfa data. + * @param {Object} regions - The processed region data. + * @returns {Promise} + */ +const generateAPI = async (allEntries, tfa, regions) => { + regions.int = { count: allEntries.length, selection: true }; + + await Promise.all([ + writeJSONFile(path.join(apiDirectory, "all.json"), allEntries), + writeJSONFile(path.join(apiDirectory, "regions.json"), regions), + ...Object.keys(tfa).map((method) => + writeJSONFile(path.join(apiDirectory, `${method}.json`), tfa[method]), + ), + ]); +}; + +/** + * Validate API files against JSON schema. + * + * @returns {Promise} + */ +const validateSchema = async () => { + const ajv = new Ajv({ strict: false, allErrors: true }); + addFormats(ajv); + require("ajv-errors")(ajv); + + const schema = await readJSONFile(jsonSchemaPath); + const validate = ajv.compile(schema); + const files = globSync(`${apiDirectory}/*.json`, { + ignore: `${apiDirectory}/regions.json`, + }); + + await Promise.all( + files.map(async (file) => { + const data = await readJSONFile(file); + validate(data); + validate.errors?.forEach((err) => { + const { message } = err; + throw new Error(`${file} - ${message}`); + }); + }), + ); +}; + +/** + * Main function to process entries, ensure directories, serialize results, and validate schema. + * + * @returns {Promise} + */ +const main = async () => { + try { + core.info("Generating API v3"); + const { allEntries, tfa, regions } = await processEntries(); + await ensureDir(apiDirectory); + await generateAPI(allEntries, tfa, regions); + await validateSchema(); + core.info("API v3 generation completed successfully"); + } catch (e) { + setFailed(e); + } +}; + +module.exports = main(); diff --git a/scripts/APIv4.js b/scripts/APIv4.js new file mode 100644 index 00000000000..9e81a72e7d2 --- /dev/null +++ b/scripts/APIv4.js @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +const fs = require("fs").promises; +const path = require("path"); +const { globSync } = require("glob"); +const core = require("@actions/core"); +const Ajv = require("ajv"); +const addFormats = require("ajv-formats"); + +// Define the path to the entries and the API output directory +const entriesGlob = "entries/*/*.json"; +const apiDirectory = "api/v4"; +const jsonSchema = "tests/schemas/APIv4.json"; + +/** + * Read and parse a JSON file asynchronously. + * + * @param {string} filePath - The path to the JSON file. + * @returns {Promise} - The parsed JSON object. + */ +const readJSONFile = (filePath) => fs.readFile(filePath, "utf8"). + then(JSON.parse); + +/** + * Write a JSON object to a file asynchronously. + * + * @param {string} filePath - The path to the output file. + * @param {Object} data - The JSON object to write. + * @returns {Promise} + */ +const writeJSONFile = (filePath, data) => fs.writeFile(filePath, + JSON.stringify(data, null, process.env.NODE_ENV !== "production" ? 2:0)); + +/** + * Ensure a directory exists, creating it if necessary. + * + * @param {string} dirPath - The path to the directory. + * @returns {Promise} + */ +const ensureDir = (dirPath) => fs.mkdir(dirPath, { recursive: true }). + catch((error) => { + if (error.code !== "EEXIST") throw error; + }); + +/** + * Process entries by loading and transforming them. + * + * @param {string[]} files - Array of file paths to process. + * @returns {Promise} - An object containing all processed entries. + */ +const processEntries = async (files) => { + const entries = {}; + + await Promise.all(files.map(async (file) => { + const data = await readJSONFile(file); + const entry = data[Object.keys(data)[0]]; + + // Add the main domain entry + entries[entry.domain] = entry; + + // Duplicate entry for each additional domain + entry["additional-domains"]?.forEach((additionalDomain) => { + entries[additionalDomain] = entry; + }); + })); + + return entries; +}; + +/** + * Generate the API files from the processed entries. + * + * @param {Object} entries - The processed entries. + * @returns {Promise} + */ +const generateApi = async (entries) => { + const tfaMethods = {}; + const allEntries = {}; + + await Promise.all([ + ensureDir(apiDirectory), + Object.entries(entries).map(async ([domain, entry]) => { + const apiEntry = { + methods: entry["tfa"], + contact: entry.contact, + "custom-software": entry["custom-software"], + "custom-hardware": entry["custom-hardware"], + documentation: entry.documentation, + recovery: entry.recovery, + notes: entry.notes, + }; + + // Add to all entries + allEntries[domain] = apiEntry; + + // Group entries by TFA/2FA methods + entry["tfa"]?.forEach((method) => { + tfaMethods[method] ||= {}; + tfaMethods[method][domain] = apiEntry; + }); + })]); + + // Write all entries to all.json and each TFA/2FA method to its own JSON file in parallel + await Promise.all([ + writeJSONFile(path.join(apiDirectory, "all.json"), allEntries), + ...Object.entries(tfaMethods). + map(([method, methodEntries]) => writeJSONFile( + path.join(apiDirectory, `${method}.json`), methodEntries))]); +}; + +/** + * Validate API files against JSON schema. + * + * @returns {Promise} + */ +const validateSchema = async () => { + const ajv = new Ajv({ strict: false, allErrors: true }); + addFormats(ajv); + require("ajv-errors")(ajv); + const schema = await readJSONFile(jsonSchema); + const validate = ajv.compile(schema); + const files = globSync(`${apiDirectory}/*.json`); + + // Validate each file against the schema + await Promise.all(files.map(async (file) => { + validate(await readJSONFile(file)); + + validate.errors?.forEach((err) => { + const { message, instancePath, keyword: title } = err; + const errorPath = instancePath?.split("/").slice(1).join("/"); + + core.error(`${errorPath} ${message}`, { file, title }); + }); + })); +}; + +/** + * Main function to orchestrate the loading, processing, and API generation. + * + * @returns {Promise} + */ +const main = async () => { + try { + core.info("Generating API v4"); + + // Get all JSON entry files + const files = globSync(entriesGlob); + // Process entries and generate the API + const entries = await processEntries(files); + await generateApi(entries); + await validateSchema(); + + core.info("API v4 generation completed successfully"); + } catch (error) { + core.setFailed(error); + } +}; + +module.exports = main(); diff --git a/tests/json.js b/tests/json.js index 5f491266861..31980cee11b 100644 --- a/tests/json.js +++ b/tests/json.js @@ -2,13 +2,14 @@ const fs = require("fs").promises; const core = require("@actions/core"); const Ajv = require("ajv"); const addFormats = require("ajv-formats"); -const schema = require("./schema.json"); const { basename } = require("node:path"); const ajv = new Ajv({ strict: false, allErrors: true }); addFormats(ajv); require("ajv-errors")(ajv); +const schemaFile = await fs.readFile("tests/schemas/entries.json", "utf8"); +const schema = JSON.parse(schemaFile); const validate = ajv.compile(schema); let errors = false; @@ -34,7 +35,7 @@ async function main() { validateJSONSchema(file, json); validateFileContents(file, entry); } catch (e) { - error(`Failed to process ${file}: ${err.message}`, { file }); + error(`Failed to process ${file}: ${e.message}`, { file }); } }), ); @@ -77,7 +78,7 @@ function validateFileContents(file, entry) { if (entry.url === `https://${entry.domain}`) error(`Unnecessary url element defined.`, { file }); - if (entry.img === `${entry.domain}.svg`) + if (entry['img'] === `${entry.domain}.svg`) error(`Unnecessary img element defined.`, { file }); if (file !== `entries/${entry.domain[0]}/${valid_name}`) diff --git a/tests/schemas/APIv3.json b/tests/schemas/APIv3.json new file mode 100644 index 00000000000..d51156b1b42 --- /dev/null +++ b/tests/schemas/APIv3.json @@ -0,0 +1,179 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": [ + { + "type": "string", + "description": "The name of the service" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "domain", + "keywords" + ], + "properties": { + "domain": { + "type": "string", + "format": "hostname" + }, + "url": { + "type": "string", + "format": "uri" + }, + "img": { + "type": "string", + "pattern": "^[a-z0-9_\\-\\+\\.]+\\.(png|svg)$" + }, + "tfa": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "sms", + "call", + "email", + "u2f", + "totp", + "custom-software", + "custom-hardware" + ] + } + }, + "documentation": { + "type": "string", + "format": "uri" + }, + "recovery": { + "type": "string", + "format": "uri" + }, + "notes": { + "type": "string", + "minLength": 10 + }, + "regions": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "pattern": "^-?[a-z]{2}$" + } + }, + "additional-domains": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": [ + { + "type": "string", + "format": "hostname" + } + ] + }, + "keywords": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "pattern": "^[a-z]+$" + } + }, + "contact": { + "type": "object", + "additionalProperties": false, + "minimum": 1, + "properties": { + "email": { + "type": "string", + "format": "email" + }, + "facebook": { + "type": "string", + "minLength": 1 + }, + "twitter": { + "type": "string", + "pattern": "^(\\w){1,15}$" + }, + "form": { + "type": "string", + "format": "uri" + }, + "language": { + "type": "string", + "pattern": "^[a-z]{2}$" + } + }, + "not": { + "allOf": [ + { + "required": [ + "form" + ] + }, + { + "required": [ + "email" + ] + } + ] + } + } + }, + "patternProperties": { + "^custom-[a-z]+$": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": [ + { + "type": "string" + } + ] + } + }, + "oneOf": [ + { + "required": [ + "tfa" + ] + }, + { + "required": [ + "contact" + ] + } + ], + "dependencies": { + "notes": ["tfa"], + "documentation": ["tfa"], + "recovery": ["tfa"], + "custom-software": { + "properties": { + "tfa": { + "contains": {"const": "custom-software"} + } + } + }, + "custom-hardware": { + "properties": { + "tfa": { + "contains": {"const": "custom-hardware"} + } + } + } + } + } + ] + } +} diff --git a/tests/schemas/APIv4.json b/tests/schemas/APIv4.json new file mode 100644 index 00000000000..ce4300bf412 --- /dev/null +++ b/tests/schemas/APIv4.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "API Entry", + "type": "object", + "patternProperties": { + "^[a-z0-9.-]+$": { + "type": "object", + "properties": { + "methods": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "sms", + "call", + "email", + "u2f", + "totp", + "custom-software", + "custom-hardware" + ] + } + }, + "contact": { + "type": "object", + "description": "Contact information for the service.", + "properties": { + "email": { + "type": "string", + "format": "email" + }, + "facebook": { + "type": "string", + "minLength": 1 + }, + "twitter": { + "type": "string", + "pattern": "^(\\w){1,15}$" + }, + "form": { + "type": "string", + "format": "uri" + }, + "language": { + "type": "string", + "pattern": "^[a-z]{2}$" + } + }, + "additionalProperties": false + }, + "custom-software": { + "type": "array", + "description": "Custom software 2FA solutions used by the service.", + "items": { + "type": "string" + } + }, + "custom-hardware": { + "type": "array", + "description": "Custom hardware 2FA solutions used by the service.", + "items": { + "type": "string" + } + }, + "documentation": { + "type": "string", + "description": "URL to the documentation for the entry.", + "format": "uri" + }, + "recovery": { + "type": "string", + "description": "URL to the recovery information for the entry.", + "format": "uri" + }, + "notes": { + "type": "string", + "description": "Additional notes for the entry." + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/tests/schema.json b/tests/schemas/entries.json similarity index 97% rename from tests/schema.json rename to tests/schemas/entries.json index 27901e01cc2..5058356fe0c 100644 --- a/tests/schema.json +++ b/tests/schemas/entries.json @@ -1,4 +1,6 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "2FA Directory entries", "type": "object", "maximum": 1, "minimum": 1,