From d54ad911f372cd0465cc56ee63333e698aab74eb Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 16:59:03 +1200 Subject: [PATCH 1/9] Replace `github-linguist` with `linguist-js` Using a Node port of Linguist makes language data gathering less prone to breakage as Metrics previously relied on `console.log` output to fetch statistics, where SemVer doesn't apply. --- package-lock.json | 52 ++++++++++++++++++++++---- package.json | 1 + source/plugins/languages/analyzers.mjs | 12 ++---- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index c46f81b74e3..e7da754ecec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", + "linguist-js": "^1.4.1", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", @@ -4672,6 +4673,11 @@ "node": ">= 6" } }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" + }, "node_modules/global": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", @@ -6335,6 +6341,21 @@ "node": ">=10" } }, + "node_modules/linguist-js": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.1.tgz", + "integrity": "sha512-VCr9FkFf6vn1U++YCtoFhXzZnNj98jBK/YID3UQrmR311HigJTSjplnZAQ1A+AvQnEfI1Lhe4gLtjJYvyLL3Pw==", + "dependencies": { + "glob": "^7.1.7", + "glob-to-regexp": "^0.4.1", + "js-yaml": "^4.1.0", + "node-fetch": "^2.6.1", + "yargs-parser": "^20.2.9" + }, + "bin": { + "linguist": "bin/index.js" + } + }, "node_modules/load-bmfont": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", @@ -9899,10 +9920,9 @@ } }, "node_modules/yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", - "dev": true, + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "engines": { "node": ">=10" } @@ -13581,6 +13601,11 @@ "is-glob": "^4.0.1" } }, + "glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" + }, "global": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", @@ -14852,6 +14877,18 @@ "nan": "~2.14.0" } }, + "linguist-js": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.1.tgz", + "integrity": "sha512-VCr9FkFf6vn1U++YCtoFhXzZnNj98jBK/YID3UQrmR311HigJTSjplnZAQ1A+AvQnEfI1Lhe4gLtjJYvyLL3Pw==", + "requires": { + "glob": "^7.1.7", + "glob-to-regexp": "^0.4.1", + "js-yaml": "^4.1.0", + "node-fetch": "^2.6.1", + "yargs-parser": "^20.2.9" + } + }, "load-bmfont": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", @@ -17687,10 +17724,9 @@ } }, "yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", - "dev": true + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" }, "yauzl": { "version": "2.10.0", diff --git a/package.json b/package.json index 378118bd838..198035e4d14 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", + "linguist-js": "^1.4.1", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index daf0926abf1..52785e1f9c1 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -1,8 +1,7 @@ +const linguist = require('linguist-js'); + /**Indepth analyzer */ export async function indepth({login, data, imports, repositories}, {skipped}) { - //Check prerequisites - if (!await imports.which("github-linguist")) - throw new Error("Feature requires github-linguist") //Compute repositories stats from fetched repositories const results = {total:0, lines:{}, stats:{}, commits:0, files:0, missed:0} @@ -46,9 +45,6 @@ export async function indepth({login, data, imports, repositories}, {skipped}) { /**Recent languages activity */ export async function recent({login, data, imports, rest, account}, {skipped = [], days = 0, load = 0, tempdir = "recent"}) { - //Check prerequisites - if (!await imports.which("github-linguist")) - throw new Error("Feature requires github-linguist") //Get user recent activity console.debug(`metrics/compute/${login}/plugins > languages > querying api`) @@ -141,9 +137,9 @@ export async function recent({login, data, imports, rest, account}, {skipped = [ /**Analyze a single repository */ async function analyze({login, imports, data}, {results, path}) { - //Spawn linguist process and map files to languages + //Gather language data console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) - const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang]))) + const {files} = await linguist(path) //Processing diff const per_page = 1 From 747b3f8821fa59a6a5e23ab68e846853caddcb80 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 17:07:57 +1200 Subject: [PATCH 2/9] Use ESModule import --- source/plugins/languages/analyzers.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index 52785e1f9c1..1293a2f9839 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -1,4 +1,4 @@ -const linguist = require('linguist-js'); +import linguist from "linguist-js" /**Indepth analyzer */ export async function indepth({login, data, imports, repositories}, {skipped}) { From 9fb9c5f58851475c96c757aa9b5b0268b1e1c4b8 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 17:14:15 +1200 Subject: [PATCH 3/9] Fix property access --- source/plugins/languages/analyzers.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index 1293a2f9839..f43666ed778 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -139,7 +139,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [ async function analyze({login, imports, data}, {results, path}) { //Gather language data console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) - const {files} = await linguist(path) + const {results:files} = await linguist(path) //Processing diff const per_page = 1 From 57c20380d06d58fb81ce030e6d585cc85f6b25c8 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 17:43:04 +1200 Subject: [PATCH 4/9] Update linguist-js Version 1.4.2 fixes a crash --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e7da754ecec..59951a60be2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", - "linguist-js": "^1.4.1", + "linguist-js": "^1.4.2", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", @@ -6342,9 +6342,9 @@ } }, "node_modules/linguist-js": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.1.tgz", - "integrity": "sha512-VCr9FkFf6vn1U++YCtoFhXzZnNj98jBK/YID3UQrmR311HigJTSjplnZAQ1A+AvQnEfI1Lhe4gLtjJYvyLL3Pw==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.2.tgz", + "integrity": "sha512-z++i/JhVHpJwcYC2LcCiJDIvSc836Szq3i6YKHMFqnMoPl0RGlf83yJyPl+hZxS/O96rJvVT11UISKe5i3S/tg==", "dependencies": { "glob": "^7.1.7", "glob-to-regexp": "^0.4.1", @@ -14878,9 +14878,9 @@ } }, "linguist-js": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.1.tgz", - "integrity": "sha512-VCr9FkFf6vn1U++YCtoFhXzZnNj98jBK/YID3UQrmR311HigJTSjplnZAQ1A+AvQnEfI1Lhe4gLtjJYvyLL3Pw==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.2.tgz", + "integrity": "sha512-z++i/JhVHpJwcYC2LcCiJDIvSc836Szq3i6YKHMFqnMoPl0RGlf83yJyPl+hZxS/O96rJvVT11UISKe5i3S/tg==", "requires": { "glob": "^7.1.7", "glob-to-regexp": "^0.4.1", diff --git a/package.json b/package.json index 198035e4d14..6902e43d2a0 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", - "linguist-js": "^1.4.1", + "linguist-js": "^1.4.2", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", From 4b7c86e7b2549d8be25456a8ae2d3cec138cc5bc Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 18:11:06 +1200 Subject: [PATCH 5/9] Change file matcher --- source/plugins/languages/analyzers.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index f43666ed778..faca8c766a4 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -164,7 +164,7 @@ async function analyze({login, imports, data}, {results, path}) { return //File marker if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { - file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file.replace(/^/, path) ?? null lang = files[file] ?? null edited.add(file) return From 08480c3b0f02e3f3b014717984dd05377a0f3bf3 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 19:26:46 +1200 Subject: [PATCH 6/9] Update linguist-js Version 1.4.3 mandates consistent absolute paths in output --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59951a60be2..71ebae22bea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", - "linguist-js": "^1.4.2", + "linguist-js": "^1.4.3", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", @@ -6342,9 +6342,9 @@ } }, "node_modules/linguist-js": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.2.tgz", - "integrity": "sha512-z++i/JhVHpJwcYC2LcCiJDIvSc836Szq3i6YKHMFqnMoPl0RGlf83yJyPl+hZxS/O96rJvVT11UISKe5i3S/tg==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz", + "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==", "dependencies": { "glob": "^7.1.7", "glob-to-regexp": "^0.4.1", @@ -14878,9 +14878,9 @@ } }, "linguist-js": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.2.tgz", - "integrity": "sha512-z++i/JhVHpJwcYC2LcCiJDIvSc836Szq3i6YKHMFqnMoPl0RGlf83yJyPl+hZxS/O96rJvVT11UISKe5i3S/tg==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz", + "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==", "requires": { "glob": "^7.1.7", "glob-to-regexp": "^0.4.1", diff --git a/package.json b/package.json index 6902e43d2a0..b711c69234d 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", - "linguist-js": "^1.4.2", + "linguist-js": "^1.4.3", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", From e5ec7422862f8c070265c45882e4c9fa12e28364 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 19:36:29 +1200 Subject: [PATCH 7/9] Change path replacer --- source/plugins/languages/analyzers.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index faca8c766a4..d7030ded247 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -164,7 +164,7 @@ async function analyze({login, imports, data}, {results, path}) { return //File marker if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { - file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file.replace(/^/, path) ?? null + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file.replace(/^/, `${path}/`) ?? null lang = files[file] ?? null edited.add(file) return From 53d71b95f3a4b501a3a29923409485efb534f74a Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 19:37:13 +1200 Subject: [PATCH 8/9] Ignore data and prose languages --- source/plugins/languages/analyzers.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index d7030ded247..0ab69e4ed6d 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -139,7 +139,7 @@ export async function recent({login, data, imports, rest, account}, {skipped = [ async function analyze({login, imports, data}, {results, path}) { //Gather language data console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) - const {results:files} = await linguist(path) + const {results:files, languages:languageResults} = await linguist(path) //Processing diff const per_page = 1 @@ -166,6 +166,8 @@ async function analyze({login, imports, data}, {results, path}) { if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file.replace(/^/, `${path}/`) ?? null lang = files[file] ?? null + if (lang in languageResults.data || lang in languageResults.prose) + lang = null edited.add(file) return } From d0c943d73ac77d69c721d3d8a8a931e236e89a42 Mon Sep 17 00:00:00 2001 From: Nixinova Date: Sun, 1 Aug 2021 19:53:49 +1200 Subject: [PATCH 9/9] Remove `github-linguist` dependency --- Dockerfile | 3 +-- source/plugins/habits/index.mjs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7a4602fd05e..e5bcad55230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ RUN chmod +x /metrics/source/app/action/index.mjs \ # Based on https://github.com/github/linguist and https://github.com/github/licensed && apt-get install -y ruby-full \ && apt-get install -y git g++ cmake pkg-config libicu-dev zlib1g-dev libcurl4-openssl-dev libssl-dev ruby-dev \ - && gem install github-linguist -v 7.15.0 \ && gem install licensed \ # Install python for node-gyp && apt-get install -y python3 \ @@ -35,4 +34,4 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true ENV PUPPETEER_BROWSER_PATH "google-chrome-stable" # Execute GitHub action -ENTRYPOINT node /metrics/source/app/action/index.mjs \ No newline at end of file +ENTRYPOINT node /metrics/source/app/action/index.mjs diff --git a/source/plugins/habits/index.mjs b/source/plugins/habits/index.mjs index 6ba61e9acfd..220db707978 100644 --- a/source/plugins/habits/index.mjs +++ b/source/plugins/habits/index.mjs @@ -100,7 +100,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled if (charts) { //Check if linguist exists console.debug(`metrics/compute/${login}/plugins > habits > searching recently used languages using linguist`) - if ((patches.length) && (await imports.which("github-linguist"))) { + if (patches.length) { //Call language analyzer (note: using content from other plugin is usually disallowed, this is mostly for legacy purposes) habits.linguist.available = true const {total, stats} = await recent_analyzer({login, data, imports, rest, account}, {days, load:from || 1000, tempdir:"habits"})