diff --git a/Dockerfile b/Dockerfile index 7a4602fd05e..e5bcad55230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ RUN chmod +x /metrics/source/app/action/index.mjs \ # Based on https://github.com/github/linguist and https://github.com/github/licensed && apt-get install -y ruby-full \ && apt-get install -y git g++ cmake pkg-config libicu-dev zlib1g-dev libcurl4-openssl-dev libssl-dev ruby-dev \ - && gem install github-linguist -v 7.15.0 \ && gem install licensed \ # Install python for node-gyp && apt-get install -y python3 \ @@ -35,4 +34,4 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true ENV PUPPETEER_BROWSER_PATH "google-chrome-stable" # Execute GitHub action -ENTRYPOINT node /metrics/source/app/action/index.mjs \ No newline at end of file +ENTRYPOINT node /metrics/source/app/action/index.mjs diff --git a/package-lock.json b/package-lock.json index c46f81b74e3..71ebae22bea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", + "linguist-js": "^1.4.3", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", @@ -4672,6 +4673,11 @@ "node": ">= 6" } }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" + }, "node_modules/global": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", @@ -6335,6 +6341,21 @@ "node": ">=10" } }, + "node_modules/linguist-js": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz", + "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==", + "dependencies": { + "glob": "^7.1.7", + "glob-to-regexp": "^0.4.1", + "js-yaml": "^4.1.0", + "node-fetch": "^2.6.1", + "yargs-parser": "^20.2.9" + }, + "bin": { + "linguist": "bin/index.js" + } + }, "node_modules/load-bmfont": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", @@ -9899,10 +9920,9 @@ } }, "node_modules/yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", - "dev": true, + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "engines": { "node": ">=10" } @@ -13581,6 +13601,11 @@ "is-glob": "^4.0.1" } }, + "glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" + }, "global": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", @@ -14852,6 +14877,18 @@ "nan": "~2.14.0" } }, + "linguist-js": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/linguist-js/-/linguist-js-1.4.3.tgz", + "integrity": "sha512-lAfJS6ZIlmCfZmefIUUwYO2Bdr6eIPcAVurEbOXR6KaXDmEH9ATiI/SOrnN9djTuatJQG7AnqJUSmud+7KHHrA==", + "requires": { + "glob": "^7.1.7", + "glob-to-regexp": "^0.4.1", + "js-yaml": "^4.1.0", + "node-fetch": "^2.6.1", + "yargs-parser": "^20.2.9" + } + }, "load-bmfont": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", @@ -17687,10 +17724,9 @@ } }, "yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", - "dev": true + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" }, "yauzl": { "version": "2.10.0", diff --git a/package.json b/package.json index 378118bd838..b711c69234d 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "faker": "^5.5.3", "jimp": "^0.16.1", "js-yaml": "^4.1.0", + "linguist-js": "^1.4.3", "marked": "^2.1.3", "memory-cache": "^0.2.0", "node-chartist": "^1.0.5", diff --git a/source/plugins/habits/index.mjs b/source/plugins/habits/index.mjs index 6ba61e9acfd..220db707978 100644 --- a/source/plugins/habits/index.mjs +++ b/source/plugins/habits/index.mjs @@ -100,7 +100,7 @@ export default async function({login, data, rest, imports, q, account}, {enabled if (charts) { //Check if linguist exists console.debug(`metrics/compute/${login}/plugins > habits > searching recently used languages using linguist`) - if ((patches.length) && (await imports.which("github-linguist"))) { + if (patches.length) { //Call language analyzer (note: using content from other plugin is usually disallowed, this is mostly for legacy purposes) habits.linguist.available = true const {total, stats} = await recent_analyzer({login, data, imports, rest, account}, {days, load:from || 1000, tempdir:"habits"}) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index daf0926abf1..0ab69e4ed6d 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -1,8 +1,7 @@ +import linguist from "linguist-js" + /**Indepth analyzer */ export async function indepth({login, data, imports, repositories}, {skipped}) { - //Check prerequisites - if (!await imports.which("github-linguist")) - throw new Error("Feature requires github-linguist") //Compute repositories stats from fetched repositories const results = {total:0, lines:{}, stats:{}, commits:0, files:0, missed:0} @@ -46,9 +45,6 @@ export async function indepth({login, data, imports, repositories}, {skipped}) { /**Recent languages activity */ export async function recent({login, data, imports, rest, account}, {skipped = [], days = 0, load = 0, tempdir = "recent"}) { - //Check prerequisites - if (!await imports.which("github-linguist")) - throw new Error("Feature requires github-linguist") //Get user recent activity console.debug(`metrics/compute/${login}/plugins > languages > querying api`) @@ -141,9 +137,9 @@ export async function recent({login, data, imports, rest, account}, {skipped = [ /**Analyze a single repository */ async function analyze({login, imports, data}, {results, path}) { - //Spawn linguist process and map files to languages + //Gather language data console.debug(`metrics/compute/${login}/plugins > languages > indepth > running linguist`) - const files = Object.fromEntries(Object.entries(JSON.parse(await imports.run("github-linguist --json", {cwd:path}, {log:false}))).flatMap(([lang, files]) => files.map(file => [file, lang]))) + const {results:files, languages:languageResults} = await linguist(path) //Processing diff const per_page = 1 @@ -168,8 +164,10 @@ async function analyze({login, imports, data}, {results, path}) { return //File marker if (/^[+]{3}\sb[/](?[\s\S]+)$/.test(line)) { - file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file ?? null + file = line.match(/^[+]{3}\sb[/](?[\s\S]+)$/)?.groups?.file.replace(/^/, `${path}/`) ?? null lang = files[file] ?? null + if (lang in languageResults.data || lang in languageResults.prose) + lang = null edited.add(file) return }