From ea450054ab1da91764083b1b4403a8d7f669b17e Mon Sep 17 00:00:00 2001 From: Boris Lykah Date: Sun, 20 Feb 2022 17:56:44 -0700 Subject: [PATCH 1/4] Collect and report number of changes --- __tests__/filter.test.ts | 6 +- __tests__/git.test.ts | 17 ++++- dist/index.js | 150 +++++++++++++++++++++++---------------- src/file.ts | 10 ++- src/filter.ts | 4 +- src/git.ts | 128 +++++++++++++++++++-------------- src/main.ts | 66 ++++++++++++++--- 7 files changed, 250 insertions(+), 131 deletions(-) diff --git a/__tests__/filter.test.ts b/__tests__/filter.test.ts index be2a1487..b1f4bbb1 100644 --- a/__tests__/filter.test.ts +++ b/__tests__/filter.test.ts @@ -150,7 +150,7 @@ describe('matching specific change status', () => { - added: "**/*" ` let filter = new Filter(yaml) - const files = [{status: ChangeStatus.Added, filename: 'file.js'}] + const files = [{status: ChangeStatus.Added, filename: 'file.js', additions: 1, deletions: 0}] const match = filter.match(files) expect(match.add).toEqual(files) }) @@ -161,7 +161,7 @@ describe('matching specific change status', () => { - added|modified: "**/*" ` let filter = new Filter(yaml) - const files = [{status: ChangeStatus.Modified, filename: 'file.js'}] + const files = [{status: ChangeStatus.Modified, filename: 'file.js', additions: 1, deletions: 1}] const match = filter.match(files) expect(match.addOrModify).toEqual(files) }) @@ -183,6 +183,6 @@ describe('matching specific change status', () => { function modified(paths: string[]): File[] { return paths.map(filename => { - return {filename, status: ChangeStatus.Modified} + return {filename, status: ChangeStatus.Modified, additions: 1, deletions: 1} }) } diff --git a/__tests__/git.test.ts b/__tests__/git.test.ts index 9645221e..d543dce6 100644 --- a/__tests__/git.test.ts +++ b/__tests__/git.test.ts @@ -2,8 +2,8 @@ import * as git from '../src/git' import {ChangeStatus} from '../src/file' describe('parsing output of the git diff command', () => { - test('parseGitDiffOutput returns files with correct change status', async () => { - const files = git.parseGitDiffOutput( + test('parseGitDiffNameStatusOutput returns files with correct change status', async () => { + const files = git.parseGitDiffNameStatusOutput( 'A\u0000LICENSE\u0000' + 'M\u0000src/index.ts\u0000' + 'D\u0000src/main.ts\u0000' ) expect(files.length).toBe(3) @@ -14,6 +14,19 @@ describe('parsing output of the git diff command', () => { expect(files[2].filename).toBe('src/main.ts') expect(files[2].status).toBe(ChangeStatus.Deleted) }) + + test('parseGitDiffNumstatOutput returns files with correct change status', async () => { + const files = git.parseGitDiffNumstatOutput( + '4\t2\tLICENSE\u0000' + '5\t0\tsrc/index.ts\u0000' + ) + expect(files.length).toBe(2) + expect(files[0].filename).toBe('LICENSE') + expect(files[0].additions).toBe(4) + expect(files[0].deletions).toBe(2) + expect(files[1].filename).toBe('src/index.ts') + expect(files[1].additions).toBe(5) + expect(files[1].deletions).toBe(0) + }) }) describe('git utility function tests (those not invoking git)', () => { diff --git a/dist/index.js b/dist/index.js index 7f134f10..83a9b337 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3811,54 +3811,39 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.isGitSha = exports.getShortName = exports.getCurrentRef = exports.listAllFilesAsAdded = exports.parseGitDiffOutput = exports.getChangesSinceMergeBase = exports.getChangesOnHead = exports.getChanges = exports.getChangesInLastCommit = exports.HEAD = exports.NULL_SHA = void 0; +exports.isGitSha = exports.getShortName = exports.getCurrentRef = exports.listAllFilesAsAdded = exports.parseGitDiffNumstatOutput = exports.getGitDiffStatusNumstat = exports.parseGitDiffNameStatusOutput = exports.getChangesSinceMergeBase = exports.getChangesOnHead = exports.getChanges = exports.getChangesInLastCommit = exports.HEAD = exports.NULL_SHA = void 0; const exec_1 = __importDefault(__webpack_require__(807)); const core = __importStar(__webpack_require__(470)); const file_1 = __webpack_require__(258); exports.NULL_SHA = '0000000000000000000000000000000000000000'; exports.HEAD = 'HEAD'; async function getChangesInLastCommit() { - core.startGroup(`Change detection in last commit`); - let output = ''; - try { - output = (await exec_1.default('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + return core.group(`Change detection in last commit`, async () => { + try { + // Calling git log on the last commit works when only the last commit may be checked out. Calling git diff HEAD^..HEAD needs two commits. + const statusOutput = (await exec_1.default('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout; + const numstatOutput = (await exec_1.default('git', ['log', '--format=', '--no-renames', '--numstat', '-z', '-n', '1'])).stdout; + const statusFiles = parseGitDiffNameStatusOutput(statusOutput); + const numstatFiles = parseGitDiffNumstatOutput(numstatOutput); + return mergeStatusNumstat(statusFiles, numstatFiles); + } + finally { + fixStdOutNullTermination(); + } + }); } exports.getChangesInLastCommit = getChangesInLastCommit; async function getChanges(base, head) { const baseRef = await ensureRefAvailable(base); const headRef = await ensureRefAvailable(head); // Get differences between ref and HEAD - core.startGroup(`Change detection ${base}..${head}`); - let output = ''; - try { - // Two dots '..' change detection - directly compares two versions - output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..${headRef}`])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + // Two dots '..' change detection - directly compares two versions + return core.group(`Change detection ${base}..${head}`, () => getGitDiffStatusNumstat(`${baseRef}..${headRef}`)); } exports.getChanges = getChanges; async function getChangesOnHead() { // Get current changes - both staged and unstaged - core.startGroup(`Change detection on HEAD`); - let output = ''; - try { - output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', 'HEAD'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + return core.group(`Change detection on HEAD`, () => getGitDiffStatusNumstat(`HEAD`)); } exports.getChangesOnHead = getChangesOnHead; async function getChangesSinceMergeBase(base, head, initialFetchDepth) { @@ -3923,19 +3908,30 @@ async function getChangesSinceMergeBase(base, head, initialFetchDepth) { diffArg = `${baseRef}..${headRef}`; } // Get changes introduced on ref compared to base - core.startGroup(`Change detection ${diffArg}`); + return getGitDiffStatusNumstat(diffArg); +} +exports.getChangesSinceMergeBase = getChangesSinceMergeBase; +async function gitDiffNameStatus(diffArg) { let output = ''; try { output = (await exec_1.default('git', ['diff', '--no-renames', '--name-status', '-z', diffArg])).stdout; } finally { fixStdOutNullTermination(); - core.endGroup(); } - return parseGitDiffOutput(output); + return output; } -exports.getChangesSinceMergeBase = getChangesSinceMergeBase; -function parseGitDiffOutput(output) { +async function gitDiffNumstat(diffArg) { + let output = ''; + try { + output = (await exec_1.default('git', ['diff', '--no-renames', '--numstat', '-z', diffArg])).stdout; + } + finally { + fixStdOutNullTermination(); + } + return output; +} +function parseGitDiffNameStatusOutput(output) { const tokens = output.split('\u0000').filter(s => s.length > 0); const files = []; for (let i = 0; i + 1 < tokens.length; i += 2) { @@ -3946,24 +3942,44 @@ function parseGitDiffOutput(output) { } return files; } -exports.parseGitDiffOutput = parseGitDiffOutput; +exports.parseGitDiffNameStatusOutput = parseGitDiffNameStatusOutput; +function mergeStatusNumstat(statusEntries, numstatEntries) { + const statusMap = {}; + statusEntries.forEach(f => statusMap[f.filename] = f); + return numstatEntries.map(f => { + const status = statusMap[f.filename]; + if (!status) { + throw new Error(`Cannot find the status entry for file: ${f.filename}`); + } + return { ...f, status: status.status }; + }); +} +async function getGitDiffStatusNumstat(diffArg) { + const statusFiles = await gitDiffNameStatus(diffArg).then(parseGitDiffNameStatusOutput); + const numstatFiles = await gitDiffNumstat(diffArg).then(parseGitDiffNumstatOutput); + return mergeStatusNumstat(statusFiles, numstatFiles); +} +exports.getGitDiffStatusNumstat = getGitDiffStatusNumstat; +function parseGitDiffNumstatOutput(output) { + const rows = output.split('\u0000').filter(s => s.length > 0); + return rows.map(row => { + const tokens = row.split('\t'); + // For the binary files set the numbers to zero. This matches the response of Github API. + const additions = tokens[0] == '-' ? 0 : Number.parseInt(tokens[0]); + const deletions = tokens[1] == '-' ? 0 : Number.parseInt(tokens[1]); + return { + filename: tokens[2], + additions, + deletions, + }; + }); +} +exports.parseGitDiffNumstatOutput = parseGitDiffNumstatOutput; async function listAllFilesAsAdded() { - core.startGroup('Listing all files tracked by git'); - let output = ''; - try { - output = (await exec_1.default('git', ['ls-files', '-z'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return output - .split('\u0000') - .filter(s => s.length > 0) - .map(path => ({ - status: file_1.ChangeStatus.Added, - filename: path - })); + return core.group(`Listing all files tracked by git`, async () => { + const emptyTreeHash = (await exec_1.default('git', ['hash-object', '-t', 'tree', '/dev/null'])).stdout; + return getGitDiffStatusNumstat(emptyTreeHash); + }); } exports.listAllFilesAsAdded = listAllFilesAsAdded; async function getCurrentRef() { @@ -4717,6 +4733,7 @@ async function run() { const filtersInput = core.getInput('filters', { required: true }); const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput; const listFiles = core.getInput('list-files', { required: false }).toLowerCase() || 'none'; + const stat = core.getInput('stat', { required: false }).toLowerCase() || 'none'; const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', { required: false })) || 10; if (!isExportFormat(listFiles)) { core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFiles}'`); @@ -4857,12 +4874,16 @@ async function getChangedFilesFromApi(token, prNumber) { if (row.status === file_1.ChangeStatus.Renamed) { files.push({ filename: row.filename, - status: file_1.ChangeStatus.Added + status: file_1.ChangeStatus.Added, + additions: row.additions, + deletions: row.deletions, }); files.push({ // 'previous_filename' for some unknown reason isn't in the type definition or documentation filename: row.previous_filename, - status: file_1.ChangeStatus.Deleted + status: file_1.ChangeStatus.Deleted, + additions: row.additions, + deletions: row.deletions, }); } else { @@ -4870,7 +4891,9 @@ async function getChangedFilesFromApi(token, prNumber) { const status = row.status === 'removed' ? file_1.ChangeStatus.Deleted : row.status; files.push({ filename: row.filename, - status + status, + additions: row.additions, + deletions: row.deletions, }); } } @@ -4885,8 +4908,8 @@ function exportResults(results, format) { core.info('Results:'); const changes = []; for (const [key, files] of Object.entries(results)) { - const value = files.length > 0; - core.startGroup(`Filter ${key} = ${value}`); + const hasMatchingFiles = files.length > 0; + core.startGroup(`Filter ${key} = ${hasMatchingFiles}`); if (files.length > 0) { changes.push(key); core.info('Matching files:'); @@ -4897,12 +4920,17 @@ function exportResults(results, format) { else { core.info('Matching files: none'); } - core.setOutput(key, value); + core.setOutput(key, hasMatchingFiles); core.setOutput(`${key}_count`, files.length); if (format !== 'none') { const filesValue = serializeExport(files, format); core.setOutput(`${key}_files`, filesValue); } + const additionCount = files.reduce((sum, f) => sum + f.additions, 0); + const deletionCount = files.reduce((sum, f) => sum + f.deletions, 0); + core.setOutput(`${key}_addition_count`, additionCount); + core.setOutput(`${key}_deletion_count`, deletionCount); + core.setOutput(`${key}_change_count`, additionCount + deletionCount); core.endGroup(); } if (results['changes'] === undefined) { @@ -5171,7 +5199,7 @@ module.exports = require("https"); /***/ 215: /***/ (function(module) { -module.exports = {"_args":[["@octokit/rest@16.43.1","C:\\Users\\Michal\\Workspace\\dorny\\pr-changed-files-filter"]],"_from":"@octokit/rest@16.43.1","_id":"@octokit/rest@16.43.1","_inBundle":false,"_integrity":"sha512-gfFKwRT/wFxq5qlNjnW2dh+qh74XgTQ2B179UX5K1HYCluioWj8Ndbgqw2PVqa1NnVJkGHp2ovMpVn/DImlmkw==","_location":"/@octokit/rest","_phantomChildren":{"@types/node":"14.0.5","deprecation":"2.3.1","once":"1.4.0","os-name":"3.1.0"},"_requested":{"type":"version","registry":true,"raw":"@octokit/rest@16.43.1","name":"@octokit/rest","escapedName":"@octokit%2frest","scope":"@octokit","rawSpec":"16.43.1","saveSpec":null,"fetchSpec":"16.43.1"},"_requiredBy":["/@actions/github"],"_resolved":"https://registry.npmjs.org/@octokit/rest/-/rest-16.43.1.tgz","_spec":"16.43.1","_where":"C:\\Users\\Michal\\Workspace\\dorny\\pr-changed-files-filter","author":{"name":"Gregor Martynus","url":"https://github.com/gr2m"},"bugs":{"url":"https://github.com/octokit/rest.js/issues"},"bundlesize":[{"path":"./dist/octokit-rest.min.js.gz","maxSize":"33 kB"}],"contributors":[{"name":"Mike de Boer","email":"info@mikedeboer.nl"},{"name":"Fabian Jakobs","email":"fabian@c9.io"},{"name":"Joe Gallo","email":"joe@brassafrax.com"},{"name":"Gregor Martynus","url":"https://github.com/gr2m"}],"dependencies":{"@octokit/auth-token":"^2.4.0","@octokit/plugin-paginate-rest":"^1.1.1","@octokit/plugin-request-log":"^1.0.0","@octokit/plugin-rest-endpoint-methods":"2.4.0","@octokit/request":"^5.2.0","@octokit/request-error":"^1.0.2","atob-lite":"^2.0.0","before-after-hook":"^2.0.0","btoa-lite":"^1.0.0","deprecation":"^2.0.0","lodash.get":"^4.4.2","lodash.set":"^4.3.2","lodash.uniq":"^4.5.0","octokit-pagination-methods":"^1.1.0","once":"^1.4.0","universal-user-agent":"^4.0.0"},"description":"GitHub REST API client for Node.js","devDependencies":{"@gimenete/type-writer":"^0.1.3","@octokit/auth":"^1.1.1","@octokit/fixtures-server":"^5.0.6","@octokit/graphql":"^4.2.0","@types/node":"^13.1.0","bundlesize":"^0.18.0","chai":"^4.1.2","compression-webpack-plugin":"^3.1.0","cypress":"^3.0.0","glob":"^7.1.2","http-proxy-agent":"^4.0.0","lodash.camelcase":"^4.3.0","lodash.merge":"^4.6.1","lodash.upperfirst":"^4.3.1","lolex":"^5.1.2","mkdirp":"^1.0.0","mocha":"^7.0.1","mustache":"^4.0.0","nock":"^11.3.3","npm-run-all":"^4.1.2","nyc":"^15.0.0","prettier":"^1.14.2","proxy":"^1.0.0","semantic-release":"^17.0.0","sinon":"^8.0.0","sinon-chai":"^3.0.0","sort-keys":"^4.0.0","string-to-arraybuffer":"^1.0.0","string-to-jsdoc-comment":"^1.0.0","typescript":"^3.3.1","webpack":"^4.0.0","webpack-bundle-analyzer":"^3.0.0","webpack-cli":"^3.0.0"},"files":["index.js","index.d.ts","lib","plugins"],"homepage":"https://github.com/octokit/rest.js#readme","keywords":["octokit","github","rest","api-client"],"license":"MIT","name":"@octokit/rest","nyc":{"ignore":["test"]},"publishConfig":{"access":"public"},"release":{"publish":["@semantic-release/npm",{"path":"@semantic-release/github","assets":["dist/*","!dist/*.map.gz"]}]},"repository":{"type":"git","url":"git+https://github.com/octokit/rest.js.git"},"scripts":{"build":"npm-run-all build:*","build:browser":"npm-run-all build:browser:*","build:browser:development":"webpack --mode development --entry . --output-library=Octokit --output=./dist/octokit-rest.js --profile --json > dist/bundle-stats.json","build:browser:production":"webpack --mode production --entry . --plugin=compression-webpack-plugin --output-library=Octokit --output-path=./dist --output-filename=octokit-rest.min.js --devtool source-map","build:ts":"npm run -s update-endpoints:typescript","coverage":"nyc report --reporter=html && open coverage/index.html","generate-bundle-report":"webpack-bundle-analyzer dist/bundle-stats.json --mode=static --no-open --report dist/bundle-report.html","lint":"prettier --check '{lib,plugins,scripts,test}/**/*.{js,json,ts}' 'docs/*.{js,json}' 'docs/src/**/*' index.js README.md package.json","lint:fix":"prettier --write '{lib,plugins,scripts,test}/**/*.{js,json,ts}' 'docs/*.{js,json}' 'docs/src/**/*' index.js README.md package.json","postvalidate:ts":"tsc --noEmit --target es6 test/typescript-validate.ts","prebuild:browser":"mkdirp dist/","pretest":"npm run -s lint","prevalidate:ts":"npm run -s build:ts","start-fixtures-server":"octokit-fixtures-server","test":"nyc mocha test/mocha-node-setup.js \"test/*/**/*-test.js\"","test:browser":"cypress run --browser chrome","update-endpoints":"npm-run-all update-endpoints:*","update-endpoints:fetch-json":"node scripts/update-endpoints/fetch-json","update-endpoints:typescript":"node scripts/update-endpoints/typescript","validate:ts":"tsc --target es6 --noImplicitAny index.d.ts"},"types":"index.d.ts","version":"16.43.1"}; +module.exports = {"name":"@octokit/rest","version":"16.43.1","publishConfig":{"access":"public"},"description":"GitHub REST API client for Node.js","keywords":["octokit","github","rest","api-client"],"author":"Gregor Martynus (https://github.com/gr2m)","contributors":[{"name":"Mike de Boer","email":"info@mikedeboer.nl"},{"name":"Fabian Jakobs","email":"fabian@c9.io"},{"name":"Joe Gallo","email":"joe@brassafrax.com"},{"name":"Gregor Martynus","url":"https://github.com/gr2m"}],"repository":"https://github.com/octokit/rest.js","dependencies":{"@octokit/auth-token":"^2.4.0","@octokit/plugin-paginate-rest":"^1.1.1","@octokit/plugin-request-log":"^1.0.0","@octokit/plugin-rest-endpoint-methods":"2.4.0","@octokit/request":"^5.2.0","@octokit/request-error":"^1.0.2","atob-lite":"^2.0.0","before-after-hook":"^2.0.0","btoa-lite":"^1.0.0","deprecation":"^2.0.0","lodash.get":"^4.4.2","lodash.set":"^4.3.2","lodash.uniq":"^4.5.0","octokit-pagination-methods":"^1.1.0","once":"^1.4.0","universal-user-agent":"^4.0.0"},"devDependencies":{"@gimenete/type-writer":"^0.1.3","@octokit/auth":"^1.1.1","@octokit/fixtures-server":"^5.0.6","@octokit/graphql":"^4.2.0","@types/node":"^13.1.0","bundlesize":"^0.18.0","chai":"^4.1.2","compression-webpack-plugin":"^3.1.0","cypress":"^3.0.0","glob":"^7.1.2","http-proxy-agent":"^4.0.0","lodash.camelcase":"^4.3.0","lodash.merge":"^4.6.1","lodash.upperfirst":"^4.3.1","lolex":"^5.1.2","mkdirp":"^1.0.0","mocha":"^7.0.1","mustache":"^4.0.0","nock":"^11.3.3","npm-run-all":"^4.1.2","nyc":"^15.0.0","prettier":"^1.14.2","proxy":"^1.0.0","semantic-release":"^17.0.0","sinon":"^8.0.0","sinon-chai":"^3.0.0","sort-keys":"^4.0.0","string-to-arraybuffer":"^1.0.0","string-to-jsdoc-comment":"^1.0.0","typescript":"^3.3.1","webpack":"^4.0.0","webpack-bundle-analyzer":"^3.0.0","webpack-cli":"^3.0.0"},"types":"index.d.ts","scripts":{"coverage":"nyc report --reporter=html && open coverage/index.html","lint":"prettier --check '{lib,plugins,scripts,test}/**/*.{js,json,ts}' 'docs/*.{js,json}' 'docs/src/**/*' index.js README.md package.json","lint:fix":"prettier --write '{lib,plugins,scripts,test}/**/*.{js,json,ts}' 'docs/*.{js,json}' 'docs/src/**/*' index.js README.md package.json","pretest":"npm run -s lint","test":"nyc mocha test/mocha-node-setup.js \"test/*/**/*-test.js\"","test:browser":"cypress run --browser chrome","build":"npm-run-all build:*","build:ts":"npm run -s update-endpoints:typescript","prebuild:browser":"mkdirp dist/","build:browser":"npm-run-all build:browser:*","build:browser:development":"webpack --mode development --entry . --output-library=Octokit --output=./dist/octokit-rest.js --profile --json > dist/bundle-stats.json","build:browser:production":"webpack --mode production --entry . --plugin=compression-webpack-plugin --output-library=Octokit --output-path=./dist --output-filename=octokit-rest.min.js --devtool source-map","generate-bundle-report":"webpack-bundle-analyzer dist/bundle-stats.json --mode=static --no-open --report dist/bundle-report.html","update-endpoints":"npm-run-all update-endpoints:*","update-endpoints:fetch-json":"node scripts/update-endpoints/fetch-json","update-endpoints:typescript":"node scripts/update-endpoints/typescript","prevalidate:ts":"npm run -s build:ts","validate:ts":"tsc --target es6 --noImplicitAny index.d.ts","postvalidate:ts":"tsc --noEmit --target es6 test/typescript-validate.ts","start-fixtures-server":"octokit-fixtures-server"},"license":"MIT","files":["index.js","index.d.ts","lib","plugins"],"nyc":{"ignore":["test"]},"release":{"publish":["@semantic-release/npm",{"path":"@semantic-release/github","assets":["dist/*","!dist/*.map.gz"]}]},"bundlesize":[{"path":"./dist/octokit-rest.min.js.gz","maxSize":"33 kB"}]}; /***/ }), diff --git a/src/file.ts b/src/file.ts index d8125a7a..b5d3ea27 100644 --- a/src/file.ts +++ b/src/file.ts @@ -1,8 +1,16 @@ -export interface File { +export interface FileStatus { filename: string status: ChangeStatus } +export interface FileNumstat { + filename: string + additions: number + deletions: number +} + +export type File = FileStatus & FileNumstat + export enum ChangeStatus { Added = 'added', Copied = 'copied', diff --git a/src/filter.ts b/src/filter.ts index cfc74a3c..417c24eb 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -1,6 +1,6 @@ import * as jsyaml from 'js-yaml' import picomatch from 'picomatch' -import {File, ChangeStatus} from './file' +import {File, ChangeStatus, FileStatus} from './file' // Type definition of object we expect to load from YAML interface FilterYaml { @@ -61,7 +61,7 @@ export class Filter { return result } - private isMatch(file: File, patterns: FilterRuleItem[]): boolean { + private isMatch(file: FileStatus, patterns: FilterRuleItem[]): boolean { return patterns.some( rule => (rule.status === undefined || rule.status.includes(file.status)) && rule.isMatch(file.filename) ) diff --git a/src/git.ts b/src/git.ts index 3d6d3be1..ba3acea5 100644 --- a/src/git.ts +++ b/src/git.ts @@ -1,21 +1,23 @@ import exec from './exec' import * as core from '@actions/core' -import {File, ChangeStatus} from './file' +import {File, ChangeStatus, FileNumstat, FileStatus} from './file' export const NULL_SHA = '0000000000000000000000000000000000000000' export const HEAD = 'HEAD' export async function getChangesInLastCommit(): Promise { - core.startGroup(`Change detection in last commit`) - let output = '' - try { - output = (await exec('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + return core.group(`Change detection in last commit`, async () => { + try { + // Calling git log on the last commit works when only the last commit may be checked out. Calling git diff HEAD^..HEAD needs two commits. + const statusOutput = (await exec('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout + const numstatOutput = (await exec('git', ['log', '--format=', '--no-renames', '--numstat', '-z', '-n', '1'])).stdout + const statusFiles = parseGitDiffNameStatusOutput(statusOutput) + const numstatFiles = parseGitDiffNumstatOutput(numstatOutput) + return mergeStatusNumstat(statusFiles, numstatFiles) + } finally { + fixStdOutNullTermination() + } + }) } export async function getChanges(base: string, head: string): Promise { @@ -23,31 +25,17 @@ export async function getChanges(base: string, head: string): Promise { const headRef = await ensureRefAvailable(head) // Get differences between ref and HEAD - core.startGroup(`Change detection ${base}..${head}`) - let output = '' - try { - // Two dots '..' change detection - directly compares two versions - output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..${headRef}`])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + // Two dots '..' change detection - directly compares two versions + return core.group(`Change detection ${base}..${head}`, () => + getGitDiffStatusNumstat(`${baseRef}..${headRef}`) + ) } export async function getChangesOnHead(): Promise { // Get current changes - both staged and unstaged - core.startGroup(`Change detection on HEAD`) - let output = '' - try { - output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', 'HEAD'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + return core.group(`Change detection on HEAD`, () => + getGitDiffStatusNumstat(`HEAD`) + ) } export async function getChangesSinceMergeBase(base: string, head: string, initialFetchDepth: number): Promise { @@ -119,21 +107,32 @@ export async function getChangesSinceMergeBase(base: string, head: string, initi } // Get changes introduced on ref compared to base - core.startGroup(`Change detection ${diffArg}`) + return getGitDiffStatusNumstat(diffArg) +} + +async function gitDiffNameStatus(diffArg: string): Promise { let output = '' try { output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', diffArg])).stdout } finally { fixStdOutNullTermination() - core.endGroup() } + return output +} - return parseGitDiffOutput(output) +async function gitDiffNumstat(diffArg: string): Promise { + let output = '' + try { + output = (await exec('git', ['diff', '--no-renames', '--numstat', '-z', diffArg])).stdout + } finally { + fixStdOutNullTermination() + } + return output } -export function parseGitDiffOutput(output: string): File[] { +export function parseGitDiffNameStatusOutput(output: string): FileStatus[] { const tokens = output.split('\u0000').filter(s => s.length > 0) - const files: File[] = [] + const files: FileStatus[] = [] for (let i = 0; i + 1 < tokens.length; i += 2) { files.push({ status: statusMap[tokens[i]], @@ -143,23 +142,46 @@ export function parseGitDiffOutput(output: string): File[] { return files } -export async function listAllFilesAsAdded(): Promise { - core.startGroup('Listing all files tracked by git') - let output = '' - try { - output = (await exec('git', ['ls-files', '-z'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } +function mergeStatusNumstat(statusEntries: FileStatus[], numstatEntries: FileNumstat[]): File[] { + const statusMap: {[key: string]: FileStatus} = {} + statusEntries.forEach(f => statusMap[f.filename] = f) - return output - .split('\u0000') - .filter(s => s.length > 0) - .map(path => ({ - status: ChangeStatus.Added, - filename: path - })) + return numstatEntries.map(f => { + const status = statusMap[f.filename] + if (!status) { + throw new Error(`Cannot find the status entry for file: ${f.filename}`); + } + return {...f, status: status.status} + }) +} + +export async function getGitDiffStatusNumstat(diffArg: string) { + const statusFiles = await gitDiffNameStatus(diffArg).then(parseGitDiffNameStatusOutput) + const numstatFiles = await gitDiffNumstat(diffArg).then(parseGitDiffNumstatOutput) + return mergeStatusNumstat(statusFiles, numstatFiles) +} + +export function parseGitDiffNumstatOutput(output: string): FileNumstat[] { + const rows = output.split('\u0000').filter(s => s.length > 0) + return rows.map(row => { + const tokens = row.split('\t') + // For the binary files set the numbers to zero. This matches the response of Github API. + const additions = tokens[0] == '-' ? 0 : Number.parseInt(tokens[0]) + const deletions = tokens[1] == '-' ? 0 : Number.parseInt(tokens[1]) + return { + filename: tokens[2], + additions, + deletions, + } + }) +} + + +export async function listAllFilesAsAdded(): Promise { + return core.group(`Listing all files tracked by git`, async () => { + const emptyTreeHash = (await exec('git', ['hash-object', '-t', 'tree', '/dev/null'])).stdout + return getGitDiffStatusNumstat(emptyTreeHash) + }) } export async function getCurrentRef(): Promise { diff --git a/src/main.ts b/src/main.ts index d2cb6784..62b011bc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,6 +24,7 @@ async function run(): Promise { const filtersInput = core.getInput('filters', {required: true}) const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput const listFiles = core.getInput('list-files', {required: false}).toLowerCase() || 'none' + const stat = core.getInput('stat', {required: false}).toLowerCase() || 'none' const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', {required: false})) || 10 if (!isExportFormat(listFiles)) { @@ -194,19 +195,25 @@ async function getChangedFilesFromApi( if (row.status === ChangeStatus.Renamed) { files.push({ filename: row.filename, - status: ChangeStatus.Added + status: ChangeStatus.Added, + additions: row.additions, + deletions: row.deletions, }) files.push({ // 'previous_filename' for some unknown reason isn't in the type definition or documentation filename: (row).previous_filename as string, - status: ChangeStatus.Deleted + status: ChangeStatus.Deleted, + additions: row.additions, + deletions: row.deletions, }) } else { // Github status and git status variants are same except for deleted files const status = row.status === 'removed' ? ChangeStatus.Deleted : (row.status as ChangeStatus) files.push({ filename: row.filename, - status + status, + additions: row.additions, + deletions: row.deletions, }) } } @@ -218,12 +225,20 @@ async function getChangedFilesFromApi( } } +interface Stat { + additionCount: number, + deletionCount: number, + fileCount: number +} + function exportResults(results: FilterResults, format: ExportFormat): void { core.info('Results:') - const changes = [] + const changes: string[] = [] + const changeStats: {[key: string]: Stat} = {} + for (const [key, files] of Object.entries(results)) { - const value = files.length > 0 - core.startGroup(`Filter ${key} = ${value}`) + const hasMatchingFiles = files.length > 0 + core.startGroup(`Filter ${key} = ${hasMatchingFiles}`) if (files.length > 0) { changes.push(key) core.info('Matching files:') @@ -234,12 +249,22 @@ function exportResults(results: FilterResults, format: ExportFormat): void { core.info('Matching files: none') } - core.setOutput(key, value) + core.setOutput(key, hasMatchingFiles) core.setOutput(`${key}_count`, files.length) if (format !== 'none') { - const filesValue = serializeExport(files, format) + const filesValue = serializeExportChangedFiles(files, format) core.setOutput(`${key}_files`, filesValue) } + + const additionCount: number = files.reduce((sum, f) => sum + f.additions, 0) + const deletionCount: number = files.reduce((sum, f) => sum + f.deletions, 0) + core.setOutput(`${key}_addition_count`, additionCount) + core.setOutput(`${key}_deletion_count`, deletionCount) + core.setOutput(`${key}_change_count`, additionCount + deletionCount) + changeStats[key] = { + additionCount, deletionCount, fileCount: files.length + } + core.endGroup() } @@ -250,9 +275,14 @@ function exportResults(results: FilterResults, format: ExportFormat): void { } else { core.info('Cannot set changes output variable - name already used by filter output') } + + if (format !== 'none') { + const statValue = serializeExportStat(changeStats, format) + core.setOutput(`stat`, statValue) + } } -function serializeExport(files: File[], format: ExportFormat): string { +function serializeExportChangedFiles(files: File[], format: ExportFormat): string { const fileNames = files.map(file => file.filename) switch (format) { case 'csv': @@ -268,6 +298,24 @@ function serializeExport(files: File[], format: ExportFormat): string { } } +function serializeExportStat(stat: {[key: string]: Stat}, format: ExportFormat): string { + switch (format) { + case 'csv': + return Object.keys(stat).sort().map(k => + [csvEscape(k), stat[k].additionCount, stat[k].deletionCount, stat[k].fileCount] + .join(',') + ).join('\n') + case 'json': + return JSON.stringify(stat) + case 'escape': + return '' + case 'shell': + return '' + default: + return '' + } +} + function isExportFormat(value: string): value is ExportFormat { return ['none', 'csv', 'shell', 'json', 'escape'].includes(value) } From bc6c444323e90c82f80328485727812f3f46f87a Mon Sep 17 00:00:00 2001 From: Boris Lykah Date: Sun, 20 Feb 2022 21:55:46 -0700 Subject: [PATCH 2/4] Add stat format option --- dist/index.js | 46 ++++++++++++++++++++++++++++++++++++---------- src/main.ts | 42 ++++++++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 28 deletions(-) diff --git a/dist/index.js b/dist/index.js index 83a9b337..01016be3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -4732,18 +4732,22 @@ async function run() { const base = core.getInput('base', { required: false }); const filtersInput = core.getInput('filters', { required: true }); const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput; - const listFiles = core.getInput('list-files', { required: false }).toLowerCase() || 'none'; - const stat = core.getInput('stat', { required: false }).toLowerCase() || 'none'; + const listFilesFormat = core.getInput('list-files', { required: false }).toLowerCase() || 'none'; + const statFormat = core.getInput('stat', { required: false }).toLowerCase() || 'none'; const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', { required: false })) || 10; - if (!isExportFormat(listFiles)) { - core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFiles}'`); + if (!isFilesExportFormat(listFilesFormat)) { + core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFilesFormat}'`); + return; + } + if (!isStatExportFormat(statFormat)) { + core.setFailed(`Input parameter 'stat' is set to invalid value '${statFormat}'`); return; } const filter = new filter_1.Filter(filtersYaml); const files = await getChangedFiles(token, base, ref, initialFetchDepth); core.info(`Detected ${files.length} changed files`); const results = filter.match(files); - exportResults(results, listFiles); + exportResults(results, listFilesFormat, statFormat); } catch (error) { core.setFailed(error.message); @@ -4904,9 +4908,10 @@ async function getChangedFilesFromApi(token, prNumber) { core.endGroup(); } } -function exportResults(results, format) { +function exportResults(results, filesFormat, statFormat) { core.info('Results:'); const changes = []; + const changeStats = {}; for (const [key, files] of Object.entries(results)) { const hasMatchingFiles = files.length > 0; core.startGroup(`Filter ${key} = ${hasMatchingFiles}`); @@ -4922,8 +4927,8 @@ function exportResults(results, format) { } core.setOutput(key, hasMatchingFiles); core.setOutput(`${key}_count`, files.length); - if (format !== 'none') { - const filesValue = serializeExport(files, format); + if (filesFormat !== 'none') { + const filesValue = serializeExportChangedFiles(files, filesFormat); core.setOutput(`${key}_files`, filesValue); } const additionCount = files.reduce((sum, f) => sum + f.additions, 0); @@ -4931,6 +4936,9 @@ function exportResults(results, format) { core.setOutput(`${key}_addition_count`, additionCount); core.setOutput(`${key}_deletion_count`, deletionCount); core.setOutput(`${key}_change_count`, additionCount + deletionCount); + changeStats[key] = { + additionCount, deletionCount, fileCount: files.length + }; core.endGroup(); } if (results['changes'] === undefined) { @@ -4941,8 +4949,12 @@ function exportResults(results, format) { else { core.info('Cannot set changes output variable - name already used by filter output'); } + if (statFormat !== 'none') { + const statValue = serializeExportStat(changeStats, statFormat); + core.setOutput(`stat`, statValue); + } } -function serializeExport(files, format) { +function serializeExportChangedFiles(files, format) { const fileNames = files.map(file => file.filename); switch (format) { case 'csv': @@ -4957,7 +4969,21 @@ function serializeExport(files, format) { return ''; } } -function isExportFormat(value) { +function serializeExportStat(stat, format) { + switch (format) { + case 'csv': + return Object.keys(stat).sort().map(k => [csv_escape_1.csvEscape(k), stat[k].additionCount, stat[k].deletionCount, stat[k].fileCount] + .join(',')).join('\n'); + case 'json': + return JSON.stringify(stat); + default: + return ''; + } +} +function isFilesExportFormat(value) { + return ['none', 'csv', 'shell', 'json', 'escape'].includes(value); +} +function isStatExportFormat(value) { return ['none', 'csv', 'shell', 'json', 'escape'].includes(value); } run(); diff --git a/src/main.ts b/src/main.ts index 62b011bc..ffc49023 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,8 @@ import * as git from './git' import {backslashEscape, shellEscape} from './list-format/shell-escape' import {csvEscape} from './list-format/csv-escape' -type ExportFormat = 'none' | 'csv' | 'json' | 'shell' | 'escape' +type FilesExportFormat = 'none' | 'csv' | 'json' | 'shell' | 'escape' +type StatExportFormat = 'none' | 'csv' | 'json' async function run(): Promise { try { @@ -23,12 +24,17 @@ async function run(): Promise { const base = core.getInput('base', {required: false}) const filtersInput = core.getInput('filters', {required: true}) const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput - const listFiles = core.getInput('list-files', {required: false}).toLowerCase() || 'none' - const stat = core.getInput('stat', {required: false}).toLowerCase() || 'none' + const listFilesFormat = core.getInput('list-files', {required: false}).toLowerCase() || 'none' + const statFormat = core.getInput('stat', {required: false}).toLowerCase() || 'none' const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', {required: false})) || 10 - if (!isExportFormat(listFiles)) { - core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFiles}'`) + if (!isFilesExportFormat(listFilesFormat)) { + core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFilesFormat}'`) + return + } + + if (!isStatExportFormat(statFormat)) { + core.setFailed(`Input parameter 'stat' is set to invalid value '${statFormat}'`) return } @@ -36,7 +42,7 @@ async function run(): Promise { const files = await getChangedFiles(token, base, ref, initialFetchDepth) core.info(`Detected ${files.length} changed files`) const results = filter.match(files) - exportResults(results, listFiles) + exportResults(results, listFilesFormat, statFormat) } catch (error) { core.setFailed(error.message) } @@ -231,7 +237,7 @@ interface Stat { fileCount: number } -function exportResults(results: FilterResults, format: ExportFormat): void { +function exportResults(results: FilterResults, filesFormat: FilesExportFormat, statFormat: StatExportFormat): void { core.info('Results:') const changes: string[] = [] const changeStats: {[key: string]: Stat} = {} @@ -251,8 +257,8 @@ function exportResults(results: FilterResults, format: ExportFormat): void { core.setOutput(key, hasMatchingFiles) core.setOutput(`${key}_count`, files.length) - if (format !== 'none') { - const filesValue = serializeExportChangedFiles(files, format) + if (filesFormat !== 'none') { + const filesValue = serializeExportChangedFiles(files, filesFormat) core.setOutput(`${key}_files`, filesValue) } @@ -276,13 +282,13 @@ function exportResults(results: FilterResults, format: ExportFormat): void { core.info('Cannot set changes output variable - name already used by filter output') } - if (format !== 'none') { - const statValue = serializeExportStat(changeStats, format) + if (statFormat !== 'none') { + const statValue = serializeExportStat(changeStats, statFormat) core.setOutput(`stat`, statValue) } } -function serializeExportChangedFiles(files: File[], format: ExportFormat): string { +function serializeExportChangedFiles(files: File[], format: FilesExportFormat): string { const fileNames = files.map(file => file.filename) switch (format) { case 'csv': @@ -298,7 +304,7 @@ function serializeExportChangedFiles(files: File[], format: ExportFormat): strin } } -function serializeExportStat(stat: {[key: string]: Stat}, format: ExportFormat): string { +function serializeExportStat(stat: {[key: string]: Stat}, format: StatExportFormat): string { switch (format) { case 'csv': return Object.keys(stat).sort().map(k => @@ -307,17 +313,17 @@ function serializeExportStat(stat: {[key: string]: Stat}, format: ExportFormat): ).join('\n') case 'json': return JSON.stringify(stat) - case 'escape': - return '' - case 'shell': - return '' default: return '' } } -function isExportFormat(value: string): value is ExportFormat { +function isFilesExportFormat(value: string): value is FilesExportFormat { return ['none', 'csv', 'shell', 'json', 'escape'].includes(value) } +function isStatExportFormat(value: string): value is StatExportFormat { + return ['none', 'csv', 'json'].includes(value) +} + run() From d6715dc71e93eec955b8010a5e01c02a16e2d070 Mon Sep 17 00:00:00 2001 From: Boris Lykah Date: Mon, 21 Feb 2022 00:17:28 -0700 Subject: [PATCH 3/4] Report unmatched files --- dist/index.js | 5 +++++ src/filter.ts | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/dist/index.js b/dist/index.js index 01016be3..600b2085 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5405,6 +5405,11 @@ class Filter { for (const [key, patterns] of Object.entries(this.rules)) { result[key] = files.filter(file => this.isMatch(file, patterns)); } + if (!this.rules.hasOwnProperty('other')) { + const matchingFilenamesList = Object.values(result).flatMap(filteredFiles => filteredFiles.map(file => file.filename)); + const matchingFilenamesSet = new Set(matchingFilenamesList); + result.other = files.filter(file => !matchingFilenamesSet.has(file.filename)); + } return result; } isMatch(file, patterns) { diff --git a/src/filter.ts b/src/filter.ts index 417c24eb..125fb0b4 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -58,6 +58,13 @@ export class Filter { for (const [key, patterns] of Object.entries(this.rules)) { result[key] = files.filter(file => this.isMatch(file, patterns)) } + + if (!this.rules.hasOwnProperty('other')) { + const matchingFilenamesList = Object.values(result).flatMap(filteredFiles => filteredFiles.map(file => file.filename)) + const matchingFilenamesSet = new Set(matchingFilenamesList) + result.other = files.filter(file => !matchingFilenamesSet.has(file.filename)) + } + return result } From e835d01d598268bd6edbd70faccbcd1fb4b6e83e Mon Sep 17 00:00:00 2001 From: Boris Lykah Date: Fri, 11 Mar 2022 14:07:52 -0700 Subject: [PATCH 4/4] Document stat feature --- README.md | 29 +++++++++++++++++++++++++++++ action.yml | 10 ++++++++++ 2 files changed, 39 insertions(+) diff --git a/README.md b/README.md index 106b471a..e84497e3 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ For more scenarios see [examples](#examples) section. ## What's New +- Add `stat` parameter that enables output of the file changes statistics per filter. - Add `ref` input parameter - Add `list-files: csv` format - Configure matrix job to run for each folder with changes using `changes` output @@ -160,6 +161,7 @@ For more information, see [CHANGELOG](https://github.com/dorny/paths-filter/blob - For each filter, it sets an output variable with the name `${FILTER_NAME}_count` to the count of matching files. - If enabled, for each filter it sets an output variable with the name `${FILTER_NAME}_files`. It will contain a list of all files matching the filter. - `changes` - JSON array with names of all filters matching any of the changed files. +- If `stat` input is set to an output format, the output variable `stat` contains JSON or CSV value with the change statistics for each filter. ## Examples @@ -504,6 +506,33 @@ jobs: +
+ Passing number of added lines from a filter to another action + +```yaml +- uses: dorny/paths-filter@v2 + id: filter + with: + # Enable listing of diff stat matching each filter. + # Paths to files will be available in `stat` output variable. + # Stat will be formatted as JSON object + stat: json + + # In this example all changed files are passed to the following action to do + # some custom processing. + filters: | + changed: + - '**' +- name: Lint Markdown + uses: johndoe/some-action@v1 + # Run action only if the change is large enough. + if: ${{fromJson(steps.filter.outputs.stat).changed.additionCount > 1000}} + with: + files: ${{ steps.filter.outputs.changed_files }} +``` + +
+ ## See also - [test-reporter](https://github.com/dorny/test-reporter) - Displays test results from popular testing frameworks directly in GitHub diff --git a/action.yml b/action.yml index dc515281..65128ddc 100644 --- a/action.yml +++ b/action.yml @@ -36,6 +36,16 @@ inputs: Backslash escapes every potentially unsafe character. required: true default: none + stat: + description: | + Enables listing of that enables output of the file change statistics per filter, similar to `git diff --shortstat`. + If some changes do not match any filter, the output includes an additional entry with the filter name 'other'. + 'none' - Disables listing of stats (default). + 'csv' - Coma separated list that has name of filter, count of additions, count of deletions, count of changed files. + If needed it uses double quotes to wrap name of filter with unsafe characters. For example, `"some filter",12,7,2`. + 'json' - Serialized as JSON object where the filter names are keys. For example, `{"some filter": {"additionCount": 12, "deletionCount": 7, "fileCount": 2}}` + required: false + default: none initial-fetch-depth: description: | How many commits are initially fetched from base branch.