From e981149391a14dd16b3b13953d7fea53b7b6553f Mon Sep 17 00:00:00 2001 From: RedYetiDev <38299977+RedYetiDev@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:17:00 -0400 Subject: [PATCH] test_runner: report coverage thresholds in `test:coverage` --- doc/api/test.md | 5 ++ lib/internal/main/test_runner.js | 7 ++- lib/internal/test_runner/harness.js | 3 + lib/internal/test_runner/test.js | 13 +++-- .../test-runner-coverage-thresholds.js | 57 +++++++++++++++---- test/parallel/test-runner-coverage.js | 6 ++ 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 19819202d1d2e1..c04a8f21582bee 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -2814,6 +2814,11 @@ are defined, while others are emitted in the order that the tests execute. numbers and the number of times they were covered. * `line` {number} The line number. * `count` {number} The number of times the line was covered. + * `thresholds` {Object} An object containing whether or not the coverage for + each coverage type. + * `function` {boolean} Whether or not the coverage surpassed the `function` threshold. + * `branch` {boolean} Whether or not the coverage surpassed the `branch` threshold. + * `line` {boolean} Whether or not the coverage surpassed the `line` threshold. * `totals` {Object} An object containing a summary of coverage for all files. * `totalLineCount` {number} The total number of lines. diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index b1f69b07771ac6..ce6af8adb9324c 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -31,8 +31,13 @@ if (isUsingInspector() && options.isolation === 'process') { options.globPatterns = ArrayPrototypeSlice(process.argv, 1); debug('test runner configuration:', options); -run(options).on('test:fail', (data) => { +const ran = run(options); +ran.on('test:fail', (data) => { if (data.todo === undefined || data.todo === false) { process.exitCode = kGenericUserError; } }); + +ran.on('test:coverage', ({ summary: { thresholds: { line, branch, function: func } } }) => { + if (!line || !branch || !func) process.exitCode = kGenericUserError; +}); diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 1bc6cddabd41a0..ce2e9ee0c58312 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -250,6 +250,9 @@ function lazyBootstrapRoot() { process.exitCode = kGenericUserError; } }); + // globalRoot.reporter.on('test:coverage', ({ summary: { thresholds: { line, branch, function: func }}}) => { + // if (!line || !branch || !func) process.exitCode = kGenericUserError; + // }); globalRoot.harness.bootstrapPromise = globalOptions.setup(globalRoot.reporter); } return globalRoot; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 09387f89c36c34..28e24d82d5ab60 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -29,7 +29,6 @@ const { SymbolDispose, } = primordials; const { getCallerLocation } = internalBinding('util'); -const { exitCodes: { kGenericUserError } } = internalBinding('errors'); const { addAbortListener } = require('internal/events/abort_listener'); const { queueMicrotask } = require('internal/process/task_queues'); const { AsyncResource } = require('async_hooks'); @@ -1010,8 +1009,12 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`); if (coverage) { - reporter.coverage(nesting, loc, coverage); - + coverage.thresholds = { + __proto__: null, + line: true, + branch: true, + function: true, + }; const coverages = [ { __proto__: null, actual: coverage.totals.coveredLinePercent, threshold: this.config.lineCoverage, name: 'line' }, @@ -1026,10 +1029,12 @@ class Test extends AsyncResource { for (let i = 0; i < coverages.length; i++) { const { threshold, actual, name } = coverages[i]; if (actual < threshold) { - process.exitCode = kGenericUserError; + coverage.thresholds[name] = false; reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`); } } + + reporter.coverage(nesting, loc, coverage); } if (harness.watching) { diff --git a/test/parallel/test-runner-coverage-thresholds.js b/test/parallel/test-runner-coverage-thresholds.js index 8ee15f4c65b7fe..0715c819296302 100644 --- a/test/parallel/test-runner-coverage-thresholds.js +++ b/test/parallel/test-runner-coverage-thresholds.js @@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() { } const fixture = fixtures.path('test-runner', 'coverage.js'); -const neededArguments = [ - '--experimental-test-coverage', - '--test-reporter', 'tap', -]; +const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs'); const coverages = [ { flag: '--test-coverage-lines', name: 'line', actual: 78.35 }, @@ -56,10 +53,12 @@ const coverages = [ ]; for (const coverage of coverages) { - test(`test passing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=25`, + '--test-reporter', 'tap', fixture, ]); @@ -70,10 +69,27 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test failing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=25`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert(stdout.summary.thresholds[coverage.name]); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test failing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=99`, + '--test-reporter', 'tap', fixture, ]); @@ -84,9 +100,25 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too high)`, async (t) => { + test(`test failing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=99`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert(!stdout.summary.thresholds[coverage.name]); + assert.strictEqual(result.status, 1); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test out-of-range ${coverage.flag} (too high)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=101`, fixture, ]); @@ -96,9 +128,10 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too low)`, async (t) => { + test(`test out-of-range ${coverage.flag} (too low)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=-1`, fixture, ]); diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 6cda6d2d1e090f..93b542f4110962 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -253,6 +253,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as const calledTwice = file.functions.find((f) => f.name === 'fnWithControlFlow'); assert.strictEqual(calledTwice.count, 2); assert.strictEqual(calledTwice.line, 35); + + assert.ok(coverage.summary.thresholds.function); }); await t.test('reports on branch coverage', () => { @@ -261,6 +263,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as const calledTwice = file.branches.find((b) => b.line === 35); assert.strictEqual(calledTwice.count, 2); + + assert.ok(coverage.summary.thresholds.branch); }); await t.test('reports on line coverage', () => { @@ -278,6 +282,8 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as const testLine = file.lines.find((l) => l.line === line.line); assert.strictEqual(testLine.count, line.count); }); + + assert.ok(coverage.summary.thresholds.line); }); });