Skip to content

Commit

Permalink
test_runner: count nested tests
Browse files Browse the repository at this point in the history
PR-URL: #47094
Fixes: #46762
Reviewed-By: Colin Ihrig <[email protected]>
Reviewed-By: Benjamin Gruenbaum <[email protected]>
  • Loading branch information
MoLow authored and RafaelGSS committed Apr 7, 2023
1 parent 7ec87fd commit 4530582
Show file tree
Hide file tree
Showing 21 changed files with 163 additions and 93 deletions.
11 changes: 11 additions & 0 deletions lib/internal/test_runner/harness.js
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,17 @@ function setup(root) {
__proto__: null,
bootstrapComplete: false,
coverage: null,
counters: {
__proto__: null,
all: 0,
failed: 0,
passed: 0,
cancelled: 0,
skipped: 0,
todo: 0,
planned: 0,
suites: 0,
},
};
root.startTime = hrtime();
return root;
Expand Down
1 change: 1 addition & 0 deletions lib/internal/test_runner/reporter/tap.js
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ function reportDetails(nesting, data = kEmptyObject) {
let details = `${_indent} ---\n`;

details += jsToYaml(_indent, 'duration_ms', duration_ms);
details += jsToYaml(_indent, 'type', data.type);
details += jsToYaml(_indent, null, error);
details += `${_indent} ...\n`;
return details;
Expand Down
33 changes: 16 additions & 17 deletions lib/internal/test_runner/runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,8 @@ const {
ArrayPrototypeSome,
ArrayPrototypeSort,
ArrayPrototypeSplice,
FunctionPrototypeCall,
Number,
ObjectAssign,
ObjectKeys,
PromisePrototypeThen,
SafePromiseAll,
SafePromiseAllReturnVoid,
Expand Down Expand Up @@ -55,8 +53,9 @@ const { YAMLToJs } = require('internal/test_runner/yaml_to_js');
const { TokenKind } = require('internal/test_runner/tap_lexer');

const {
isSupportedFileType,
countCompletedTest,
doesPathMatchFilter,
isSupportedFileType,
} = require('internal/test_runner/utils');
const { basename, join, resolve } = require('path');
const { once } = require('events');
Expand All @@ -67,7 +66,7 @@ const {

const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch'];
const kFilterArgValues = ['--test-reporter', '--test-reporter-destination'];
const kDiagnosticsFilterArgs = ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];

const kCanceledTests = new SafeSet()
.add(kCancelledByParent).add(kAborted).add(kTestTimeoutFailure);
Expand Down Expand Up @@ -151,10 +150,10 @@ function getRunArgs({ path, inspectPort }) {

class FileTest extends Test {
#buffer = [];
#counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
#reportedChildren = 0;
failedSubtests = false;
#skipReporting() {
return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
}
#checkNestedComment({ comment }) {
const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
Expand Down Expand Up @@ -204,11 +203,19 @@ class FileTest extends Test {
const method = pass ? 'ok' : 'fail';
this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
if (nesting === 0) {
FunctionPrototypeCall(super.countSubtest,
{ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled },
this.#counters);
this.failedSubtests ||= !pass;
}
this.#reportedChildren++;
countCompletedTest({
name: node.description,
finished: true,
skipped: skip,
isTodo: todo,
passed: pass,
cancelled,
nesting,
reportedType: diagnostics.type,
}, this.root.harness);
break;

}
Expand All @@ -233,14 +240,6 @@ class FileTest extends Test {
this.reportStarted();
this.#handleReportItem(ast);
}
countSubtest(counters) {
if (this.#counters.all === 0) {
return super.countSubtest(counters);
}
ArrayPrototypeForEach(ObjectKeys(counters), (key) => {
counters[key] += this.#counters[key];
});
}
reportStarted() {}
report() {
const skipReporting = this.#skipReporting();
Expand Down
59 changes: 24 additions & 35 deletions lib/internal/test_runner/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const { MockTracker } = require('internal/test_runner/mock');
const { TestsStream } = require('internal/test_runner/tests_stream');
const {
createDeferredCallback,
countCompletedTest,
isTestFailureError,
parseCommandLine,
} = require('internal/test_runner/utils');
Expand Down Expand Up @@ -186,6 +187,7 @@ class Test extends AsyncResource {
this.runOnlySubtests = this.only;
this.testNumber = 0;
this.timeout = kDefaultTimeout;
this.root = this;
} else {
const nesting = parent.parent === null ? parent.nesting :
parent.nesting + 1;
Expand All @@ -197,6 +199,7 @@ class Test extends AsyncResource {
this.runOnlySubtests = !this.only;
this.testNumber = parent.subtests.length + 1;
this.timeout = parent.timeout;
this.root = parent.root;
}

switch (typeof concurrency) {
Expand Down Expand Up @@ -575,31 +578,7 @@ class Test extends AsyncResource {
this.postRun();
}

countSubtest(counters) {
// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (this.skipped) {
counters.skipped++;
} else if (this.isTodo) {
counters.todo++;
} else if (this.cancelled) {
counters.cancelled++;
} else if (!this.passed) {
counters.failed++;
} else {
counters.passed++;
}

if (!this.passed) {
counters.totalFailed++;
}
counters.all++;
}

postRun(pendingSubtestsError) {
const counters = {
__proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0,
};
// If the test was failed before it even started, then the end time will
// be earlier than the start time. Correct that here.
if (this.endTime < this.startTime) {
Expand All @@ -610,19 +589,22 @@ class Test extends AsyncResource {
// The test has run, so recursively cancel any outstanding subtests and
// mark this test as failed if any subtests failed.
this.pendingSubtests = [];
let failed = 0;
for (let i = 0; i < this.subtests.length; i++) {
const subtest = this.subtests[i];

if (!subtest.finished) {
subtest.#cancel(pendingSubtestsError);
subtest.postRun(pendingSubtestsError);
}
subtest.countSubtest(counters);
if (!subtest.passed) {
failed++;
}
}

if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
const msg = `${counters.totalFailed} ${subtestString} failed`;
if ((this.passed || this.parent === null) && failed > 0) {
const subtestString = `subtest${failed > 1 ? 's' : ''}`;
const msg = `${failed} ${subtestString} failed`;

this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
}
Expand All @@ -637,18 +619,19 @@ class Test extends AsyncResource {
this.parent.processPendingSubtests();
} else if (!this.reported) {
this.reported = true;
this.reporter.plan(this.nesting, kFilename, counters.all);
this.reporter.plan(this.nesting, kFilename, this.root.harness.counters.planned);

for (let i = 0; i < this.diagnostics.length; i++) {
this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
}

this.reporter.diagnostic(this.nesting, kFilename, `tests ${counters.all}`);
this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`);
this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`);
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`);
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${counters.skipped}`);
this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`);
this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.root.harness.counters.all}`);
this.reporter.diagnostic(this.nesting, kFilename, `suites ${this.root.harness.counters.suites}`);
this.reporter.diagnostic(this.nesting, kFilename, `pass ${this.root.harness.counters.passed}`);
this.reporter.diagnostic(this.nesting, kFilename, `fail ${this.root.harness.counters.failed}`);
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${this.root.harness.counters.cancelled}`);
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${this.root.harness.counters.skipped}`);
this.reporter.diagnostic(this.nesting, kFilename, `todo ${this.root.harness.counters.todo}`);
this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`);

if (this.harness?.coverage) {
Expand Down Expand Up @@ -689,6 +672,7 @@ class Test extends AsyncResource {
}

report() {
countCompletedTest(this);
if (this.subtests.length > 0) {
this.reporter.plan(this.subtests[0].nesting, kFilename, this.subtests.length);
} else {
Expand All @@ -703,6 +687,10 @@ class Test extends AsyncResource {
directive = this.reporter.getTodo(this.message);
}

if (this.reportedType) {
details.type = this.reportedType;
}

if (this.passed) {
this.reporter.ok(this.nesting, kFilename, this.testNumber, this.name, details, directive);
} else {
Expand Down Expand Up @@ -746,6 +734,7 @@ class TestHook extends Test {
}

class Suite extends Test {
reportedType = 'suite';
constructor(options) {
super(options);

Expand Down
25 changes: 25 additions & 0 deletions lib/internal/test_runner/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,33 @@ function parseCommandLine() {
return globalTestOptions;
}

function countCompletedTest(test, harness = test.root.harness) {
if (test.nesting === 0) {
harness.counters.planned++;
}
if (test.reportedType === 'suite') {
harness.counters.suites++;
return;
}
// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (test.skipped) {
harness.counters.skipped++;
} else if (test.isTodo) {
harness.counters.todo++;
} else if (test.cancelled) {
harness.counters.cancelled++;
} else if (!test.passed) {
harness.counters.failed++;
} else {
harness.counters.passed++;
}
harness.counters.all++;
}

module.exports = {
convertStringToRegExp,
countCompletedTest,
createDeferredCallback,
doesPathMatchFilter,
isSupportedFileType,
Expand Down
7 changes: 4 additions & 3 deletions test/message/test_runner_abort.out
Original file line number Diff line number Diff line change
Expand Up @@ -260,10 +260,11 @@ not ok 4 - callback abort signal
*
...
1..4
# tests 4
# pass 0
# tests 22
# suites 0
# pass 8
# fail 0
# cancelled 4
# cancelled 14
# skipped 0
# todo 0
# duration_ms *
9 changes: 6 additions & 3 deletions test/message/test_runner_abort_suite.out
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ TAP version 13
not ok 1 - describe timeout signal
---
duration_ms: *
type: 'suite'
failureType: 'testAborted'
error: 'The operation was aborted due to timeout'
code: 23
Expand All @@ -78,6 +79,7 @@ not ok 1 - describe timeout signal
not ok 2 - describe abort signal
---
duration_ms: *
type: 'suite'
failureType: 'testAborted'
error: 'This operation was aborted'
code: 20
Expand All @@ -94,10 +96,11 @@ not ok 2 - describe abort signal
*
...
1..2
# tests 2
# pass 0
# tests 9
# suites 2
# pass 4
# fail 0
# cancelled 2
# cancelled 5
# skipped 0
# todo 0
# duration_ms *
Loading

0 comments on commit 4530582

Please sign in to comment.