Skip to content

Commit

Permalink
test_runner: add initial CLI runner
Browse files Browse the repository at this point in the history
This commit introduces an initial version of a CLI-based
test runner.
  • Loading branch information
cjihrig committed Apr 9, 2022
1 parent 3ac7f86 commit 805361e
Show file tree
Hide file tree
Showing 22 changed files with 571 additions and 128 deletions.
10 changes: 10 additions & 0 deletions doc/api/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -1052,6 +1052,15 @@ minimum allocation from the secure heap. The minimum value is `2`.
The maximum value is the lesser of `--secure-heap` or `2147483647`.
The value given must be a power of two.

### `--test`

<!-- YAML
added: REPLACEME
-->

Starts the Node.js command line test runner. See the documentation on
[running tests from the command line][] for more details.

### `--test-only`

<!-- YAML
Expand Down Expand Up @@ -2033,6 +2042,7 @@ $ node --max-old-space-size=1536 index.js
[jitless]: https://v8.dev/blog/jitless
[libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
[remote code execution]: https://www.owasp.org/index.php/Code_Injection
[running tests from the command line]: test.md#running-tests-from-the-command-line
[security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
[timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
[ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html
44 changes: 44 additions & 0 deletions doc/api/test.md
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,49 @@ test('a test that creates asynchronous activity', (t) => {
});
```

## Running tests from the command line

The Node.js test runner can be invoked from the command line by passing the
[`--test`][] flag. By default, Node.js will recursively search the current
directory for test files:

```bash
node --test
```

Alternatively, one or more paths can be provided as the final argument(s) to
the Node.js command, as shown below.

```bash
node --test test1.js test2.mjs test/
```

In this example, the test runner will execute the files `test1.js` and
`test2.mjs`. The test runner will also recursively search the `test/` directory
for test files to execute.

When searching for test files to execute, the test runner behaves as follows:

* Any files explicitly provided by the user are executed.
* `node_modules` directories are skipped unless explicitly provided by the
user.
* `.js`, `.cjs`, and `.mjs` filenames matching the following patterns are
treated as test files:
* `test` - Files whose basename is the string `'test'`. Examples: `test.js`,
`test.cjs`, `test.mjs`.
* `test-.+` - Files whose basename starts with the string `'test-'` followed
by one or more characters. Examples: `test-example.js`,
`test-another-example.mjs`.
* `.+[\.\-\_]test` - Files whose basename ends with one or more characters
followed by `.test`, `-test`, or `_test`. Examples: `example.test.js`,
`example-test.cjs`, `example_test.mjs`.

Each matching test file is executed in a separate child process. If the child
process finishes with an exit code of 0, the test is considered passing.
Otherwise, the test is considered to be a failure. Test files must be
executable by Node.js, but are not required to use the `node:test` module
internally.

## `test([name][, options][, fn])`

<!-- YAML
Expand Down Expand Up @@ -368,5 +411,6 @@ behaves in the same fashion as the top level [`test()`][] function.

[TAP]: https://testanything.org/
[`--test-only`]: cli.md#--test-only
[`--test`]: cli.md#--test
[`TestContext`]: #class-testcontext
[`test()`]: #testname-options-fn
3 changes: 3 additions & 0 deletions doc/node.1
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,9 @@ the secure heap. The default is 0. The value must be a power of two.
.It Fl -secure-heap-min Ns = Ns Ar n
Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two.
.
.It Fl -test
Starts the Node.js command line test runner.
.
.It Fl -test-only
Configures the test runner to only execute top level tests that have the `only`
option set.
Expand Down
20 changes: 17 additions & 3 deletions lib/internal/errors.js
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() {
desc.set !== undefined;
}

function inspectWithNoCustomRetry(obj, options) {
const utilInspect = lazyInternalUtilInspect();

try {
return utilInspect.inspect(obj, options);
} catch {
return utilInspect.inspect(obj, { ...options, customInspect: false });
}
}

// A specialized Error that includes an additional info property with
// additional information about the error condition.
// It has the properties present in a UVException but with a custom error
Expand Down Expand Up @@ -862,6 +872,7 @@ module.exports = {
getMessage,
hideInternalStackFrames,
hideStackFrames,
inspectWithNoCustomRetry,
isErrorStackTraceLimitWritable,
isStackOverflowError,
kEnhanceStackBeforeInspector,
Expand Down Expand Up @@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) {
assert(typeof failureType === 'string',
"The 'failureType' argument must be of type string.");

const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
let msg = error?.message ?? error;

this.failureType = error?.failureType ?? failureType;
this.cause = error;
if (typeof msg !== 'string') {
msg = inspectWithNoCustomRetry(msg);
}

this.failureType = failureType;
this.cause = error;
return msg;
}, Error);
E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',
Expand Down
135 changes: 135 additions & 0 deletions lib/internal/main/test_runner.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
'use strict';
const {
ArrayFrom,
ArrayPrototypeFilter,
ArrayPrototypeIncludes,
ArrayPrototypePush,
ArrayPrototypeSlice,
ArrayPrototypeSort,
Promise,
SafeSet,
StringPrototypeEndsWith,
} = primordials;
const {
prepareMainThreadExecution,
} = require('internal/bootstrap/pre_execution');
const { spawn } = require('child_process');
const { readdirSync, statSync } = require('fs');
const console = require('internal/console/global');
const {
codes: {
ERR_TEST_FAILURE,
},
} = require('internal/errors');
const test = require('internal/test_runner/harness');
const { kSubtestsFailed } = require('internal/test_runner/test');
const { doesPathMatchFilter } = require('internal/test_runner/utils');
const { join, resolve } = require('path');
const kFilterArgs = ['--test'];

prepareMainThreadExecution(false);
markBootstrapComplete();

// TODO(cjihrig): Replace this with recursive readdir once it lands.
function processPath(path, testFiles, userSupplied = false) {
const stats = statSync(path);

if (stats.isFile()) {
if (userSupplied || doesPathMatchFilter(path)) {
testFiles.add(path);
}
} else if (stats.isDirectory()) {
if (!userSupplied && StringPrototypeEndsWith(path, 'node_modules')) {
return;
}

const entries = readdirSync(path);

for (let i = 0; i < entries.length; i++) {
processPath(join(path, entries[i]), testFiles);
}
}
}

function createTestFileList() {
const testPaths = process.argv.length > 1 ?
ArrayPrototypeSlice(process.argv, 1) : ['.'];
const testFiles = new SafeSet();

try {
for (let i = 0; i < testPaths.length; i++) {
const absolutePath = resolve(testPaths[i]);

processPath(absolutePath, testFiles, true);
}
} catch (err) {
if (err?.code === 'ENOENT') {
console.error(`Could not find '${err.path}'`);
process.exit(1);
}

throw err;
}

return ArrayPrototypeSort(ArrayFrom(testFiles));
}

function runTestFile(path) {
return test(path, () => {
return new Promise((resolve, reject) => {
const args = ArrayPrototypeFilter(process.execArgv, (arg) => {
return !ArrayPrototypeIncludes(kFilterArgs, arg);
});
ArrayPrototypePush(args, path);

const child = spawn(process.execPath, args);
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
// instead of just displaying it all if the child fails.
let stdout = '';
let stderr = '';
let err;

child.on('error', (error) => {
err = error;
});

child.stdout.setEncoding('utf8');
child.stderr.setEncoding('utf8');

child.stdout.on('data', (chunk) => {
stdout += chunk;
});

child.stderr.on('data', (chunk) => {
stderr += chunk;
});

child.once('exit', (code, signal) => {
if (code !== 0 || signal !== null) {
if (!err) {
err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
err.exitCode = code;
err.signal = signal;
err.stdout = stdout;
err.stderr = stderr;
// The stack will not be useful since the failures came from tests
// in a child process.
err.stack = undefined;
}

return reject(err);
}

resolve();
});
});
});
}

(async function main() {
const testFiles = createTestFileList();

for (let i = 0; i < testFiles.length; i++) {
runTestFile(testFiles[i]);
}
})();
Loading

0 comments on commit 805361e

Please sign in to comment.