From fb29d8b33f0e844db28847fa14e4ff8f0f9dba62 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Tue, 16 Jul 2024 16:56:28 +0300 Subject: [PATCH 01/12] feat: add --run-failed option --- docs/cli.md | 4 ++ docs/config.md | 4 ++ src/cli/index.js | 3 ++ src/config/defaults.js | 1 + src/config/options.js | 2 + src/config/types.ts | 2 + src/testplane.ts | 64 +++++++++++++++++++++++++++++- test/src/cli/index.js | 6 +++ test/src/config/options.js | 27 +++++++++++++ test/src/testplane.js | 79 ++++++++++++++++++++++++++++++++++++-- test/utils.js | 2 + 11 files changed, 188 insertions(+), 6 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 19d176865..acd67154f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -41,6 +41,7 @@ shows the following --reporter test reporters --grep run only tests matching the pattern --update-refs update screenshot references or gather them if they do not exist ("assertView" command) + --run-failed only run tests that failed on the last run (tests from "failedTestsPath") --inspect [inspect] nodejs inspector on [=[host:]port] --inspect-brk [inspect-brk] nodejs inspector with break at the start --repl [type] run one test, call `browser.switchToRepl` in test code to open repl interface (default: false) @@ -224,6 +225,9 @@ Also, during the test development process, it may be necessary to execute comman Ability to run selected text in terminal will be available after this [issue](https://youtrack.jetbrains.com/issue/WEB-49916/Debug-JS-file-selection) will be resolved. +### Running only failed tests +Sometimes you may want to just re-run failed tests instead of running all. To do so, you can use the CLI option `--run-failed`. Testplane saves a list of all failed tests to `failedTestsPath` (default: `.testplane/failed-tests.json`). When you add `--run-failed` option, testplane reads all tests from `failedTestsPath` and disables all others, before running the tests. + ### Environment variables #### TESTPLANE_SKIP_BROWSERS diff --git a/docs/config.md b/docs/config.md index 7114edc10..4e14f15e8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -68,6 +68,7 @@ - [parallelLimit](#parallellimit) - [fileExtensions](#fileextensions) - [testRunEnv](#testrunenv) +- [failedTestsPath](#failedtestspath) - [plugins](#plugins) - [Parallel execution plugin code](#parallel-execution-plugin-code) - [List of useful plugins](#list-of-useful-plugins) @@ -593,6 +594,9 @@ export const { } ``` +### failedTestsPath +Testplane will save a list of all failed tests to `failedTestsPath` + ### plugins Testplane plugins are commonly used to extend built-in functionality. For example, [html-reporter](https://github.com/gemini-testing/html-reporter) and [@testplane/safari-commands](https://github.com/gemini-testing/testplane-safari-commands). diff --git a/src/cli/index.js b/src/cli/index.js index 47c6bac6e..b796797c4 100644 --- a/src/cli/index.js +++ b/src/cli/index.js @@ -58,6 +58,7 @@ exports.run = (opts = {}) => { "--update-refs", 'update screenshot references or gather them if they do not exist ("assertView" command)', ) + .option("--run-failed", "only run tests that failed during the last run") .option("--inspect [inspect]", "nodejs inspector on [=[host:]port]") .option("--inspect-brk [inspect-brk]", "nodejs inspector with break at the start") .option( @@ -78,6 +79,7 @@ exports.run = (opts = {}) => { set: sets, grep, updateRefs, + runFailed, require: requireModules, inspect, inspectBrk, @@ -95,6 +97,7 @@ exports.run = (opts = {}) => { sets, grep, updateRefs, + runFailed, requireModules, inspectMode: (inspect || inspectBrk) && { inspect, inspectBrk }, replMode: { diff --git a/src/config/defaults.js b/src/config/defaults.js index a046ab4a7..a78007aac 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -94,6 +94,7 @@ module.exports = { headless: null, isolation: null, testRunEnv: NODEJS_TEST_RUN_ENV, + failedTestsPath: ".testplane/failed-tests.json", devServer: { command: null, cwd: null, diff --git a/src/config/options.js b/src/config/options.js index 1c57df8c5..ca1b28b61 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -107,6 +107,8 @@ const rootSection = section( plugins: options.anyObject(), + failedTestsPath: options.string("failedTestsPath"), + sets: map( section({ files: option({ diff --git a/src/config/types.ts b/src/config/types.ts index fe91cce0b..f6b5cb2fd 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -136,6 +136,8 @@ export interface CommonConfig { resetCursor: boolean; headers: Record | null; + failedTestsPath: string; + system: SystemConfig; headless: "old" | "new" | boolean | null; isolation: boolean; diff --git a/src/testplane.ts b/src/testplane.ts index 2307f0ddc..00a896698 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -16,13 +16,16 @@ import logger from "./utils/logger"; import { isRunInNodeJsEnv } from "./utils/config"; import { initDevServer } from "./dev-server"; import { ConfigInput } from "./config/types"; -import { MasterEventHandler, Test } from "./types"; +import { MasterEventHandler, Test, TestResult } from "./types"; +import * as fs from "fs/promises"; +import path from "path"; interface RunOpts { browsers: string[]; sets: string[]; grep: RegExp; updateRefs: boolean; + runFailed: boolean; requireModules: string[]; inspectMode: { inspect: boolean; @@ -48,14 +51,21 @@ export interface Testplane { prependListener: MasterEventHandler; } +export type FailedListItem = { + browserId?: string; + fullTitle: string; +}; + export class Testplane extends BaseTestplane { protected failed: boolean; + protected failedList: FailedListItem[]; protected runner: NodejsEnvRunner | BrowserEnvRunner | null; constructor(config?: string | ConfigInput) { super(config); this.failed = false; + this.failedList = []; this.runner = null; } @@ -80,6 +90,7 @@ export class Testplane extends BaseTestplane { sets, grep, updateRefs, + runFailed, requireModules, inspectMode, replMode, @@ -101,7 +112,29 @@ export class Testplane extends BaseTestplane { ); this.runner = runner; - this.on(MasterEvents.TEST_FAIL, () => this._fail()).on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + this.on(MasterEvents.TEST_FAIL, () => this._fail()); + this.on(MasterEvents.TEST_FAIL, res => this._addFailedTest(res)); + this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + + if (runFailed) { + let previousRunFailedTests: FailedListItem[]; + this.on(MasterEvents.INIT, async () => { + previousRunFailedTests = await this._readFailed(); + }); + + this.on(MasterEvents.AFTER_TESTS_READ, testCollection => { + if (previousRunFailedTests.length) { + testCollection.disableAll(); + previousRunFailedTests.forEach(({ fullTitle, browserId }) => { + testCollection.enableTest(fullTitle, browserId); + }); + } + }); + } + + this.on(MasterEvents.RUNNER_END, async () => { + await this._saveFailed(); + }); await initReporters(reporters, this); @@ -119,6 +152,25 @@ export class Testplane extends BaseTestplane { return !this.isFailed(); } + protected async _readFailed(): Promise { + try { + const data = await fs.readFile(this._config.failedTestsPath, "utf8"); + return JSON.parse(data); + } catch { + return []; + } + } + + protected async _saveFailed(): Promise { + const dirname = path.dirname(this._config.failedTestsPath); + try { + await fs.access(dirname); + } catch { + await fs.mkdir(dirname, { recursive: true }); + } + await fs.writeFile(this._config.failedTestsPath, JSON.stringify(this.failedList)); + } + protected async _readTests( testPaths: string[] | TestCollection, opts: Partial, @@ -169,6 +221,14 @@ export class Testplane extends BaseTestplane { this.failed = true; } + protected _addFailedTest(result: TestResult): void { + this.failedList.push({ + fullTitle: result.fullTitle(), + browserId: result.browserId, + }); + this._saveFailed(); + } + isWorker(): boolean { return false; } diff --git a/test/src/cli/index.js b/test/src/cli/index.js index 97cdec943..ba1bd24d4 100644 --- a/test/src/cli/index.js +++ b/test/src/cli/index.js @@ -170,6 +170,12 @@ describe("cli", () => { assert.calledWithMatch(Testplane.prototype.run, any, { updateRefs: true }); }); + it("should use run failed mode from cli", async () => { + await run_("--run-failed"); + + assert.calledWithMatch(Testplane.prototype.run, any, { runFailed: true }); + }); + it("should use require modules from cli", async () => { const stubTestplaneCli = proxyquire("src/cli", { "../utils/module": { requireModule: sandbox.stub() }, diff --git a/test/src/config/options.js b/test/src/config/options.js index 4ab17b9d4..474f878d5 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -463,6 +463,33 @@ describe("config options", () => { }); }); + describe("failedTestsPath", () => { + it("should throw error if failedTestsPath is not a string", () => { + const readConfig = { failedTestsPath: () => {} }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"failedTestsPath" must be a string'); + }); + + it("should set default failedTestsPath option if it does not set in config file", () => { + const config = createConfig(); + + assert.equal(config.failedTestsPath, defaults.failedTestsPath); + }); + + it("should override failedTestsPath option", () => { + const newString = "newString"; + const readConfig = { failedTestsPath: newString }; + + Config.read.returns(readConfig); + + const config = createConfig(); + + assert.equal(config.failedTestsPath, newString); + }); + }); + describe("prepareEnvironment", () => { it("should throw error if prepareEnvironment is not a null or function", () => { const readConfig = { prepareEnvironment: "String" }; diff --git a/test/src/testplane.js b/test/src/testplane.js index f298e30a4..17f00990a 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -22,7 +22,7 @@ const { makeConfigStub } = require("../utils"); describe("testplane", () => { const sandbox = sinon.createSandbox(); - let Testplane, initReporters, signalHandler; + let Testplane, initReporters, signalHandler, fsRead, fsWrite, fsAccess, fsMkdir, disableAll, enableTest; const mkTestplane_ = config => { Config.create.returns(config || makeConfigStub()); @@ -52,10 +52,22 @@ describe("testplane", () => { initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); - - Testplane = proxyquire("src/testplane", { + fsRead = sandbox.stub().resolves("[]"); + fsWrite = sandbox.stub().resolves(); + fsAccess = sandbox.stub().resolves(); + fsMkdir = sandbox.stub().resolves(); + disableAll = sandbox.stub(); + enableTest = sandbox.stub(); + + Testplane = proxyquire.noPreserveCache().noCallThru()("src/testplane", { "./reporters": { initReporters }, "./signal-handler": signalHandler, + "fs/promises": { + readFile: fsRead, + writeFile: fsWrite, + access: fsAccess, + mkdir: fsMkdir, + }, }).Testplane; }); @@ -384,11 +396,67 @@ describe("testplane", () => { }); it('should return "false" if there are failed tests', () => { - mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL)); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + }; + mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL, results)); return runTestplane().then(success => assert.isFalse(success)); }); + it("should save failed tests", async () => { + const results = { + fullTitle: () => "Title", + browserId: "chrome", + }; + mkNodejsEnvRunner_(runner => { + runner.emit(RunnerEvents.TEST_FAIL, results), runner.emit(RunnerEvents.RUNNER_END); + }); + + await runTestplane(); + + assert.calledWith( + fsWrite, + "some-path", + JSON.stringify([ + { + fullTitle: results.fullTitle(), + browserId: results.browserId, + }, + ]), + ); + }); + + it("should run only failed with --run-failed", async () => { + const failed = [ + { + fullTitle: "Title", + browserId: "chrome", + }, + ]; + const testCollection = { + disableAll, + enableTest, + }; + + fsRead.resolves(JSON.stringify(failed)); + mkNodejsEnvRunner_(runner => { + runner.emit(RunnerEvents.INIT); + runner.emit(RunnerEvents.AFTER_TESTS_READ, testCollection); + }); + + sandbox.stub(Testplane.prototype, "_readTests"); + Testplane.prototype._readTests.returns([]); + + await runTestplane([], { + runFailed: true, + }); + + assert.called(disableAll); + assert.calledWith(enableTest, "Title", "chrome"); + }); + it("should halt if there were some errors", () => { const testplane = mkTestplane_(); const err = new Error(); @@ -467,6 +535,7 @@ describe("testplane", () => { it("all runner events with passed event data", () => { const runner = mkNodejsEnvRunner_(); const testplane = mkTestplane_(); + sandbox.stub(testplane, "_addFailedTest"); const omitEvents = ["EXIT", "NEW_BROWSER", "UPDATE_REFERENCE"]; return testplane.run().then(() => { @@ -761,6 +830,8 @@ describe("testplane", () => { it('should return "true" after some test fail', () => { const testplane = mkTestplane_(); + sandbox.stub(testplane, "_addFailedTest"); + mkNodejsEnvRunner_(runner => { runner.emit(RunnerEvents.TEST_FAIL); diff --git a/test/utils.js b/test/utils.js index 79748fba4..f3758ddfc 100644 --- a/test/utils.js +++ b/test/utils.js @@ -30,6 +30,7 @@ function makeConfigStub(opts = {}) { testRunEnv: NODEJS_TEST_RUN_ENV, }, sets: {}, + failedTestsPath: "some-path", }); const config = { @@ -38,6 +39,7 @@ function makeConfigStub(opts = {}) { system: opts.system, sets: opts.sets, configPath: opts.configPath, + failedTestsPath: opts.failedTestsPath, }; opts.browsers.forEach(browserId => { From 945378f3110e4e166d2fa20cb5c39509ba835b44 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Wed, 17 Jul 2024 13:48:40 +0300 Subject: [PATCH 02/12] refactor: refactor using fs-extra --- src/testplane.ts | 18 ++++++++---------- test/src/testplane.js | 36 +++++++++++++++--------------------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/testplane.ts b/src/testplane.ts index 00a896698..3ab8d1929 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -1,5 +1,7 @@ import { CommanderStatic } from "@gemini-testing/commander"; import _ from "lodash"; +import * as fs from "fs-extra"; +import path from "path"; import { Stats as RunnerStats } from "./stats"; import { BaseTestplane } from "./base-testplane"; import { MainRunner as NodejsEnvRunner } from "./runner"; @@ -17,8 +19,6 @@ import { isRunInNodeJsEnv } from "./utils/config"; import { initDevServer } from "./dev-server"; import { ConfigInput } from "./config/types"; import { MasterEventHandler, Test, TestResult } from "./types"; -import * as fs from "fs/promises"; -import path from "path"; interface RunOpts { browsers: string[]; @@ -154,21 +154,19 @@ export class Testplane extends BaseTestplane { protected async _readFailed(): Promise { try { - const data = await fs.readFile(this._config.failedTestsPath, "utf8"); - return JSON.parse(data); + const data = await fs.readJSON(this._config.failedTestsPath); + return data; } catch { + //No error because it may be convinient to always use --run-failed, even on the first run + logger.warn(`WARNING: Could not read ${this._config.failedTestsPath}`); return []; } } protected async _saveFailed(): Promise { const dirname = path.dirname(this._config.failedTestsPath); - try { - await fs.access(dirname); - } catch { - await fs.mkdir(dirname, { recursive: true }); - } - await fs.writeFile(this._config.failedTestsPath, JSON.stringify(this.failedList)); + await fs.ensureDir(dirname); + await fs.writeJSON(this._config.failedTestsPath, this.failedList, { spaces: 4 }); } protected async _readTests( diff --git a/test/src/testplane.js b/test/src/testplane.js index 17f00990a..2a29215bc 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -22,7 +22,7 @@ const { makeConfigStub } = require("../utils"); describe("testplane", () => { const sandbox = sinon.createSandbox(); - let Testplane, initReporters, signalHandler, fsRead, fsWrite, fsAccess, fsMkdir, disableAll, enableTest; + let Testplane, initReporters, signalHandler, fsReadJSON, fsWriteJSON, fsEnsureDir, disableAll, enableTest; const mkTestplane_ = config => { Config.create.returns(config || makeConfigStub()); @@ -52,21 +52,19 @@ describe("testplane", () => { initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); - fsRead = sandbox.stub().resolves("[]"); - fsWrite = sandbox.stub().resolves(); - fsAccess = sandbox.stub().resolves(); - fsMkdir = sandbox.stub().resolves(); + fsReadJSON = sandbox.stub().resolves([]); + fsWriteJSON = sandbox.stub().resolves(); + fsEnsureDir = sandbox.stub().resolves(); disableAll = sandbox.stub(); enableTest = sandbox.stub(); Testplane = proxyquire.noPreserveCache().noCallThru()("src/testplane", { "./reporters": { initReporters }, "./signal-handler": signalHandler, - "fs/promises": { - readFile: fsRead, - writeFile: fsWrite, - access: fsAccess, - mkdir: fsMkdir, + "fs-extra": { + readJSON: fsReadJSON, + writeJSON: fsWriteJSON, + ensureDir: fsEnsureDir, }, }).Testplane; }); @@ -416,16 +414,12 @@ describe("testplane", () => { await runTestplane(); - assert.calledWith( - fsWrite, - "some-path", - JSON.stringify([ - { - fullTitle: results.fullTitle(), - browserId: results.browserId, - }, - ]), - ); + assert.calledWith(fsWriteJSON, "some-path", [ + { + fullTitle: results.fullTitle(), + browserId: results.browserId, + }, + ]); }); it("should run only failed with --run-failed", async () => { @@ -440,7 +434,7 @@ describe("testplane", () => { enableTest, }; - fsRead.resolves(JSON.stringify(failed)); + fsReadJSON.resolves(failed); mkNodejsEnvRunner_(runner => { runner.emit(RunnerEvents.INIT); runner.emit(RunnerEvents.AFTER_TESTS_READ, testCollection); From 86a1d9ab2957b3ff4ebde57307587b9be4cc16c8 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Wed, 17 Jul 2024 14:19:38 +0300 Subject: [PATCH 03/12] refactor: use outputJSON from fs-extra --- src/testplane.ts | 16 +++++++--------- test/src/testplane.js | 10 ++++------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/testplane.ts b/src/testplane.ts index 3ab8d1929..7d5427aca 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -1,7 +1,6 @@ import { CommanderStatic } from "@gemini-testing/commander"; import _ from "lodash"; -import * as fs from "fs-extra"; -import path from "path"; +import fs from "fs-extra"; import { Stats as RunnerStats } from "./stats"; import { BaseTestplane } from "./base-testplane"; import { MainRunner as NodejsEnvRunner } from "./runner"; @@ -154,19 +153,18 @@ export class Testplane extends BaseTestplane { protected async _readFailed(): Promise { try { - const data = await fs.readJSON(this._config.failedTestsPath); - return data; + return await fs.readJSON(this._config.failedTestsPath); } catch { - //No error because it may be convinient to always use --run-failed, even on the first run - logger.warn(`WARNING: Could not read ${this._config.failedTestsPath}`); + // No error because it may be convinient to always use --run-failed, even on the first run + logger.warn( + `Could not read failed tests data at ${this._config.failedTestsPath}. Running all tests instead`, + ); return []; } } protected async _saveFailed(): Promise { - const dirname = path.dirname(this._config.failedTestsPath); - await fs.ensureDir(dirname); - await fs.writeJSON(this._config.failedTestsPath, this.failedList, { spaces: 4 }); + await fs.outputJSON(this._config.failedTestsPath, this.failedList, { spaces: 4 }); } protected async _readTests( diff --git a/test/src/testplane.js b/test/src/testplane.js index 2a29215bc..7a1cd4141 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -22,7 +22,7 @@ const { makeConfigStub } = require("../utils"); describe("testplane", () => { const sandbox = sinon.createSandbox(); - let Testplane, initReporters, signalHandler, fsReadJSON, fsWriteJSON, fsEnsureDir, disableAll, enableTest; + let Testplane, initReporters, signalHandler, fsReadJSON, fsOutputJSON, disableAll, enableTest; const mkTestplane_ = config => { Config.create.returns(config || makeConfigStub()); @@ -53,8 +53,7 @@ describe("testplane", () => { initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); fsReadJSON = sandbox.stub().resolves([]); - fsWriteJSON = sandbox.stub().resolves(); - fsEnsureDir = sandbox.stub().resolves(); + fsOutputJSON = sandbox.stub().resolves(); disableAll = sandbox.stub(); enableTest = sandbox.stub(); @@ -63,8 +62,7 @@ describe("testplane", () => { "./signal-handler": signalHandler, "fs-extra": { readJSON: fsReadJSON, - writeJSON: fsWriteJSON, - ensureDir: fsEnsureDir, + outputJSON: fsOutputJSON, }, }).Testplane; }); @@ -414,7 +412,7 @@ describe("testplane", () => { await runTestplane(); - assert.calledWith(fsWriteJSON, "some-path", [ + assert.calledWith(fsOutputJSON, "some-path", [ { fullTitle: results.fullTitle(), browserId: results.browserId, From a2b07daef9ccbac1c040f076425953c0c4bb0b03 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Thu, 18 Jul 2024 18:56:44 +0300 Subject: [PATCH 04/12] feat: implement lastFailed config option --- docs/cli.md | 4 -- docs/config.md | 14 +++++-- src/cli/index.js | 3 -- src/config/defaults.js | 6 ++- src/config/index.ts | 1 + src/config/options.js | 6 ++- src/config/types.ts | 6 ++- src/test-reader/test-parser.js | 24 ++++++++++++ src/testplane.ts | 38 ++++++------------ test/src/cli/index.js | 6 --- test/src/config/options.js | 60 ++++++++++++++++++++++------ test/src/test-reader/test-parser.js | 5 +++ test/src/testplane.js | 61 ++++++++--------------------- test/utils.js | 8 +++- 14 files changed, 138 insertions(+), 104 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index acd67154f..19d176865 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -41,7 +41,6 @@ shows the following --reporter test reporters --grep run only tests matching the pattern --update-refs update screenshot references or gather them if they do not exist ("assertView" command) - --run-failed only run tests that failed on the last run (tests from "failedTestsPath") --inspect [inspect] nodejs inspector on [=[host:]port] --inspect-brk [inspect-brk] nodejs inspector with break at the start --repl [type] run one test, call `browser.switchToRepl` in test code to open repl interface (default: false) @@ -225,9 +224,6 @@ Also, during the test development process, it may be necessary to execute comman Ability to run selected text in terminal will be available after this [issue](https://youtrack.jetbrains.com/issue/WEB-49916/Debug-JS-file-selection) will be resolved. -### Running only failed tests -Sometimes you may want to just re-run failed tests instead of running all. To do so, you can use the CLI option `--run-failed`. Testplane saves a list of all failed tests to `failedTestsPath` (default: `.testplane/failed-tests.json`). When you add `--run-failed` option, testplane reads all tests from `failedTestsPath` and disables all others, before running the tests. - ### Environment variables #### TESTPLANE_SKIP_BROWSERS diff --git a/docs/config.md b/docs/config.md index 4e14f15e8..38ecb9ef7 100644 --- a/docs/config.md +++ b/docs/config.md @@ -68,7 +68,7 @@ - [parallelLimit](#parallellimit) - [fileExtensions](#fileextensions) - [testRunEnv](#testrunenv) -- [failedTestsPath](#failedtestspath) +- [lastFailed](#lastfailed) - [plugins](#plugins) - [Parallel execution plugin code](#parallel-execution-plugin-code) - [List of useful plugins](#list-of-useful-plugins) @@ -594,8 +594,16 @@ export const { } ``` -### failedTestsPath -Testplane will save a list of all failed tests to `failedTestsPath` +### lastFailed +Allows you to run only tests that failed on the last run. + +```js +lastFailed: { + only: true, // true means run only failed, false - run all (default: false) + input: '.testplane/failed.json', // File to read failed tests list from (default: '.testplane/failed.json') + output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') +} +``` ### plugins Testplane plugins are commonly used to extend built-in functionality. For example, [html-reporter](https://github.com/gemini-testing/html-reporter) and [@testplane/safari-commands](https://github.com/gemini-testing/testplane-safari-commands). diff --git a/src/cli/index.js b/src/cli/index.js index b796797c4..47c6bac6e 100644 --- a/src/cli/index.js +++ b/src/cli/index.js @@ -58,7 +58,6 @@ exports.run = (opts = {}) => { "--update-refs", 'update screenshot references or gather them if they do not exist ("assertView" command)', ) - .option("--run-failed", "only run tests that failed during the last run") .option("--inspect [inspect]", "nodejs inspector on [=[host:]port]") .option("--inspect-brk [inspect-brk]", "nodejs inspector with break at the start") .option( @@ -79,7 +78,6 @@ exports.run = (opts = {}) => { set: sets, grep, updateRefs, - runFailed, require: requireModules, inspect, inspectBrk, @@ -97,7 +95,6 @@ exports.run = (opts = {}) => { sets, grep, updateRefs, - runFailed, requireModules, inspectMode: (inspect || inspectBrk) && { inspect, inspectBrk }, replMode: { diff --git a/src/config/defaults.js b/src/config/defaults.js index a78007aac..8e87ad725 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -94,7 +94,11 @@ module.exports = { headless: null, isolation: null, testRunEnv: NODEJS_TEST_RUN_ENV, - failedTestsPath: ".testplane/failed-tests.json", + lastFailed: { + only: false, + output: ".testplane/failed.json", + input: ".testplane/failed.json", + }, devServer: { command: null, cwd: null, diff --git a/src/config/index.ts b/src/config/index.ts index 2f7432ff3..3edd1fdcc 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -72,6 +72,7 @@ export class Config { const browserOptions = _.extend({}, browser, { id: id, system: this.system, + lastFailed: this.lastFailed, }); return new BrowserConfig(browserOptions); diff --git a/src/config/options.js b/src/config/options.js index ca1b28b61..d91cf7575 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -107,7 +107,11 @@ const rootSection = section( plugins: options.anyObject(), - failedTestsPath: options.string("failedTestsPath"), + lastFailed: section({ + only: options.boolean("lastFailed.only"), + input: options.string("lastFailed.input"), + output: options.string("lastFailed.output"), + }), sets: map( section({ diff --git a/src/config/types.ts b/src/config/types.ts index f6b5cb2fd..bddb24ee7 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -136,7 +136,11 @@ export interface CommonConfig { resetCursor: boolean; headers: Record | null; - failedTestsPath: string; + lastFailed: { + only: boolean; + input: string; + output: string; + }; system: SystemConfig; headless: "old" | "new" | boolean | null; diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 4401a8cd2..585fb4be8 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -14,9 +14,12 @@ const { MasterEvents } = require("../events"); const _ = require("lodash"); const clearRequire = require("clear-require"); const path = require("path"); +const fs = require("fs-extra"); +const logger = require("../utils/logger"); class TestParser extends EventEmitter { #opts; + #failedTests; #buildInstructions; /** @@ -27,6 +30,7 @@ class TestParser extends EventEmitter { super(); this.#opts = opts; + this.#failedTests = []; this.#buildInstructions = new InstructionsList(); } @@ -66,6 +70,14 @@ class TestParser extends EventEmitter { const esmDecorator = f => f + `?rand=${rand}`; await readFiles(files, { esmDecorator, config: mochaOpts, eventBus }); + try { + this.#failedTests = await fs.readJSON(config.lastFailed.input); + } catch { + logger.warn( + `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, + ); + } + revertTransformHook(); } @@ -113,6 +125,18 @@ class TestParser extends EventEmitter { treeBuilder.addTestFilter(test => grep.test(test.fullTitle())); } + if (config.lastFailed && config.lastFailed.only) { + const failedStringified = this.#failedTests.map(data => JSON.stringify(data)); + treeBuilder.addTestFilter(test => + failedStringified.includes( + JSON.stringify({ + fullTitle: test.fullTitle(), + browserId: test.browserId, + }), + ), + ); + } + const rootSuite = treeBuilder.applyFilters().getRootSuite(); const tests = rootSuite.getTests(); diff --git a/src/testplane.ts b/src/testplane.ts index 7d5427aca..7215a1d98 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -24,7 +24,6 @@ interface RunOpts { sets: string[]; grep: RegExp; updateRefs: boolean; - runFailed: boolean; requireModules: string[]; inspectMode: { inspect: boolean; @@ -39,9 +38,15 @@ interface RunOpts { devtools: boolean; } +export type FailedListItem = { + browserId?: string; + fullTitle: string; +}; + interface ReadTestsOpts extends Pick { silent: boolean; ignore: string | string[]; + failed: FailedListItem[]; } export interface Testplane { @@ -50,11 +55,6 @@ export interface Testplane { prependListener: MasterEventHandler; } -export type FailedListItem = { - browserId?: string; - fullTitle: string; -}; - export class Testplane extends BaseTestplane { protected failed: boolean; protected failedList: FailedListItem[]; @@ -89,7 +89,6 @@ export class Testplane extends BaseTestplane { sets, grep, updateRefs, - runFailed, requireModules, inspectMode, replMode, @@ -115,22 +114,6 @@ export class Testplane extends BaseTestplane { this.on(MasterEvents.TEST_FAIL, res => this._addFailedTest(res)); this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); - if (runFailed) { - let previousRunFailedTests: FailedListItem[]; - this.on(MasterEvents.INIT, async () => { - previousRunFailedTests = await this._readFailed(); - }); - - this.on(MasterEvents.AFTER_TESTS_READ, testCollection => { - if (previousRunFailedTests.length) { - testCollection.disableAll(); - previousRunFailedTests.forEach(({ fullTitle, browserId }) => { - testCollection.enableTest(fullTitle, browserId); - }); - } - }); - } - this.on(MasterEvents.RUNNER_END, async () => { await this._saveFailed(); }); @@ -153,18 +136,19 @@ export class Testplane extends BaseTestplane { protected async _readFailed(): Promise { try { - return await fs.readJSON(this._config.failedTestsPath); + logger.log("!!!"); + return await fs.readJSON(this._config.lastFailed.input); } catch { - // No error because it may be convinient to always use --run-failed, even on the first run + // No error because it may be convinient to always use --last-failed-only, even on the first run logger.warn( - `Could not read failed tests data at ${this._config.failedTestsPath}. Running all tests instead`, + `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, ); return []; } } protected async _saveFailed(): Promise { - await fs.outputJSON(this._config.failedTestsPath, this.failedList, { spaces: 4 }); + await fs.outputJSON(this._config.lastFailed.output, this.failedList, { spaces: 4 }); } protected async _readTests( diff --git a/test/src/cli/index.js b/test/src/cli/index.js index ba1bd24d4..97cdec943 100644 --- a/test/src/cli/index.js +++ b/test/src/cli/index.js @@ -170,12 +170,6 @@ describe("cli", () => { assert.calledWithMatch(Testplane.prototype.run, any, { updateRefs: true }); }); - it("should use run failed mode from cli", async () => { - await run_("--run-failed"); - - assert.calledWithMatch(Testplane.prototype.run, any, { runFailed: true }); - }); - it("should use require modules from cli", async () => { const stubTestplaneCli = proxyquire("src/cli", { "../utils/module": { requireModule: sandbox.stub() }, diff --git a/test/src/config/options.js b/test/src/config/options.js index 474f878d5..e0199b876 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -463,30 +463,68 @@ describe("config options", () => { }); }); - describe("failedTestsPath", () => { - it("should throw error if failedTestsPath is not a string", () => { - const readConfig = { failedTestsPath: () => {} }; + describe("lastFailed", () => { + describe("only", () => { + it("should throw error if only is not a boolean", () => { + const readConfig = { + lastFailed: { + only: "String", + }, + }; - Config.read.returns(readConfig); + Config.read.returns(readConfig); - assert.throws(() => createConfig(), Error, '"failedTestsPath" must be a string'); + assert.throws(() => createConfig(), Error, '"lastFailed.only" must be a boolean'); + }); + }); + + describe("input", () => { + it("should throw error if input is not a string", () => { + const readConfig = { + lastFailed: { + input: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string'); + }); + }); + + describe("output", () => { + it("should throw error if output is not a string", () => { + const readConfig = { + lastFailed: { + output: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must be a string'); + }); }); - it("should set default failedTestsPath option if it does not set in config file", () => { + it("should set default lastFailed option if it does not set in config file", () => { const config = createConfig(); - assert.equal(config.failedTestsPath, defaults.failedTestsPath); + assert.deepEqual(config.lastFailed, defaults.lastFailed); }); - it("should override failedTestsPath option", () => { - const newString = "newString"; - const readConfig = { failedTestsPath: newString }; + it("should override lastFailed option", () => { + const newValue = { + input: "some-path", + output: "some-path", + only: true, + }; + const readConfig = { lastFailed: newValue }; Config.read.returns(readConfig); const config = createConfig(); - assert.equal(config.failedTestsPath, newString); + assert.deepEqual(config.lastFailed, newValue); }); }); diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index eb3f7d6b0..a26ff9ea4 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -21,6 +21,7 @@ describe("test-reader/test-parser", () => { const sandbox = sinon.createSandbox(); let TestParser; + let fsReadJSON; let clearRequire; let readFiles; let setupTransformHook; @@ -32,12 +33,16 @@ describe("test-reader/test-parser", () => { clearRequire = sandbox.stub().named("clear-require"); readFiles = sandbox.stub().named("readFiles").resolves(); setupTransformHook = sandbox.stub().named("setupTransformHook").returns(sinon.stub()); + fsReadJSON = sandbox.stub().resolves([]); TestParser = proxyquire("src/test-reader/test-parser", { "clear-require": clearRequire, "./mocha-reader": { readFiles }, "./controllers/browser-version-controller": browserVersionController, "./test-transformer": { setupTransformHook }, + "fs-extra": { + readJSON: fsReadJSON, + }, }).TestParser; sandbox.stub(InstructionsList.prototype, "push").returnsThis(); diff --git a/test/src/testplane.js b/test/src/testplane.js index 7a1cd4141..6b51b0a99 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -22,7 +22,7 @@ const { makeConfigStub } = require("../utils"); describe("testplane", () => { const sandbox = sinon.createSandbox(); - let Testplane, initReporters, signalHandler, fsReadJSON, fsOutputJSON, disableAll, enableTest; + let Testplane, initReporters, signalHandler, fsReadJSON, fsOutputJSON; const mkTestplane_ = config => { Config.create.returns(config || makeConfigStub()); @@ -54,8 +54,6 @@ describe("testplane", () => { signalHandler = new AsyncEmitter(); fsReadJSON = sandbox.stub().resolves([]); fsOutputJSON = sandbox.stub().resolves(); - disableAll = sandbox.stub(); - enableTest = sandbox.stub(); Testplane = proxyquire.noPreserveCache().noCallThru()("src/testplane", { "./reporters": { initReporters }, @@ -225,7 +223,10 @@ describe("testplane", () => { describe("repl mode", () => { it("should not reset test timeout to 0 if run not in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: false } }); @@ -234,7 +235,10 @@ describe("testplane", () => { it("should reset test timeout to 0 if run in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: true } }); @@ -326,7 +330,12 @@ describe("testplane", () => { await runTestplane(testPaths, { browsers, grep, sets, replMode }); - assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { browsers, grep, sets, replMode }); + assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { + browsers, + grep, + sets, + replMode, + }); }); it("should accept test collection as first parameter", async () => { @@ -412,51 +421,13 @@ describe("testplane", () => { await runTestplane(); - assert.calledWith(fsOutputJSON, "some-path", [ + assert.calledWith(fsOutputJSON, "some-other-path", [ { fullTitle: results.fullTitle(), browserId: results.browserId, }, ]); }); - - it("should run only failed with --run-failed", async () => { - const failed = [ - { - fullTitle: "Title", - browserId: "chrome", - }, - ]; - const testCollection = { - disableAll, - enableTest, - }; - - fsReadJSON.resolves(failed); - mkNodejsEnvRunner_(runner => { - runner.emit(RunnerEvents.INIT); - runner.emit(RunnerEvents.AFTER_TESTS_READ, testCollection); - }); - - sandbox.stub(Testplane.prototype, "_readTests"); - Testplane.prototype._readTests.returns([]); - - await runTestplane([], { - runFailed: true, - }); - - assert.called(disableAll); - assert.calledWith(enableTest, "Title", "chrome"); - }); - - it("should halt if there were some errors", () => { - const testplane = mkTestplane_(); - const err = new Error(); - - mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.ERROR, err)); - - return testplane.run().then(() => assert.calledOnceWith(testplane.halt, err)); - }); }); describe("should passthrough", () => { diff --git a/test/utils.js b/test/utils.js index f3758ddfc..581469f2f 100644 --- a/test/utils.js +++ b/test/utils.js @@ -30,7 +30,11 @@ function makeConfigStub(opts = {}) { testRunEnv: NODEJS_TEST_RUN_ENV, }, sets: {}, - failedTestsPath: "some-path", + lastFailed: { + only: false, + input: "some-path", + output: "some-other-path", + }, }); const config = { @@ -39,7 +43,7 @@ function makeConfigStub(opts = {}) { system: opts.system, sets: opts.sets, configPath: opts.configPath, - failedTestsPath: opts.failedTestsPath, + lastFailed: opts.lastFailed, }; opts.browsers.forEach(browserId => { From d45a4d046d6207440546e3695e3e7f6a52887549 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Fri, 19 Jul 2024 10:28:39 +0300 Subject: [PATCH 05/12] refactor: remove unused code --- src/testplane.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/testplane.ts b/src/testplane.ts index 7215a1d98..083995b2b 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -134,19 +134,6 @@ export class Testplane extends BaseTestplane { return !this.isFailed(); } - protected async _readFailed(): Promise { - try { - logger.log("!!!"); - return await fs.readJSON(this._config.lastFailed.input); - } catch { - // No error because it may be convinient to always use --last-failed-only, even on the first run - logger.warn( - `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, - ); - return []; - } - } - protected async _saveFailed(): Promise { await fs.outputJSON(this._config.lastFailed.output, this.failedList, { spaces: 4 }); } From bd9176df16000b80c5337bd6d361eeb197387c87 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Fri, 19 Jul 2024 18:17:59 +0300 Subject: [PATCH 06/12] refactor: refactor code --- docs/config.md | 24 +++++++++++----------- src/config/options.js | 24 ++++++++++++++++++++-- src/config/types.ts | 8 ++++---- src/test-reader/test-parser.js | 14 +++++++------ src/testplane.ts | 9 +++++---- test/src/config/options.js | 4 ++-- test/src/test-reader/test-parser.js | 7 ++----- test/src/testplane.js | 31 ++++++++++++++++------------- 8 files changed, 72 insertions(+), 49 deletions(-) diff --git a/docs/config.md b/docs/config.md index 38ecb9ef7..b029ed9a1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -68,12 +68,12 @@ - [parallelLimit](#parallellimit) - [fileExtensions](#fileextensions) - [testRunEnv](#testrunenv) -- [lastFailed](#lastfailed) - [plugins](#plugins) - [Parallel execution plugin code](#parallel-execution-plugin-code) - [List of useful plugins](#list-of-useful-plugins) - [prepareBrowser](#preparebrowser) - [prepareEnvironment](#prepareenvironment) +- [lastFailed](#lastfailed) @@ -594,17 +594,6 @@ export const { } ``` -### lastFailed -Allows you to run only tests that failed on the last run. - -```js -lastFailed: { - only: true, // true means run only failed, false - run all (default: false) - input: '.testplane/failed.json', // File to read failed tests list from (default: '.testplane/failed.json') - output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') -} -``` - ### plugins Testplane plugins are commonly used to extend built-in functionality. For example, [html-reporter](https://github.com/gemini-testing/html-reporter) and [@testplane/safari-commands](https://github.com/gemini-testing/testplane-safari-commands). @@ -743,3 +732,14 @@ Full list of parameters: - waitServerTimeout (optional) `Number` - timeout to wait for server to be ready (ms). 60_000 by default - probeRequestTimeout (optional) `Number` - one request timeout (ms), after which request will be aborted. 10_000 by default - probeRequestInterval (optional) `Number` - interval between ready probe requests (ms). 1_000 by default + +### lastFailed +Allows you to run only tests that failed on the last run. Disabled by default - it means run all tests, but the file with the failed tests is always written. + +```js +lastFailed: { + only: true, // true means run only failed, false - run all (default: false) + input: '.testplane/failed.json', // File to read failed tests list from (default: '.testplane/failed.json') + output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') +} +``` \ No newline at end of file diff --git a/src/config/options.js b/src/config/options.js index d91cf7575..a3ec98c6e 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -109,8 +109,28 @@ const rootSection = section( lastFailed: section({ only: options.boolean("lastFailed.only"), - input: options.string("lastFailed.input"), - output: options.string("lastFailed.output"), + input: option({ + defaultValue: defaults.lastFailed.input, + validate: value => { + if (!_.isString(value)) { + throw new Error('"lastFailed.input" must be a string'); + } + if (!value.endsWith(".json")) { + throw new Error('"lastFailed.input" must have .json extension'); + } + }, + }), + output: option({ + defaultValue: defaults.lastFailed.output, + validate: value => { + if (!_.isString(value)) { + throw new Error('"lastFailed.output" must be a string'); + } + if (!value.endsWith(".json")) { + throw new Error('"lastFailed.output" must have .json extension'); + } + }, + }), }), sets: map( diff --git a/src/config/types.ts b/src/config/types.ts index bddb24ee7..5fb991dc8 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -136,16 +136,16 @@ export interface CommonConfig { resetCursor: boolean; headers: Record | null; + system: SystemConfig; + headless: "old" | "new" | boolean | null; + isolation: boolean; + lastFailed: { only: boolean; input: string; output: string; }; - system: SystemConfig; - headless: "old" | "new" | boolean | null; - isolation: boolean; - openAndWaitOpts: { timeout?: number; waitNetworkIdle: boolean; diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 585fb4be8..104dc5fdb 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -70,12 +70,14 @@ class TestParser extends EventEmitter { const esmDecorator = f => f + `?rand=${rand}`; await readFiles(files, { esmDecorator, config: mochaOpts, eventBus }); - try { - this.#failedTests = await fs.readJSON(config.lastFailed.input); - } catch { - logger.warn( - `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, - ); + if (config.lastFailed.only) { + try { + this.#failedTests = await fs.readJSON(config.lastFailed.input); + } catch { + logger.warn( + `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, + ); + } } revertTransformHook(); diff --git a/src/testplane.ts b/src/testplane.ts index 083995b2b..1c1ae625e 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -110,8 +110,10 @@ export class Testplane extends BaseTestplane { ); this.runner = runner; - this.on(MasterEvents.TEST_FAIL, () => this._fail()); - this.on(MasterEvents.TEST_FAIL, res => this._addFailedTest(res)); + this.on(MasterEvents.TEST_FAIL, res => { + this._fail(); + this._addFailedTest(res); + }); this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); this.on(MasterEvents.RUNNER_END, async () => { @@ -135,7 +137,7 @@ export class Testplane extends BaseTestplane { } protected async _saveFailed(): Promise { - await fs.outputJSON(this._config.lastFailed.output, this.failedList, { spaces: 4 }); + await fs.outputJSON(this._config.lastFailed.output, this.failedList); // No spaces because users usually don't need to read it } protected async _readTests( @@ -193,7 +195,6 @@ export class Testplane extends BaseTestplane { fullTitle: result.fullTitle(), browserId: result.browserId, }); - this._saveFailed(); } isWorker(): boolean { diff --git a/test/src/config/options.js b/test/src/config/options.js index e0199b876..4f375e3fb 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -514,8 +514,8 @@ describe("config options", () => { it("should override lastFailed option", () => { const newValue = { - input: "some-path", - output: "some-path", + input: "some-path.json", + output: "some-path.json", only: true, }; const readConfig = { lastFailed: newValue }; diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index a26ff9ea4..4810a8fa5 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -14,6 +14,7 @@ const proxyquire = require("proxyquire").noCallThru(); const path = require("path"); const { EventEmitter } = require("events"); const _ = require("lodash"); +const fs = require("fs-extra"); const { NEW_BUILD_INSTRUCTION } = TestReaderEvents; @@ -21,7 +22,6 @@ describe("test-reader/test-parser", () => { const sandbox = sinon.createSandbox(); let TestParser; - let fsReadJSON; let clearRequire; let readFiles; let setupTransformHook; @@ -33,18 +33,15 @@ describe("test-reader/test-parser", () => { clearRequire = sandbox.stub().named("clear-require"); readFiles = sandbox.stub().named("readFiles").resolves(); setupTransformHook = sandbox.stub().named("setupTransformHook").returns(sinon.stub()); - fsReadJSON = sandbox.stub().resolves([]); TestParser = proxyquire("src/test-reader/test-parser", { "clear-require": clearRequire, "./mocha-reader": { readFiles }, "./controllers/browser-version-controller": browserVersionController, "./test-transformer": { setupTransformHook }, - "fs-extra": { - readJSON: fsReadJSON, - }, }).TestParser; + sandbox.stub(fs, "readJSON").resolves([]); sandbox.stub(InstructionsList.prototype, "push").returnsThis(); sandbox.stub(InstructionsList.prototype, "exec").returns(new Suite()); }); diff --git a/test/src/testplane.js b/test/src/testplane.js index 6b51b0a99..c7f0a8eca 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -1,6 +1,7 @@ "use strict"; const _ = require("lodash"); +const fs = require("fs-extra"); const { EventEmitter } = require("events"); const pluginsLoader = require("plugins-loader"); const Promise = require("bluebird"); @@ -22,7 +23,7 @@ const { makeConfigStub } = require("../utils"); describe("testplane", () => { const sandbox = sinon.createSandbox(); - let Testplane, initReporters, signalHandler, fsReadJSON, fsOutputJSON; + let Testplane, initReporters, signalHandler; const mkTestplane_ = config => { Config.create.returns(config || makeConfigStub()); @@ -49,19 +50,15 @@ describe("testplane", () => { sandbox.stub(RuntimeConfig, "getInstance").returns({ extend: sandbox.stub() }); sandbox.stub(TestReader.prototype, "read").resolves(); sandbox.stub(RunnerStats, "create"); + sandbox.stub(fs, "readJSON").resolves([]); + sandbox.stub(fs, "outputJSON").resolves(); initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); - fsReadJSON = sandbox.stub().resolves([]); - fsOutputJSON = sandbox.stub().resolves(); - Testplane = proxyquire.noPreserveCache().noCallThru()("src/testplane", { + Testplane = proxyquire("src/testplane", { "./reporters": { initReporters }, "./signal-handler": signalHandler, - "fs-extra": { - readJSON: fsReadJSON, - outputJSON: fsOutputJSON, - }, }).Testplane; }); @@ -421,7 +418,7 @@ describe("testplane", () => { await runTestplane(); - assert.calledWith(fsOutputJSON, "some-other-path", [ + assert.calledWith(fs.outputJSON, "some-other-path", [ { fullTitle: results.fullTitle(), browserId: results.browserId, @@ -498,7 +495,10 @@ describe("testplane", () => { it("all runner events with passed event data", () => { const runner = mkNodejsEnvRunner_(); const testplane = mkTestplane_(); - sandbox.stub(testplane, "_addFailedTest"); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + }; const omitEvents = ["EXIT", "NEW_BROWSER", "UPDATE_REFERENCE"]; return testplane.run().then(() => { @@ -506,9 +506,9 @@ describe("testplane", () => { const spy = sinon.spy().named(`${name} handler`); testplane.on(event, spy); - runner.emit(event, "some-data"); + runner.emit(event, results); - assert.calledWith(spy, "some-data"); + assert.calledWith(spy, results); }); }); }); @@ -793,10 +793,13 @@ describe("testplane", () => { it('should return "true" after some test fail', () => { const testplane = mkTestplane_(); - sandbox.stub(testplane, "_addFailedTest"); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + }; mkNodejsEnvRunner_(runner => { - runner.emit(RunnerEvents.TEST_FAIL); + runner.emit(RunnerEvents.TEST_FAIL, results); assert.isTrue(testplane.isFailed()); }); From ac41831cd7030e16b7ab1bc7d457a9c1de00f2ea Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Mon, 22 Jul 2024 16:07:58 +0300 Subject: [PATCH 07/12] refactor: add browser version --- src/test-reader/test-parser.js | 21 ++++++++++----------- src/testplane.ts | 6 +++--- test/src/testplane.js | 2 ++ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 104dc5fdb..d060f8ea3 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -16,6 +16,7 @@ const clearRequire = require("clear-require"); const path = require("path"); const fs = require("fs-extra"); const logger = require("../utils/logger"); +const { getShortMD5 } = require("../utils/crypto"); class TestParser extends EventEmitter { #opts; @@ -30,7 +31,7 @@ class TestParser extends EventEmitter { super(); this.#opts = opts; - this.#failedTests = []; + this.#failedTests = new Set(); this.#buildInstructions = new InstructionsList(); } @@ -72,7 +73,11 @@ class TestParser extends EventEmitter { if (config.lastFailed.only) { try { - this.#failedTests = await fs.readJSON(config.lastFailed.input); + this.#failedTests = new Set(); + + for (const test of await fs.readJSON(config.lastFailed.input)) { + this.#failedTests.add(getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`)); + } } catch { logger.warn( `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, @@ -128,15 +133,9 @@ class TestParser extends EventEmitter { } if (config.lastFailed && config.lastFailed.only) { - const failedStringified = this.#failedTests.map(data => JSON.stringify(data)); - treeBuilder.addTestFilter(test => - failedStringified.includes( - JSON.stringify({ - fullTitle: test.fullTitle(), - browserId: test.browserId, - }), - ), - ); + treeBuilder.addTestFilter(test => { + return this.#failedTests.has(getShortMD5(`${test.fullTitle()}${test.browserId}${test.browserVersion}`)); + }); } const rootSuite = treeBuilder.applyFilters().getRootSuite(); diff --git a/src/testplane.ts b/src/testplane.ts index 1c1ae625e..7fc00cf5a 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -39,6 +39,7 @@ interface RunOpts { } export type FailedListItem = { + browserVersion?: string; browserId?: string; fullTitle: string; }; @@ -116,9 +117,7 @@ export class Testplane extends BaseTestplane { }); this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); - this.on(MasterEvents.RUNNER_END, async () => { - await this._saveFailed(); - }); + this.on(MasterEvents.RUNNER_END, async () => await this._saveFailed()); await initReporters(reporters, this); @@ -194,6 +193,7 @@ export class Testplane extends BaseTestplane { this.failedList.push({ fullTitle: result.fullTitle(), browserId: result.browserId, + browserVersion: result.browserVersion, }); } diff --git a/test/src/testplane.js b/test/src/testplane.js index c7f0a8eca..3c46df02e 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -411,6 +411,7 @@ describe("testplane", () => { const results = { fullTitle: () => "Title", browserId: "chrome", + browserVersion: "1", }; mkNodejsEnvRunner_(runner => { runner.emit(RunnerEvents.TEST_FAIL, results), runner.emit(RunnerEvents.RUNNER_END); @@ -422,6 +423,7 @@ describe("testplane", () => { { fullTitle: results.fullTitle(), browserId: results.browserId, + browserVersion: "1", }, ]); }); From c6ee9010a2ff4489bd8c6072fb3fa192b12d93ee Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Tue, 23 Jul 2024 11:54:28 +0300 Subject: [PATCH 08/12] feat: allow arrays in lastFailed input --- docs/config.md | 2 +- src/config/options.js | 9 ++++++--- src/config/types.ts | 2 +- src/test-reader/test-parser.js | 14 +++++++++----- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/config.md b/docs/config.md index b029ed9a1..666ab5aec 100644 --- a/docs/config.md +++ b/docs/config.md @@ -739,7 +739,7 @@ Allows you to run only tests that failed on the last run. Disabled by default - ```js lastFailed: { only: true, // true means run only failed, false - run all (default: false) - input: '.testplane/failed.json', // File to read failed tests list from (default: '.testplane/failed.json') + input: ['.testplane/failed.json', '.testplane/failed2.json'], // File/files to read failed tests list from (default: '.testplane/failed.json') output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') } ``` \ No newline at end of file diff --git a/src/config/options.js b/src/config/options.js index a3ec98c6e..1db415ee9 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -112,12 +112,15 @@ const rootSection = section( input: option({ defaultValue: defaults.lastFailed.input, validate: value => { - if (!_.isString(value)) { - throw new Error('"lastFailed.input" must be a string'); + if (!_.isString(value) && !_.isArray(value)) { + throw new Error('"lastFailed.input" must be a string or an array'); } - if (!value.endsWith(".json")) { + if (_.isString(value) && !value.endsWith(".json")) { throw new Error('"lastFailed.input" must have .json extension'); } + if (_.isArray(value) && value.filter(v => v.endsWith(".json"))) { + throw new Error('"lastFailed.input" entities must have .json extension'); + } }, }), output: option({ diff --git a/src/config/types.ts b/src/config/types.ts index 5fb991dc8..547affe87 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -142,7 +142,7 @@ export interface CommonConfig { lastFailed: { only: boolean; - input: string; + input: string | Array; output: string; }; diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index d060f8ea3..d0eda860e 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -74,13 +74,17 @@ class TestParser extends EventEmitter { if (config.lastFailed.only) { try { this.#failedTests = new Set(); - - for (const test of await fs.readJSON(config.lastFailed.input)) { - this.#failedTests.add(getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`)); + const inputPaths = _.isArray(config.lastFailed.input) + ? config.lastFailed.input + : [config.lastFailed.input]; + for (const inputPath of inputPaths) { + for (const test of await fs.readJSON(inputPath)) { + this.#failedTests.add(getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`)); + } } } catch { logger.warn( - `Could not read failed tests data at ${this._config.lastFailed.input}. Running all tests instead`, + `Could not read failed tests data at ${config.lastFailed.input}. Running all tests instead`, ); } } @@ -132,7 +136,7 @@ class TestParser extends EventEmitter { treeBuilder.addTestFilter(test => grep.test(test.fullTitle())); } - if (config.lastFailed && config.lastFailed.only) { + if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { treeBuilder.addTestFilter(test => { return this.#failedTests.has(getShortMD5(`${test.fullTitle()}${test.browserId}${test.browserVersion}`)); }); From fb8c50d27d62176abe66b90ad41f49cc42cf9094 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Tue, 23 Jul 2024 18:16:07 +0300 Subject: [PATCH 09/12] fix: fix bugs --- src/config/options.js | 6 +++--- src/test-reader/test-parser.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config/options.js b/src/config/options.js index 1db415ee9..201120b33 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -115,11 +115,11 @@ const rootSection = section( if (!_.isString(value) && !_.isArray(value)) { throw new Error('"lastFailed.input" must be a string or an array'); } - if (_.isString(value) && !value.endsWith(".json")) { + if (!_.isArray(value) && !value.endsWith(".json")) { throw new Error('"lastFailed.input" must have .json extension'); } - if (_.isArray(value) && value.filter(v => v.endsWith(".json"))) { - throw new Error('"lastFailed.input" entities must have .json extension'); + if (_.isArray(value) && value.filter(v => !v.endsWith(".json")).length) { + throw new Error('"lastFailed.input" elements must have .json extension'); } }, }), diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index d0eda860e..764e899b9 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -76,7 +76,7 @@ class TestParser extends EventEmitter { this.#failedTests = new Set(); const inputPaths = _.isArray(config.lastFailed.input) ? config.lastFailed.input - : [config.lastFailed.input]; + : config.lastFailed.input.split(",").map(v => v.trim()); for (const inputPath of inputPaths) { for (const test of await fs.readJSON(inputPath)) { this.#failedTests.add(getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`)); From f15ac1709a4d3bcc6f04a080dc731da951dc9755 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Thu, 25 Jul 2024 14:52:04 +0300 Subject: [PATCH 10/12] test: add tests --- src/test-reader/test-parser.js | 8 ++- test/src/config/options.js | 74 +++++++++++++++++++++- test/src/test-reader/test-parser.js | 96 ++++++++++++++++++++++++++++- test/src/testplane.js | 12 ++++ 4 files changed, 184 insertions(+), 6 deletions(-) diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 764e899b9..23319c20a 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -35,6 +35,8 @@ class TestParser extends EventEmitter { this.#buildInstructions = new InstructionsList(); } + getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); + async loadFiles(files, config) { const eventBus = new EventEmitter(); const { @@ -79,7 +81,7 @@ class TestParser extends EventEmitter { : config.lastFailed.input.split(",").map(v => v.trim()); for (const inputPath of inputPaths) { for (const test of await fs.readJSON(inputPath)) { - this.#failedTests.add(getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`)); + this.#failedTests.add(this.getFailedTestId(test)); } } } catch { @@ -137,8 +139,8 @@ class TestParser extends EventEmitter { } if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { - treeBuilder.addTestFilter(test => { - return this.#failedTests.has(getShortMD5(`${test.fullTitle()}${test.browserId}${test.browserVersion}`)); + treeBuilder.addTestFilter(({ fullTitle, ...rest }) => { + return this.#failedTests.has(this.getFailedTestId({ fullTitle: fullTitle(), ...rest })); }); } diff --git a/test/src/config/options.js b/test/src/config/options.js index 4f375e3fb..360ba8aea 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -488,7 +488,55 @@ describe("config options", () => { Config.read.returns(readConfig); - assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string'); + assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string or an array'); + }); + + it("should throw error if input is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must have .json extension'); + }); + + it("should not throw error if input is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + + it("should throw error if input is an array that contains a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json", "string"], + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" elements must have .json extension'); + }); + + it("should not throw error if input is an array that contains only strings with .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json"], + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); }); }); @@ -504,6 +552,30 @@ describe("config options", () => { assert.throws(() => createConfig(), Error, '"lastFailed.output" must be a string'); }); + + it("should throw error if output is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must have .json extension'); + }); + + it("should not throw error if output is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); }); it("should set default lastFailed option if it does not set in config file", () => { diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index 4810a8fa5..14b9e0426 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -322,6 +322,68 @@ describe("test-reader/test-parser", () => { }); }); + describe("failed tests", () => { + it("should read if config.lastFailed.only is set", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "file.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "file.json"); + }); + + it("should read from one file if config.lastFailed.input is a string", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + }); + + it("should read from multiple files if config.lastFailed.input is a string with commas", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json, failed2.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should read from multiple files if config.lastFailed.input is an array", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: ["failed.json", "failed2.json"], + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should not read if config.lastFailed.only is not set", async () => { + await loadFiles_(); + + assert.notCalled(fs.readJSON); + }); + }); + describe("read files", () => { it("should read passed files", async () => { const files = ["foo/bar", "baz/qux"]; @@ -477,13 +539,14 @@ describe("test-reader/test-parser", () => { }); describe("parse", () => { - const parse_ = async ({ files, browserId, config, grep } = {}) => { + const parse_ = async ({ files, browserId, config, grep } = {}, loadFilesConfig) => { + loadFilesConfig = loadFilesConfig || makeConfigStub(); config = _.defaults(config, { desiredCapabilities: {}, }); const parser = new TestParser(); - await parser.loadFiles([], makeConfigStub()); + await parser.loadFiles([], loadFilesConfig); return parser.parse(files || [], { browserId, config, grep }); }; @@ -496,6 +559,35 @@ describe("test-reader/test-parser", () => { sandbox.stub(Suite.prototype, "getTests").returns([]); }); + describe("addTestFilter", () => { + it("should not call if config.lastFailed.only is not set", async () => { + await parse_(); + + assert.notCalled(TreeBuilder.prototype.addTestFilter); + }); + + it("should call addTestFilter if config.lastFailed.only is set", async () => { + fs.readJSON.resolves([ + { + fullTitle: "title", + browserId: "chrome", + browserVersion: "1", + }, + ]); + + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await parse_({ config }, config); + + assert.calledOnce(TreeBuilder.prototype.addTestFilter); + }); + }); + it("should execute build instructions", async () => { await parse_(); diff --git a/test/src/testplane.js b/test/src/testplane.js index 3c46df02e..8c4f826fd 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -401,12 +401,22 @@ describe("testplane", () => { const results = { fullTitle: () => "Title", browserId: "chrome", + browserVersion: "1", }; mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL, results)); return runTestplane().then(success => assert.isFalse(success)); }); + it("should halt if there were some errors", () => { + const testplane = mkTestplane_(); + const err = new Error(); + + mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.ERROR, err)); + + return testplane.run().then(() => assert.calledOnceWith(testplane.halt, err)); + }); + it("should save failed tests", async () => { const results = { fullTitle: () => "Title", @@ -500,6 +510,7 @@ describe("testplane", () => { const results = { fullTitle: () => "Title", browserId: "chrome", + browserVersion: "1", }; const omitEvents = ["EXIT", "NEW_BROWSER", "UPDATE_REFERENCE"]; @@ -798,6 +809,7 @@ describe("testplane", () => { const results = { fullTitle: () => "Title", browserId: "chrome", + browserVersion: "1", }; mkNodejsEnvRunner_(runner => { From 709514daa1e0887d16c212c30da36f2c65faaa98 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Mon, 29 Jul 2024 11:51:48 +0300 Subject: [PATCH 11/12] refactor: minor fixes --- src/test-reader/test-parser.js | 8 ++++---- test/src/test-reader/test-parser.js | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 23319c20a..5cce762f9 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -18,6 +18,8 @@ const fs = require("fs-extra"); const logger = require("../utils/logger"); const { getShortMD5 } = require("../utils/crypto"); +const getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); + class TestParser extends EventEmitter { #opts; #failedTests; @@ -35,8 +37,6 @@ class TestParser extends EventEmitter { this.#buildInstructions = new InstructionsList(); } - getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); - async loadFiles(files, config) { const eventBus = new EventEmitter(); const { @@ -81,7 +81,7 @@ class TestParser extends EventEmitter { : config.lastFailed.input.split(",").map(v => v.trim()); for (const inputPath of inputPaths) { for (const test of await fs.readJSON(inputPath)) { - this.#failedTests.add(this.getFailedTestId(test)); + this.#failedTests.add(getFailedTestId(test)); } } } catch { @@ -140,7 +140,7 @@ class TestParser extends EventEmitter { if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { treeBuilder.addTestFilter(({ fullTitle, ...rest }) => { - return this.#failedTests.has(this.getFailedTestId({ fullTitle: fullTitle(), ...rest })); + return this.#failedTests.has(getFailedTestId({ fullTitle: fullTitle(), ...rest })); }); } diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index 14b9e0426..f36278d4c 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -567,12 +567,25 @@ describe("test-reader/test-parser", () => { }); it("should call addTestFilter if config.lastFailed.only is set", async () => { - fs.readJSON.resolves([ + const tests = [ + { + fullTitle: () => "title", + browserId: "chrome", + browserVersion: "1", + }, { - fullTitle: "title", + fullTitle: () => "title2", browserId: "chrome", browserVersion: "1", }, + ]; + + fs.readJSON.resolves([ + { + fullTitle: tests[0].fullTitle(), + browserId: tests[0].browserId, + browserVersion: tests[0].browserVersion, + }, ]); const config = makeConfigStub({ @@ -584,7 +597,8 @@ describe("test-reader/test-parser", () => { await parse_({ config }, config); - assert.calledOnce(TreeBuilder.prototype.addTestFilter); + assert.equal(TreeBuilder.prototype.addTestFilter.lastCall.args[0](tests[0]), true); + assert.equal(TreeBuilder.prototype.addTestFilter.lastCall.args[0](tests[1]), false); }); }); From b85ef024e16614798c3163add29fa1e028efd393 Mon Sep 17 00:00:00 2001 From: Ivan Kabir Date: Mon, 29 Jul 2024 13:49:21 +0300 Subject: [PATCH 12/12] refactor: minor changes --- test/src/test-reader/test-parser.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index f36278d4c..d52ac9540 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -597,8 +597,10 @@ describe("test-reader/test-parser", () => { await parse_({ config }, config); - assert.equal(TreeBuilder.prototype.addTestFilter.lastCall.args[0](tests[0]), true); - assert.equal(TreeBuilder.prototype.addTestFilter.lastCall.args[0](tests[1]), false); + const filter = TreeBuilder.prototype.addTestFilter.lastCall.args[0]; + + assert.equal(filter(tests[0]), true); + assert.equal(filter(tests[1]), false); }); });