diff --git a/docs/config.md b/docs/config.md index 7114edc1..666ab5ae 100644 --- a/docs/config.md +++ b/docs/config.md @@ -73,6 +73,7 @@ - [List of useful plugins](#list-of-useful-plugins) - [prepareBrowser](#preparebrowser) - [prepareEnvironment](#prepareenvironment) +- [lastFailed](#lastfailed) @@ -731,3 +732,14 @@ Full list of parameters: - waitServerTimeout (optional) `Number` - timeout to wait for server to be ready (ms). 60_000 by default - probeRequestTimeout (optional) `Number` - one request timeout (ms), after which request will be aborted. 10_000 by default - probeRequestInterval (optional) `Number` - interval between ready probe requests (ms). 1_000 by default + +### lastFailed +Allows you to run only tests that failed on the last run. Disabled by default - it means run all tests, but the file with the failed tests is always written. + +```js +lastFailed: { + only: true, // true means run only failed, false - run all (default: false) + input: ['.testplane/failed.json', '.testplane/failed2.json'], // File/files to read failed tests list from (default: '.testplane/failed.json') + output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') +} +``` \ No newline at end of file diff --git a/src/config/defaults.js b/src/config/defaults.js index a046ab4a..8e87ad72 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -94,6 +94,11 @@ module.exports = { headless: null, isolation: null, testRunEnv: NODEJS_TEST_RUN_ENV, + lastFailed: { + only: false, + output: ".testplane/failed.json", + input: ".testplane/failed.json", + }, devServer: { command: null, cwd: null, diff --git a/src/config/index.ts b/src/config/index.ts index 2f7432ff..3edd1fdc 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -72,6 +72,7 @@ export class Config { const browserOptions = _.extend({}, browser, { id: id, system: this.system, + lastFailed: this.lastFailed, }); return new BrowserConfig(browserOptions); diff --git a/src/config/options.js b/src/config/options.js index 1c57df8c..201120b3 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -107,6 +107,35 @@ const rootSection = section( plugins: options.anyObject(), + lastFailed: section({ + only: options.boolean("lastFailed.only"), + input: option({ + defaultValue: defaults.lastFailed.input, + validate: value => { + if (!_.isString(value) && !_.isArray(value)) { + throw new Error('"lastFailed.input" must be a string or an array'); + } + if (!_.isArray(value) && !value.endsWith(".json")) { + throw new Error('"lastFailed.input" must have .json extension'); + } + if (_.isArray(value) && value.filter(v => !v.endsWith(".json")).length) { + throw new Error('"lastFailed.input" elements must have .json extension'); + } + }, + }), + output: option({ + defaultValue: defaults.lastFailed.output, + validate: value => { + if (!_.isString(value)) { + throw new Error('"lastFailed.output" must be a string'); + } + if (!value.endsWith(".json")) { + throw new Error('"lastFailed.output" must have .json extension'); + } + }, + }), + }), + sets: map( section({ files: option({ diff --git a/src/config/types.ts b/src/config/types.ts index fe91cce0..547affe8 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -140,6 +140,12 @@ export interface CommonConfig { headless: "old" | "new" | boolean | null; isolation: boolean; + lastFailed: { + only: boolean; + input: string | Array; + output: string; + }; + openAndWaitOpts: { timeout?: number; waitNetworkIdle: boolean; diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 4401a8cd..5cce762f 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -14,9 +14,15 @@ const { MasterEvents } = require("../events"); const _ = require("lodash"); const clearRequire = require("clear-require"); const path = require("path"); +const fs = require("fs-extra"); +const logger = require("../utils/logger"); +const { getShortMD5 } = require("../utils/crypto"); + +const getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); class TestParser extends EventEmitter { #opts; + #failedTests; #buildInstructions; /** @@ -27,6 +33,7 @@ class TestParser extends EventEmitter { super(); this.#opts = opts; + this.#failedTests = new Set(); this.#buildInstructions = new InstructionsList(); } @@ -66,6 +73,24 @@ class TestParser extends EventEmitter { const esmDecorator = f => f + `?rand=${rand}`; await readFiles(files, { esmDecorator, config: mochaOpts, eventBus }); + if (config.lastFailed.only) { + try { + this.#failedTests = new Set(); + const inputPaths = _.isArray(config.lastFailed.input) + ? config.lastFailed.input + : config.lastFailed.input.split(",").map(v => v.trim()); + for (const inputPath of inputPaths) { + for (const test of await fs.readJSON(inputPath)) { + this.#failedTests.add(getFailedTestId(test)); + } + } + } catch { + logger.warn( + `Could not read failed tests data at ${config.lastFailed.input}. Running all tests instead`, + ); + } + } + revertTransformHook(); } @@ -113,6 +138,12 @@ class TestParser extends EventEmitter { treeBuilder.addTestFilter(test => grep.test(test.fullTitle())); } + if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { + treeBuilder.addTestFilter(({ fullTitle, ...rest }) => { + return this.#failedTests.has(getFailedTestId({ fullTitle: fullTitle(), ...rest })); + }); + } + const rootSuite = treeBuilder.applyFilters().getRootSuite(); const tests = rootSuite.getTests(); diff --git a/src/testplane.ts b/src/testplane.ts index 2307f0dd..7fc00cf5 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -1,5 +1,6 @@ import { CommanderStatic } from "@gemini-testing/commander"; import _ from "lodash"; +import fs from "fs-extra"; import { Stats as RunnerStats } from "./stats"; import { BaseTestplane } from "./base-testplane"; import { MainRunner as NodejsEnvRunner } from "./runner"; @@ -16,7 +17,7 @@ import logger from "./utils/logger"; import { isRunInNodeJsEnv } from "./utils/config"; import { initDevServer } from "./dev-server"; import { ConfigInput } from "./config/types"; -import { MasterEventHandler, Test } from "./types"; +import { MasterEventHandler, Test, TestResult } from "./types"; interface RunOpts { browsers: string[]; @@ -37,9 +38,16 @@ interface RunOpts { devtools: boolean; } +export type FailedListItem = { + browserVersion?: string; + browserId?: string; + fullTitle: string; +}; + interface ReadTestsOpts extends Pick { silent: boolean; ignore: string | string[]; + failed: FailedListItem[]; } export interface Testplane { @@ -50,12 +58,14 @@ export interface Testplane { export class Testplane extends BaseTestplane { protected failed: boolean; + protected failedList: FailedListItem[]; protected runner: NodejsEnvRunner | BrowserEnvRunner | null; constructor(config?: string | ConfigInput) { super(config); this.failed = false; + this.failedList = []; this.runner = null; } @@ -101,7 +111,13 @@ export class Testplane extends BaseTestplane { ); this.runner = runner; - this.on(MasterEvents.TEST_FAIL, () => this._fail()).on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + this.on(MasterEvents.TEST_FAIL, res => { + this._fail(); + this._addFailedTest(res); + }); + this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + + this.on(MasterEvents.RUNNER_END, async () => await this._saveFailed()); await initReporters(reporters, this); @@ -119,6 +135,10 @@ export class Testplane extends BaseTestplane { return !this.isFailed(); } + protected async _saveFailed(): Promise { + await fs.outputJSON(this._config.lastFailed.output, this.failedList); // No spaces because users usually don't need to read it + } + protected async _readTests( testPaths: string[] | TestCollection, opts: Partial, @@ -169,6 +189,14 @@ export class Testplane extends BaseTestplane { this.failed = true; } + protected _addFailedTest(result: TestResult): void { + this.failedList.push({ + fullTitle: result.fullTitle(), + browserId: result.browserId, + browserVersion: result.browserVersion, + }); + } + isWorker(): boolean { return false; } diff --git a/test/src/config/options.js b/test/src/config/options.js index 4ab17b9d..360ba8ae 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -463,6 +463,143 @@ describe("config options", () => { }); }); + describe("lastFailed", () => { + describe("only", () => { + it("should throw error if only is not a boolean", () => { + const readConfig = { + lastFailed: { + only: "String", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.only" must be a boolean'); + }); + }); + + describe("input", () => { + it("should throw error if input is not a string", () => { + const readConfig = { + lastFailed: { + input: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string or an array'); + }); + + it("should throw error if input is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must have .json extension'); + }); + + it("should not throw error if input is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + + it("should throw error if input is an array that contains a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json", "string"], + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" elements must have .json extension'); + }); + + it("should not throw error if input is an array that contains only strings with .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json"], + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + }); + + describe("output", () => { + it("should throw error if output is not a string", () => { + const readConfig = { + lastFailed: { + output: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must be a string'); + }); + + it("should throw error if output is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must have .json extension'); + }); + + it("should not throw error if output is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + }); + + it("should set default lastFailed option if it does not set in config file", () => { + const config = createConfig(); + + assert.deepEqual(config.lastFailed, defaults.lastFailed); + }); + + it("should override lastFailed option", () => { + const newValue = { + input: "some-path.json", + output: "some-path.json", + only: true, + }; + const readConfig = { lastFailed: newValue }; + + Config.read.returns(readConfig); + + const config = createConfig(); + + assert.deepEqual(config.lastFailed, newValue); + }); + }); + describe("prepareEnvironment", () => { it("should throw error if prepareEnvironment is not a null or function", () => { const readConfig = { prepareEnvironment: "String" }; diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index eb3f7d6b..d52ac954 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -14,6 +14,7 @@ const proxyquire = require("proxyquire").noCallThru(); const path = require("path"); const { EventEmitter } = require("events"); const _ = require("lodash"); +const fs = require("fs-extra"); const { NEW_BUILD_INSTRUCTION } = TestReaderEvents; @@ -40,6 +41,7 @@ describe("test-reader/test-parser", () => { "./test-transformer": { setupTransformHook }, }).TestParser; + sandbox.stub(fs, "readJSON").resolves([]); sandbox.stub(InstructionsList.prototype, "push").returnsThis(); sandbox.stub(InstructionsList.prototype, "exec").returns(new Suite()); }); @@ -320,6 +322,68 @@ describe("test-reader/test-parser", () => { }); }); + describe("failed tests", () => { + it("should read if config.lastFailed.only is set", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "file.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "file.json"); + }); + + it("should read from one file if config.lastFailed.input is a string", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + }); + + it("should read from multiple files if config.lastFailed.input is a string with commas", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json, failed2.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should read from multiple files if config.lastFailed.input is an array", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: ["failed.json", "failed2.json"], + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should not read if config.lastFailed.only is not set", async () => { + await loadFiles_(); + + assert.notCalled(fs.readJSON); + }); + }); + describe("read files", () => { it("should read passed files", async () => { const files = ["foo/bar", "baz/qux"]; @@ -475,13 +539,14 @@ describe("test-reader/test-parser", () => { }); describe("parse", () => { - const parse_ = async ({ files, browserId, config, grep } = {}) => { + const parse_ = async ({ files, browserId, config, grep } = {}, loadFilesConfig) => { + loadFilesConfig = loadFilesConfig || makeConfigStub(); config = _.defaults(config, { desiredCapabilities: {}, }); const parser = new TestParser(); - await parser.loadFiles([], makeConfigStub()); + await parser.loadFiles([], loadFilesConfig); return parser.parse(files || [], { browserId, config, grep }); }; @@ -494,6 +559,51 @@ describe("test-reader/test-parser", () => { sandbox.stub(Suite.prototype, "getTests").returns([]); }); + describe("addTestFilter", () => { + it("should not call if config.lastFailed.only is not set", async () => { + await parse_(); + + assert.notCalled(TreeBuilder.prototype.addTestFilter); + }); + + it("should call addTestFilter if config.lastFailed.only is set", async () => { + const tests = [ + { + fullTitle: () => "title", + browserId: "chrome", + browserVersion: "1", + }, + { + fullTitle: () => "title2", + browserId: "chrome", + browserVersion: "1", + }, + ]; + + fs.readJSON.resolves([ + { + fullTitle: tests[0].fullTitle(), + browserId: tests[0].browserId, + browserVersion: tests[0].browserVersion, + }, + ]); + + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await parse_({ config }, config); + + const filter = TreeBuilder.prototype.addTestFilter.lastCall.args[0]; + + assert.equal(filter(tests[0]), true); + assert.equal(filter(tests[1]), false); + }); + }); + it("should execute build instructions", async () => { await parse_(); diff --git a/test/src/testplane.js b/test/src/testplane.js index f298e30a..8c4f826f 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -1,6 +1,7 @@ "use strict"; const _ = require("lodash"); +const fs = require("fs-extra"); const { EventEmitter } = require("events"); const pluginsLoader = require("plugins-loader"); const Promise = require("bluebird"); @@ -49,6 +50,8 @@ describe("testplane", () => { sandbox.stub(RuntimeConfig, "getInstance").returns({ extend: sandbox.stub() }); sandbox.stub(TestReader.prototype, "read").resolves(); sandbox.stub(RunnerStats, "create"); + sandbox.stub(fs, "readJSON").resolves([]); + sandbox.stub(fs, "outputJSON").resolves(); initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); @@ -217,7 +220,10 @@ describe("testplane", () => { describe("repl mode", () => { it("should not reset test timeout to 0 if run not in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: false } }); @@ -226,7 +232,10 @@ describe("testplane", () => { it("should reset test timeout to 0 if run in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: true } }); @@ -318,7 +327,12 @@ describe("testplane", () => { await runTestplane(testPaths, { browsers, grep, sets, replMode }); - assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { browsers, grep, sets, replMode }); + assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { + browsers, + grep, + sets, + replMode, + }); }); it("should accept test collection as first parameter", async () => { @@ -384,7 +398,12 @@ describe("testplane", () => { }); it('should return "false" if there are failed tests', () => { - mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL)); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL, results)); return runTestplane().then(success => assert.isFalse(success)); }); @@ -397,6 +416,27 @@ describe("testplane", () => { return testplane.run().then(() => assert.calledOnceWith(testplane.halt, err)); }); + + it("should save failed tests", async () => { + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => { + runner.emit(RunnerEvents.TEST_FAIL, results), runner.emit(RunnerEvents.RUNNER_END); + }); + + await runTestplane(); + + assert.calledWith(fs.outputJSON, "some-other-path", [ + { + fullTitle: results.fullTitle(), + browserId: results.browserId, + browserVersion: "1", + }, + ]); + }); }); describe("should passthrough", () => { @@ -467,6 +507,11 @@ describe("testplane", () => { it("all runner events with passed event data", () => { const runner = mkNodejsEnvRunner_(); const testplane = mkTestplane_(); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; const omitEvents = ["EXIT", "NEW_BROWSER", "UPDATE_REFERENCE"]; return testplane.run().then(() => { @@ -474,9 +519,9 @@ describe("testplane", () => { const spy = sinon.spy().named(`${name} handler`); testplane.on(event, spy); - runner.emit(event, "some-data"); + runner.emit(event, results); - assert.calledWith(spy, "some-data"); + assert.calledWith(spy, results); }); }); }); @@ -761,8 +806,14 @@ describe("testplane", () => { it('should return "true" after some test fail', () => { const testplane = mkTestplane_(); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => { - runner.emit(RunnerEvents.TEST_FAIL); + runner.emit(RunnerEvents.TEST_FAIL, results); assert.isTrue(testplane.isFailed()); }); diff --git a/test/utils.js b/test/utils.js index 79748fba..581469f2 100644 --- a/test/utils.js +++ b/test/utils.js @@ -30,6 +30,11 @@ function makeConfigStub(opts = {}) { testRunEnv: NODEJS_TEST_RUN_ENV, }, sets: {}, + lastFailed: { + only: false, + input: "some-path", + output: "some-other-path", + }, }); const config = { @@ -38,6 +43,7 @@ function makeConfigStub(opts = {}) { system: opts.system, sets: opts.sets, configPath: opts.configPath, + lastFailed: opts.lastFailed, }; opts.browsers.forEach(browserId => {