feat: add lastFailed config option

docs/config.md

@@ -73,6 +73,7 @@
   - [List of useful plugins](#list-of-useful-plugins)
 - [prepareBrowser](#preparebrowser)
 - [prepareEnvironment](#prepareenvironment)
+- [lastFailed](#lastfailed)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -731,3 +732,14 @@ Full list of parameters:
      - waitServerTimeout (optional) `Number` - timeout to wait for server to be ready (ms). 60_000 by default
      - probeRequestTimeout (optional) `Number` - one request timeout (ms), after which request will be aborted. 10_000 by default
      - probeRequestInterval (optional) `Number` - interval between ready probe requests (ms). 1_000 by default
+
+### lastFailed
+Allows you to run only tests that failed on the last run. Disabled by default - it means run all tests, but the file with the failed tests is always written.
+
+```js
+lastFailed: {
+    only: true, // true means run only failed, false - run all (default: false)
+    input: ['.testplane/failed.json', '.testplane/failed2.json'], // File/files to read failed tests list from (default: '.testplane/failed.json')
+    output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json')
+}
+```

src/config/defaults.js

@@ -94,6 +94,11 @@ module.exports = {
     headless: null,
     isolation: null,
     testRunEnv: NODEJS_TEST_RUN_ENV,
+    lastFailed: {
+        only: false,
+        output: ".testplane/failed.json",
+        input: ".testplane/failed.json",
+    },
     devServer: {
         command: null,
         cwd: null,

src/config/index.ts

@@ -72,6 +72,7 @@ export class Config {
             const browserOptions = _.extend({}, browser, {
                 id: id,
                 system: this.system,
+                lastFailed: this.lastFailed,
             });
 
             return new BrowserConfig(browserOptions);

src/config/options.js

@@ -107,6 +107,35 @@ const rootSection = section(
 
         plugins: options.anyObject(),
 
+        lastFailed: section({
+            only: options.boolean("lastFailed.only"),
+            input: option({
+                defaultValue: defaults.lastFailed.input,
+                validate: value => {
+                    if (!_.isString(value) && !_.isArray(value)) {
+                        throw new Error('"lastFailed.input" must be a string or an array');
+                    }
+                    if (!_.isArray(value) && !value.endsWith(".json")) {
+                        throw new Error('"lastFailed.input" must have .json extension');
+                    }
+                    if (_.isArray(value) && value.filter(v => !v.endsWith(".json")).length) {
+                        throw new Error('"lastFailed.input" elements must have .json extension');
+                    }
+                },
+            }),
+            output: option({
+                defaultValue: defaults.lastFailed.output,
+                validate: value => {
+                    if (!_.isString(value)) {
+                        throw new Error('"lastFailed.output" must be a string');
+                    }
+                    if (!value.endsWith(".json")) {
+                        throw new Error('"lastFailed.output" must have .json extension');
+                    }
+                },
+            }),
+        }),
+
         sets: map(
             section({
                 files: option({

src/config/types.ts

@@ -140,6 +140,12 @@ export interface CommonConfig {
     headless: "old" | "new" | boolean | null;
     isolation: boolean;
 
+    lastFailed: {
+        only: boolean;
+        input: string | Array<string>;
+        output: string;
+    };
+
     openAndWaitOpts: {
         timeout?: number;
         waitNetworkIdle: boolean;

src/test-reader/test-parser.js

@@ -14,9 +14,15 @@ const { MasterEvents } = require("../events");
 const _ = require("lodash");
 const clearRequire = require("clear-require");
 const path = require("path");
+const fs = require("fs-extra");
+const logger = require("../utils/logger");
+const { getShortMD5 } = require("../utils/crypto");
+
+const getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`);
 
 class TestParser extends EventEmitter {
     #opts;
+    #failedTests;
     #buildInstructions;
 
     /**
@@ -27,6 +33,7 @@ class TestParser extends EventEmitter {
         super();
 
         this.#opts = opts;
+        this.#failedTests = new Set();
         this.#buildInstructions = new InstructionsList();
     }
 
@@ -66,6 +73,24 @@ class TestParser extends EventEmitter {
         const esmDecorator = f => f + `?rand=${rand}`;
         await readFiles(files, { esmDecorator, config: mochaOpts, eventBus });
 
+        if (config.lastFailed.only) {
+            try {
+                this.#failedTests = new Set();
+                const inputPaths = _.isArray(config.lastFailed.input)
+                    ? config.lastFailed.input
+                    : config.lastFailed.input.split(",").map(v => v.trim());
+                for (const inputPath of inputPaths) {
+                    for (const test of await fs.readJSON(inputPath)) {
+                        this.#failedTests.add(getFailedTestId(test));
+                    }
+                }
+            } catch {
+                logger.warn(
+                    `Could not read failed tests data at ${config.lastFailed.input}. Running all tests instead`,
+                );
+            }
+        }
+
         revertTransformHook();
     }
 
@@ -113,6 +138,12 @@ class TestParser extends EventEmitter {
             treeBuilder.addTestFilter(test => grep.test(test.fullTitle()));
         }
 
+        if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) {
+            treeBuilder.addTestFilter(({ fullTitle, ...rest }) => {
+                return this.#failedTests.has(getFailedTestId({ fullTitle: fullTitle(), ...rest }));
+            });
+        }
+
         const rootSuite = treeBuilder.applyFilters().getRootSuite();
 
         const tests = rootSuite.getTests();

src/testplane.ts

@@ -1,5 +1,6 @@
 import { CommanderStatic } from "@gemini-testing/commander";
 import _ from "lodash";
+import fs from "fs-extra";
 import { Stats as RunnerStats } from "./stats";
 import { BaseTestplane } from "./base-testplane";
 import { MainRunner as NodejsEnvRunner } from "./runner";
@@ -16,7 +17,7 @@ import logger from "./utils/logger";
 import { isRunInNodeJsEnv } from "./utils/config";
 import { initDevServer } from "./dev-server";
 import { ConfigInput } from "./config/types";
-import { MasterEventHandler, Test } from "./types";
+import { MasterEventHandler, Test, TestResult } from "./types";
 
 interface RunOpts {
     browsers: string[];
@@ -37,9 +38,16 @@ interface RunOpts {
     devtools: boolean;
 }
 
+export type FailedListItem = {
+    browserVersion?: string;
+    browserId?: string;
+    fullTitle: string;
+};
+
 interface ReadTestsOpts extends Pick<RunOpts, "browsers" | "sets" | "grep" | "replMode"> {
     silent: boolean;
     ignore: string | string[];
+    failed: FailedListItem[];
 }
 
 export interface Testplane {
@@ -50,12 +58,14 @@ export interface Testplane {
 
 export class Testplane extends BaseTestplane {
     protected failed: boolean;
+    protected failedList: FailedListItem[];
     protected runner: NodejsEnvRunner | BrowserEnvRunner | null;
 
     constructor(config?: string | ConfigInput) {
         super(config);
 
         this.failed = false;
+        this.failedList = [];
         this.runner = null;
     }
 
@@ -101,7 +111,13 @@ export class Testplane extends BaseTestplane {
         );
         this.runner = runner;
 
-        this.on(MasterEvents.TEST_FAIL, () => this._fail()).on(MasterEvents.ERROR, (err: Error) => this.halt(err));
+        this.on(MasterEvents.TEST_FAIL, res => {
+            this._fail();
+            this._addFailedTest(res);
+        });
+        this.on(MasterEvents.ERROR, (err: Error) => this.halt(err));
+
+        this.on(MasterEvents.RUNNER_END, async () => await this._saveFailed());
 
         await initReporters(reporters, this);
 
@@ -119,6 +135,10 @@ export class Testplane extends BaseTestplane {
         return !this.isFailed();
     }
 
+    protected async _saveFailed(): Promise<void> {
+        await fs.outputJSON(this._config.lastFailed.output, this.failedList); // No spaces because users usually don't need to read it
+    }
+
     protected async _readTests(
         testPaths: string[] | TestCollection,
         opts: Partial<ReadTestsOpts>,
@@ -169,6 +189,14 @@ export class Testplane extends BaseTestplane {
         this.failed = true;
     }
 
+    protected _addFailedTest(result: TestResult): void {
+        this.failedList.push({
+            fullTitle: result.fullTitle(),
+            browserId: result.browserId,
+            browserVersion: result.browserVersion,
+        });
+    }
+
     isWorker(): boolean {
         return false;
     }

test/src/config/options.js

@@ -463,6 +463,143 @@ describe("config options", () => {
         });
     });
 
+    describe("lastFailed", () => {
+        describe("only", () => {
+            it("should throw error if only is not a boolean", () => {
+                const readConfig = {
+                    lastFailed: {
+                        only: "String",
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.only" must be a boolean');
+            });
+        });
+
+        describe("input", () => {
+            it("should throw error if input is not a string", () => {
+                const readConfig = {
+                    lastFailed: {
+                        input: false,
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string or an array');
+            });
+
+            it("should throw error if input is a string without .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        input: "string",
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.input" must have .json extension');
+            });
+
+            it("should not throw error if input is a string with .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        input: "string.json",
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.doesNotThrow(() => createConfig());
+            });
+
+            it("should throw error if input is an array that contains a string without .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        input: ["string.json", "string"],
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.input" elements must have .json extension');
+            });
+
+            it("should not throw error if input is an array that contains only strings with .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        input: ["string.json"],
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.doesNotThrow(() => createConfig());
+            });
+        });
+
+        describe("output", () => {
+            it("should throw error if output is not a string", () => {
+                const readConfig = {
+                    lastFailed: {
+                        output: false,
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.output" must be a string');
+            });
+
+            it("should throw error if output is a string without .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        output: "string",
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.throws(() => createConfig(), Error, '"lastFailed.output" must have .json extension');
+            });
+
+            it("should not throw error if output is a string with .json at the end", () => {
+                const readConfig = {
+                    lastFailed: {
+                        output: "string.json",
+                    },
+                };
+
+                Config.read.returns(readConfig);
+
+                assert.doesNotThrow(() => createConfig());
+            });
+        });
+
+        it("should set default lastFailed option if it does not set in config file", () => {
+            const config = createConfig();
+
+            assert.deepEqual(config.lastFailed, defaults.lastFailed);
+        });
+
+        it("should override lastFailed option", () => {
+            const newValue = {
+                input: "some-path.json",
+                output: "some-path.json",
+                only: true,
+            };
+            const readConfig = { lastFailed: newValue };
+
+            Config.read.returns(readConfig);
+
+            const config = createConfig();
+
+            assert.deepEqual(config.lastFailed, newValue);
+        });
+    });
+
     describe("prepareEnvironment", () => {
         it("should throw error if prepareEnvironment is not a null or function", () => {
             const readConfig = { prepareEnvironment: "String" };