diff --git a/CHANGELOG.md b/CHANGELOG.md index cd12eae9a220..72272f4a0e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - `[jest-each]` Add support for keyPaths in test titles ([#6457](https://github.com/facebook/jest/pull/6457)) - `[jest-cli]` Add `jest --init` option that generates a basic configuration file with a short description for each option ([#6442](https://github.com/facebook/jest/pull/6442)) +- `[jest.retryTimes]` Add `jest.retryTimes()` option that allows failed tests to be retried n-times when using jest-circus. ([#6498](https://github.com/facebook/jest/pull/6498)) ### Fixes diff --git a/docs/JestObjectAPI.md b/docs/JestObjectAPI.md index 31ed2e713d56..65b6338e800e 100644 --- a/docs/JestObjectAPI.md +++ b/docs/JestObjectAPI.md @@ -21,6 +21,7 @@ The `jest` object is automatically in scope within every test file. The methods - [`jest.resetAllMocks()`](#jestresetallmocks) - [`jest.restoreAllMocks()`](#jestrestoreallmocks) - [`jest.resetModules()`](#jestresetmodules) +- [`jest.retryTimes()`](#jestretrytimes) - [`jest.runAllTicks()`](#jestrunallticks) - [`jest.runAllTimers()`](#jestrunalltimers) - [`jest.advanceTimersByTime(msToRun)`](#jestadvancetimersbytimemstorun) @@ -312,6 +313,37 @@ test('works too', () => { Returns the `jest` object for chaining. +### `jest.retryTimes()` + +Runs failed tests n-times until they pass or until the max number of retries are exhausted. This only works with jest-circus! + +Example in a test: + +```js +jest.retryTimes(3); +test('will fail', () => { + expect(true).toBe(false); +}); +``` + +To run with jest circus: + +Install jest-circus + +``` +yarn add --dev jest-circus +``` + +Then set as the testRunner in your jest config: + +```js +module.exports = { + testRunner: 'jest-circus/runner', +}; +``` + +Returns the `jest` object for chaining. + ### `jest.runAllTicks()` Exhausts the **micro**-task queue (usually interfaced in node via `process.nextTick`). diff --git a/e2e/__tests__/test_retries.test.js b/e2e/__tests__/test_retries.test.js new file mode 100644 index 000000000000..4d20afe36425 --- /dev/null +++ b/e2e/__tests__/test_retries.test.js @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const runJest = require('../runJest'); + +const ConditionalTest = require('../../scripts/ConditionalTest'); + +ConditionalTest.skipSuiteOnJasmine(); + +describe('Test Retries', () => { + const outputFileName = 'retries.result.json'; + const outputFilePath = path.join( + process.cwd(), + 'e2e/test-retries/', + outputFileName, + ); + + afterAll(() => { + fs.unlinkSync(outputFilePath); + }); + + it('retries failed tests if configured', () => { + let jsonResult; + + const reporterConfig = { + reporters: [ + ['/reporters/RetryReporter.js', {output: outputFilePath}], + ], + }; + + runJest('test-retries', [ + '--config', + JSON.stringify(reporterConfig), + 'retry.test.js', + ]); + + const testOutput = fs.readFileSync(outputFilePath, 'utf8'); + + try { + jsonResult = JSON.parse(testOutput); + } catch (err) { + throw new Error( + `Can't parse the JSON result from ${outputFileName}, ${err.toString()}`, + ); + } + + expect(jsonResult.numPassedTests).toBe(0); + expect(jsonResult.numFailedTests).toBe(1); + expect(jsonResult.numPendingTests).toBe(0); + expect(jsonResult.testResults[0].testResults[0].invocations).toBe(4); + }); + + it('does not retry by default', () => { + let jsonResult; + + const reporterConfig = { + reporters: [ + ['/reporters/RetryReporter.js', {output: outputFilePath}], + ], + }; + + runJest('test-retries', [ + '--config', + JSON.stringify(reporterConfig), + 'control.test.js', + ]); + + const testOutput = fs.readFileSync(outputFilePath, 'utf8'); + + try { + jsonResult = JSON.parse(testOutput); + } catch (err) { + throw new Error( + `Can't parse the JSON result from ${outputFileName}, ${err.toString()}`, + ); + } + + expect(jsonResult.numPassedTests).toBe(0); + expect(jsonResult.numFailedTests).toBe(1); + expect(jsonResult.numPendingTests).toBe(0); + expect(jsonResult.testResults[0].testResults[0].invocations).toBe(1); + }); +}); diff --git a/e2e/runJest.js b/e2e/runJest.js index c4b859c0ee5c..d8d4a37cdcb1 100644 --- a/e2e/runJest.js +++ b/e2e/runJest.js @@ -55,6 +55,7 @@ function runJest( NODE_PATH: options.nodePath, }) : process.env; + const result = spawnSync(JEST_PATH, args || [], { cwd: dir, env, diff --git a/e2e/test-retries/__tests__/control.test.js b/e2e/test-retries/__tests__/control.test.js new file mode 100644 index 000000000000..6943fbdb844e --- /dev/null +++ b/e2e/test-retries/__tests__/control.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +it('retryTimes not set', () => { + expect(true).toBeFalsy(); +}); diff --git a/e2e/test-retries/__tests__/retry.test.js b/e2e/test-retries/__tests__/retry.test.js new file mode 100644 index 000000000000..187ca0227723 --- /dev/null +++ b/e2e/test-retries/__tests__/retry.test.js @@ -0,0 +1,13 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +jest.retryTimes(3); + +it('retryTimes set', () => { + expect(true).toBeFalsy(); +}); diff --git a/e2e/test-retries/package.json b/e2e/test-retries/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/test-retries/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/e2e/test-retries/reporters/RetryReporter.js b/e2e/test-retries/reporters/RetryReporter.js new file mode 100644 index 000000000000..d2bb466e9a2d --- /dev/null +++ b/e2e/test-retries/reporters/RetryReporter.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2014-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +'use strict'; + +const fs = require('fs'); + +/** + * RetryReporter + * Reporter for testing output of onRunComplete + */ +class RetryReporter { + constructor(globalConfig, options) { + this._options = options; + } + + onRunComplete(contexts, results) { + if (this._options.output) { + fs.writeFileSync(this._options.output, JSON.stringify(results, null, 2), { + encoding: 'utf8', + }); + } + } +} + +module.exports = RetryReporter; diff --git a/packages/jest-circus/src/event_handler.js b/packages/jest-circus/src/event_handler.js index 23782bf028e5..c30537c14e77 100644 --- a/packages/jest-circus/src/event_handler.js +++ b/packages/jest-circus/src/event_handler.js @@ -114,6 +114,7 @@ const handler: EventHandler = (event, state): void => { case 'test_start': { state.currentlyRunningTest = event.test; event.test.startedAt = Date.now(); + event.test.invocations += 1; break; } case 'test_fn_failure': { diff --git a/packages/jest-circus/src/legacy_code_todo_rewrite/jest_adapter_init.js b/packages/jest-circus/src/legacy_code_todo_rewrite/jest_adapter_init.js index 6a1e50f8614a..4fc29a28d48a 100644 --- a/packages/jest-circus/src/legacy_code_todo_rewrite/jest_adapter_init.js +++ b/packages/jest-circus/src/legacy_code_todo_rewrite/jest_adapter_init.js @@ -137,6 +137,7 @@ export const runAndTransformResultsToJestFormat = async ({ duration: testResult.duration, failureMessages: testResult.errors, fullName: ancestorTitles.concat(title).join(' '), + invocations: testResult.invocations, location: testResult.location, numPassingAsserts: 0, status, diff --git a/packages/jest-circus/src/run.js b/packages/jest-circus/src/run.js index b929757c43de..0853cb31f67e 100644 --- a/packages/jest-circus/src/run.js +++ b/packages/jest-circus/src/run.js @@ -46,8 +46,27 @@ const _runTestsForDescribeBlock = async (describeBlock: DescribeBlock) => { for (const hook of beforeAll) { await _callHook({describeBlock, hook}); } + + // Tests that fail and are retried we run after other tests + const retryTimes = parseInt(global[Symbol.for('RETRY_TIMES')], 10) || 0; + const deferredRetryTests = []; + for (const test of describeBlock.tests) { await _runTest(test); + + if (retryTimes > 0 && test.errors.length > 0) { + deferredRetryTests.push(test); + } + } + + // Re-run failed tests n-times if configured + for (const test of deferredRetryTests) { + let numRetriesAvailable = retryTimes; + + while (numRetriesAvailable > 0 && test.errors.length > 0) { + await _runTest(test); + numRetriesAvailable--; + } } for (const child of describeBlock.children) { diff --git a/packages/jest-circus/src/utils.js b/packages/jest-circus/src/utils.js index f2ecfcf4a708..eda159eb7d74 100644 --- a/packages/jest-circus/src/utils.js +++ b/packages/jest-circus/src/utils.js @@ -80,6 +80,7 @@ export const makeTest = ( duration: null, errors: [], fn, + invocations: 0, mode: _mode, name: convertDescriptorToString(name), parent, @@ -276,6 +277,7 @@ const makeTestResults = (describeBlock: DescribeBlock, config): TestResults => { testResults.push({ duration: test.duration, errors: test.errors.map(_formatError), + invocations: test.invocations, location, status, testPath, diff --git a/packages/jest-cli/src/cli/args.js b/packages/jest-cli/src/cli/args.js index 6ecb7597daa3..b12f0a6c84a2 100644 --- a/packages/jest-cli/src/cli/args.js +++ b/packages/jest-cli/src/cli/args.js @@ -578,7 +578,7 @@ export const options = { description: 'Allows the use of a custom results processor. ' + 'This processor must be a node module that exports ' + - 'a function expecting as the first argument the result object', + 'a function expecting as the first argument the result object.', type: 'string', }, testRunner: { diff --git a/packages/jest-runtime/src/index.js b/packages/jest-runtime/src/index.js index 70a1b6c9e5d3..bc868d8fbbb5 100644 --- a/packages/jest-runtime/src/index.js +++ b/packages/jest-runtime/src/index.js @@ -830,6 +830,11 @@ class Runtime { return jestObject; }; + const retryTimes = (numTestRetries: number) => { + this._environment.global[Symbol.for('RETRY_TIMES')] = numTestRetries; + return jestObject; + }; + const jestObject = { addMatchers: (matchers: Object) => this._environment.global.jasmine.addMatchers(matchers), @@ -855,6 +860,7 @@ class Runtime { resetModuleRegistry: resetModules, resetModules, restoreAllMocks, + retryTimes, runAllImmediates: () => this._environment.fakeTimers.runAllImmediates(), runAllTicks: () => this._environment.fakeTimers.runAllTicks(), runAllTimers: () => this._environment.fakeTimers.runAllTimers(), diff --git a/types/Circus.js b/types/Circus.js index 6dc18e37f2f7..6490cf24932f 100644 --- a/types/Circus.js +++ b/types/Circus.js @@ -194,6 +194,7 @@ export type TestEntry = {| asyncError: Exception, // Used if the test failure contains no usable stack trace errors: TestError, fn: ?TestFn, + invocations: number, mode: TestMode, name: TestName, parent: DescribeBlock, diff --git a/types/Jest.js b/types/Jest.js index ec01cd6d54c9..88966e299f2c 100644 --- a/types/Jest.js +++ b/types/Jest.js @@ -32,6 +32,7 @@ export type Jest = {| resetModuleRegistry(): Jest, resetModules(): Jest, restoreAllMocks(): Jest, + retryTimes(numRetries: number): Jest, runAllImmediates(): void, runAllTicks(): void, runAllTimers(): void,