Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(runner): Make exit code configurable when tests are failing #3116

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions lib/browser_collection.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,24 @@ class BrowserCollection {
return this.browsers.map((browser) => browser.serialize())
}

getResults () {
calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
if (results.disconnected || singleRunBrowserNotCaptured) {
return 1
}
if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
return 0
}
if (results.error) {
johnjbarton marked this conversation as resolved.
Show resolved Hide resolved
return 1
}
if (failOnFailingTestSuite === false) {
// Tests executed without infrastructure error, exit with 0 independent of test status.
return 0
}
return results.failed ? 1 : 0
}

getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
const results = this.browsers.reduce((previous, current) => {
previous.success += current.lastResult.success
previous.failed += current.lastResult.failed
Expand All @@ -56,7 +73,7 @@ class BrowserCollection {
}, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0})

// compute exit status code
results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0
results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)

return results
}
Expand Down
6 changes: 6 additions & 0 deletions lib/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) {
options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
}

if (helper.isString(options.failOnFailingTestSuite)) {
options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
}

if (helper.isString(options.formatError)) {
let required
try {
Expand Down Expand Up @@ -196,6 +200,8 @@ function describeStart () {
.describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
.describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
.describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
.describe('help', 'Print usage and options.')
}

Expand Down
9 changes: 1 addition & 8 deletions lib/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {

const emitRunCompleteIfAllBrowsersDone = () => {
if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {
const results = singleRunBrowsers.getResults()
if (singleRunBrowserNotCaptured) {
results.exitCode = 1
} else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
results.exitCode = 0
this.log.warn('Test suite was empty.')
}
this.emit('run_complete', singleRunBrowsers, results)
this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))
}
}

Expand Down
86 changes: 86 additions & 0 deletions test/unit/browser_collection.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -258,4 +258,90 @@ describe('BrowserCollection', () => {
})
})
})

// ============================================================================
// server.calculateExitCode
// ============================================================================
describe('calculateExitCode', () => {
const EXIT_CODE_ERROR = 1
const EXIT_CODE_SUCCESS = 0

describe('no tests', () => {
const results = {
success: 0,
failed: 0,
error: true
}
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall fail if failOnEmptyTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is set', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
})
describe('all test passed, no errors', () => {
const results = {
success: 10,
failed: 0,
error: false
}
it('shall fail if singleRunBrowserNotCaptured is true', () => {
const res = collection.calculateExitCode(results, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
describe('all test passed, with error', () => {
const results = {
success: 10,
failed: 5,
error: false
}
it('shall fail if singleRunBrowserNotCaptured is true', () => {
const res = collection.calculateExitCode(results, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnEmptyTestSuite not is set', () => {
const res = collection.calculateExitCode(results, false, false)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall fail if failOnFailingTestSuite is true', () => {
const res = collection.calculateExitCode(results, false, true, true)
expect(res).to.be.equal(EXIT_CODE_ERROR)
})
it('shall pass if failOnFailingTestSuite is false', () => {
const res = collection.calculateExitCode(results, false, true, false)
expect(res).to.be.equal(EXIT_CODE_SUCCESS)
})
})
})
})