Skip to content

Commit

Permalink
fix(vitest): show all failed tests when rerunning a test (#6022)
Browse files Browse the repository at this point in the history
  • Loading branch information
sheremet-va authored Jul 2, 2024
1 parent a820b15 commit 91ba6f9
Show file tree
Hide file tree
Showing 14 changed files with 194 additions and 63 deletions.
2 changes: 1 addition & 1 deletion packages/vitest/src/api/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ export class WebSocketReporter implements Reporter {
})
}

onFinished(files?: File[], errors?: unknown[]) {
onFinished(files: File[], errors: unknown[]) {
this.clients.forEach((client) => {
client.onFinished?.(files, errors)?.catch?.(noop)
})
Expand Down
2 changes: 1 addition & 1 deletion packages/vitest/src/node/pools/typecheck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool {

// triggered by TSC watcher, not Vitest watcher, so we need to emulate what Vitest does in this case
if (ctx.config.watch && !ctx.runningPromise) {
await ctx.report('onFinished', files)
await ctx.report('onFinished', files, [])
await ctx.report('onWatcherStart', files, [
...(project.config.typecheck.ignoreSourceErrors ? [] : sourceErrors),
...ctx.state.getUnhandledErrors(),
Expand Down
121 changes: 71 additions & 50 deletions packages/vitest/src/node/reporters/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
getFullName,
getSafeTimers,
getSuites,
getTestName,
getTests,
hasFailed,
hasFailedSnapshot,
Expand All @@ -33,8 +34,8 @@ import {
formatTimeString,
getStateString,
getStateSymbol,
pointer,
renderSnapshotSummary,
taskFail,
} from './renderers/utils'

const BADGE_PADDING = ' '
Expand Down Expand Up @@ -63,6 +64,7 @@ export abstract class BaseReporter implements Reporter {
start = 0
end = 0
watchFilters?: string[]
failedUnwatchedFiles: Task[] = []
isTTY: boolean
ctx: Vitest = undefined!

Expand Down Expand Up @@ -115,59 +117,65 @@ export abstract class BaseReporter implements Reporter {
if (this.isTTY) {
return
}
const logger = this.ctx.logger
for (const pack of packs) {
const task = this.ctx.state.idMap.get(pack[0])
if (
task
&& 'filepath' in task
&& task.result?.state
&& task.result?.state !== 'run'
) {
const tests = getTests(task)
const failed = tests.filter(t => t.result?.state === 'fail')
const skipped = tests.filter(
t => t.mode === 'skip' || t.mode === 'todo',
)
let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`)
if (failed.length) {
state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}`
}
if (skipped.length) {
state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}`
}
let suffix = c.dim(' (') + state + c.dim(')')
if (task.result.duration) {
const color
= task.result.duration > this.ctx.config.slowTestThreshold
? c.yellow
: c.gray
suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`)
}
if (this.ctx.config.logHeapUsage && task.result.heap != null) {
suffix += c.magenta(
` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`,
)
}

let title = ` ${getStateSymbol(task)} `
if (task.projectName) {
title += formatProjectName(task.projectName)
}
title += `${task.name} ${suffix}`
logger.log(title)

// print short errors, full errors will be at the end in summary
for (const test of failed) {
logger.log(c.red(` ${pointer} ${getFullName(test, c.dim(' > '))}`))
test.result?.errors?.forEach((e) => {
logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`))
})
}
if (task) {
this.printTask(task)
}
}
}

protected printTask(task: Task) {
if (
!('filepath' in task)
|| !task.result?.state
|| task.result?.state === 'run') {
return
}
const logger = this.ctx.logger

const tests = getTests(task)
const failed = tests.filter(t => t.result?.state === 'fail')
const skipped = tests.filter(
t => t.mode === 'skip' || t.mode === 'todo',
)
let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`)
if (failed.length) {
state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}`
}
if (skipped.length) {
state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}`
}
let suffix = c.dim(' (') + state + c.dim(')')
if (task.result.duration) {
const color
= task.result.duration > this.ctx.config.slowTestThreshold
? c.yellow
: c.gray
suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`)
}
if (this.ctx.config.logHeapUsage && task.result.heap != null) {
suffix += c.magenta(
` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`,
)
}

let title = ` ${getStateSymbol(task)} `
if (task.projectName) {
title += formatProjectName(task.projectName)
}
title += `${task.name} ${suffix}`
logger.log(title)

// print short errors, full errors will be at the end in summary
for (const test of failed) {
logger.log(c.red(` ${taskFail} ${getTestName(test, c.dim(' > '))}`))
test.result?.errors?.forEach((e) => {
logger.log(c.red(` ${F_RIGHT} ${(e as any)?.message}`))
})
}
}

onWatcherStart(
files = this.ctx.state.getFiles(),
errors = this.ctx.state.getUnhandledErrors(),
Expand Down Expand Up @@ -233,6 +241,9 @@ export abstract class BaseReporter implements Reporter {
onWatcherRerun(files: string[], trigger?: string) {
this.resetLastRunLog()
this.watchFilters = files
this.failedUnwatchedFiles = this.ctx.state.getFiles().filter((file) => {
return !files.includes(file.filepath) && hasFailed(file)
})

files.forEach((filepath) => {
let reruns = this._filesInWatchMode.get(filepath) ?? 0
Expand Down Expand Up @@ -274,6 +285,12 @@ export abstract class BaseReporter implements Reporter {
)
}

if (!this.isTTY) {
for (const task of this.failedUnwatchedFiles) {
this.printTask(task)
}
}

this._timeStart = new Date()
this.start = performance.now()
}
Expand Down Expand Up @@ -375,7 +392,11 @@ export abstract class BaseReporter implements Reporter {
}

reportTestSummary(files: File[], errors: unknown[]) {
const tests = getTests(files)
const affectedFiles = [
...this.failedUnwatchedFiles,
...files,
]
const tests = getTests(affectedFiles)
const logger = this.ctx.logger

const executionTime = this.end - this.start
Expand Down Expand Up @@ -437,7 +458,7 @@ export abstract class BaseReporter implements Reporter {
}
}

logger.log(padTitle('Test Files'), getStateString(files))
logger.log(padTitle('Test Files'), getStateString(affectedFiles))
logger.log(padTitle('Tests'), getStateString(tests))
if (this.ctx.projects.some(c => c.config.typecheck.enabled)) {
const failed = tests.filter(
Expand Down
5 changes: 4 additions & 1 deletion packages/vitest/src/node/reporters/basic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import type { File } from '../../types/tasks'
import { BaseReporter } from './base'

export class BasicReporter extends BaseReporter {
isTTY = false
constructor() {
super()
this.isTTY = false
}

reportSummary(files: File[], errors: unknown[]) {
// non-tty mode doesn't add a new line
Expand Down
10 changes: 10 additions & 0 deletions packages/vitest/src/node/reporters/default.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ export class DefaultReporter extends BaseReporter {
files = this.ctx.state.getFiles(),
errors = this.ctx.state.getUnhandledErrors(),
) {
// print failed tests without their errors to keep track of previously failed tests
// this can happen if there are multiple test errors, and user changed a file
// that triggered a rerun of unrelated tests - in that case they want to see
// the error for the test they are currently working on, but still keep track of
// the other failed tests
this.renderer?.update([
...this.failedUnwatchedFiles,
...files,
])

this.stopListRender()
this.ctx.logger.log()
super.onFinished(files, errors)
Expand Down
13 changes: 9 additions & 4 deletions packages/vitest/src/node/reporters/renderers/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ export const hookSpinnerMap = new WeakMap<Task, Map<string, () => string>>()
export const pointer = c.yellow(F_POINTER)
export const skipped = c.dim(c.gray(F_DOWN))

export const benchmarkPass = c.green(F_DOT)
export const testPass = c.green(F_CHECK)
export const taskFail = c.red(F_CROSS)
export const suiteFail = c.red(F_POINTER)
export const pending = c.gray('·')

export function getCols(delta = 0) {
let length = process.stdout?.columns
if (!length || Number.isNaN(length)) {
Expand Down Expand Up @@ -154,10 +160,9 @@ export function getStateSymbol(task: Task) {
}

if (!task.result) {
return c.gray('·')
return pending
}

// pending
if (task.result.state === 'run') {
if (task.type === 'suite') {
return pointer
Expand All @@ -171,11 +176,11 @@ export function getStateSymbol(task: Task) {
}

if (task.result.state === 'pass') {
return task.meta?.benchmark ? c.green(F_DOT) : c.green(F_CHECK)
return task.meta?.benchmark ? benchmarkPass : testPass
}

if (task.result.state === 'fail') {
return task.type === 'suite' ? pointer : c.red(F_CROSS)
return task.type === 'suite' ? suiteFail : taskFail
}

return ' '
Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/types/reporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ export interface Reporter {
onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable<void>
onCollected?: (files?: File[]) => Awaitable<void>
onFinished?: (
files?: File[],
errors?: unknown[],
files: File[],
errors: unknown[],
coverage?: unknown
) => Awaitable<void>
onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable<void>
Expand Down
1 change: 0 additions & 1 deletion test/core/vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ export default defineConfig({
port: 3022,
},
test: {
reporters: ['dot'],
api: {
port: 3023,
},
Expand Down
4 changes: 2 additions & 2 deletions test/reporters/tests/merge-reports.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,13 @@ test('merge reports', async () => {
test 1-2
❯ first.test.ts (2 tests | 1 failed) <time>
❯ first.test.ts > test 1-2
× test 1-2
→ expected 1 to be 2 // Object.is equality
stdout | second.test.ts > test 2-1
test 2-1
❯ second.test.ts (3 tests | 1 failed) <time>
❯ second.test.ts > test 2-1
× test 2-1
→ expected 1 to be 2 // Object.is equality
Test Files 2 failed (2)
Expand Down
6 changes: 6 additions & 0 deletions test/watch/fixtures/single-failed/basic.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { expect, it } from 'vitest';

it('works correctly', () => {
console.log('log basic')
expect(1).toBe(1)
})
6 changes: 6 additions & 0 deletions test/watch/fixtures/single-failed/failed.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { it } from 'vitest';

it('fails', () => {
console.log('log fail')
throw new Error('failed')
})
11 changes: 11 additions & 0 deletions test/watch/fixtures/single-failed/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { defineConfig } from 'vitest/config'

// Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks
process.stdin.isTTY = true
process.stdin.setRawMode = () => process.stdin

export default defineConfig({
test: {
watch: true,
},
})
6 changes: 5 additions & 1 deletion test/watch/fixtures/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { defineConfig } from 'vitest/config'
import { defaultExclude, defineConfig } from 'vitest/config'

// Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks
process.stdin.isTTY = true
Expand All @@ -7,6 +7,10 @@ process.stdin.setRawMode = () => process.stdin
export default defineConfig({
test: {
watch: true,
exclude: [
...defaultExclude,
'**/single-failed/**',
],

// This configuration is edited by tests
reporters: 'verbose',
Expand Down
Loading

0 comments on commit 91ba6f9

Please sign in to comment.