Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
9b4c308
fix(browser): keep failure screenshot per attempt
hi-ogawa Oct 20, 2026
f38f2a1
Merge remote-tracking branch 'upstream/main' into fix/issue-11614
hi-ogawa Oct 21, 2026
18d5c8d
fix(browser): clear attempt in onAfterRetryTask
hi-ogawa Oct 21, 2026
e01e221
refactor(browser): rename attempts to testAttemptIndexMap
hi-ogawa Oct 21, 2026
7a429de
chore(ui): add todo for per-attempt failure screenshot
hi-ogawa Oct 21, 2026
63d7929
refactor(browser): clarify failure screenshot naming
hi-ogawa Oct 21, 2026
735ee67
refactor(browser): shorten failure screenshot naming comment
hi-ogawa Oct 21, 2026
5faa356
refactor: track error attempts per test in reporter queue
hi-ogawa Oct 21, 2026
fd691b3
refactor: use objects for reporter errors queue
hi-ogawa Oct 21, 2026
4776998
refactor: use array for reporter task attempts
hi-ogawa Oct 21, 2026
3086999
refactor: use for-of for reporter error merge
hi-ogawa Oct 21, 2026
2618fd4
refactor: rename errors queue find callback param
hi-ogawa Oct 21, 2026
8c3746f
refactor: alias reporter errors queue item type
hi-ogawa Oct 21, 2026
51baee4
refactor: flatten reporter errors queue sources
hi-ogawa Oct 21, 2026
0a910d4
chore: comment undefined attempt in errors queue
hi-ogawa Oct 21, 2026
eca3bbf
refactor: drop undefined error from reporter errors queue
hi-ogawa Oct 21, 2026
6dfeea7
refactor: mark reporter error attempt optional
hi-ogawa Oct 21, 2026
ef56eb9
refactor: group attempts by task for failure screenshots
hi-ogawa Oct 21, 2026
72942b1
refactor: require attempt for failure screenshot lookup
hi-ogawa Oct 21, 2026
2b9285c
test: merge retry and repeat failure screenshot tests
hi-ogawa Oct 21, 2026
e9eeace
test: rename failure screenshot test helpers
hi-ogawa Oct 21, 2026
da3bc08
test: include failure screenshot header in snapshot
hi-ogawa Oct 21, 2026
75a5280
test: cover merged errors across attempts
hi-ogawa Oct 21, 2026
7e3d925
test: cover distinct soft errors across attempts
hi-ogawa Oct 21, 2026
67da903
test: comment shared screenshots across soft errors
hi-ogawa Oct 21, 2026
3353f17
feat: show attempts in failure header of retried tests
hi-ogawa Oct 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion packages/browser/src/client/tester/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,12 @@ function createBrowserRunner(
}

private traces = new Map<string, string[]>()
private testAttemptIndexMap = new Map<string, TestTryOptions>()

async onBeforeTryTask(test: Test, options: TestTryOptions) {
await userEvent.cleanup()
super.onBeforeTryTask?.(test, options)
this.testAttemptIndexMap.set(test.id, options)
const trace = this.config.browser.trace
const { retry, repeats } = options
const shouldTrace =
Expand Down Expand Up @@ -120,6 +122,7 @@ function createBrowserRunner(
test: Test,
{ retry, repeats }: { retry: number; repeats: number },
) => {
this.testAttemptIndexMap.delete(test.id)
const hasActiveTraceView = getBrowserState().browserTraceAttempts.has(test.id)
if (hasActiveTraceView) {
const status = test.result?.state
Expand Down Expand Up @@ -192,14 +195,15 @@ function createBrowserRunner(
Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot'
)
) {
const { retry, repeats } = this.testAttemptIndexMap.get(task.id) ?? { retry: 0, repeats: 0 }
const screenshot = await page
.screenshot({
timeout: this.config.browser.providerOptions?.actionTimeout ?? 5_000,
path: resolve(
this.config.attachmentsDir,
'failure-screenshots',
basename(task.file.filepath),
`${task.fullTestName.replace(/\W/g, '-')}.png`,
`${getFailureScreenshotName(task, retry, repeats)}.png`,
),
})
.catch((err) => {
Expand All @@ -209,6 +213,8 @@ function createBrowserRunner(
await recordArtifact(task, {
type: 'internal:failureScreenshot',
attachments: [{ contentType: 'image/png', path: screenshot, originalPath: screenshot }],
retryIndex: retry,
repeatIndex: repeats,
} as const)
}
}
Expand Down Expand Up @@ -456,6 +462,16 @@ async function updateTestFilesLocations(files: File[], sourceMaps: Map<string, a
await Promise.all(promises)
}

function getFailureScreenshotName(test: Test, retry: number, repeats: number) {
const name = test.fullTestName.replace(/\W/g, '-')
const maxRetries = typeof test.retry === 'number' ? test.retry : (test.retry?.count ?? 0)
// add '<name>-<repeat>-<retry>' suffix only when test enables retry or repeat
if (!maxRetries && !test.repeats) {
return name
}
return `${name}-${repeats}-${retry}`
}

function getTraceName(task: Task, retryCount: number, repeatsCount: number) {
const name = getTestName(task, '-').replace(/[^a-z0-9]/gi, '-')
return `${name}-${repeatsCount}-${retryCount}`
Expand Down
3 changes: 2 additions & 1 deletion packages/ui/client/components/FailureScreenshot.vue
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ const { task } = defineProps<{
const showScreenshot = ref(false)
const artifact = computed(() => {
if (task.type === 'test') {
const artifact = task.artifacts.find(
// TODO: show each attempt's screenshot next to that attempt's errors, not only the last one
const artifact = task.artifacts.findLast(
(artifact) => artifact.type === 'internal:failureScreenshot',
)
Expand Down
108 changes: 81 additions & 27 deletions packages/vitest/src/node/reporters/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { ParsedStack, SerializedError } from '@vitest/utils'
import type {
File,
Task,
TaskResultAttempt,
TestAnnotation,
TestBenchmark,
TestBenchmarkTask,
Expand Down Expand Up @@ -1459,24 +1460,30 @@ export abstract class BaseReporter implements Reporter {
}

private printTaskErrors(tasks: Task[], errorDivider: () => void) {
const errorsQueue: [error: TestError | undefined, tests: Task[]][] = []
interface ErrorsQueueItem {
error: TestError
// `undefined` for tasks without attempts (suites, files)
sources: { task: Task; attempt?: TaskResultAttempt }[]
}
const errorsQueue: ErrorsQueueItem[] = []

for (const task of tasks) {
// Merge identical errors
task.result?.errors?.forEach((error) => {
let previous
for (const { error, attempt } of getErrorsWithAttempt(task)) {
let previous: ErrorsQueueItem | undefined

if (error?.stack) {
previous = errorsQueue.find((i) => {
if (i[0]?.stack !== error.stack || i[0]?.diff !== error.diff) {
if (error.stack) {
previous = errorsQueue.find((item) => {
if (item.error.stack !== error.stack || item.error.diff !== error.diff) {
return false
}

const itemTask = item.sources[0].task
const currentProjectName = (task as File)?.projectName || task.file?.projectName || ''
const projectName = (i[1][0] as File)?.projectName || i[1][0].file?.projectName || ''
const projectName = (itemTask as File)?.projectName || itemTask.file?.projectName || ''

const currentAnnotations = task.type === 'test' && task.annotations
const itemAnnotations = i[1][0].type === 'test' && i[1][0].annotations
const itemAnnotations = itemTask.type === 'test' && itemTask.annotations

return (
projectName === currentProjectName && deepEqual(currentAnnotations, itemAnnotations)
Expand All @@ -1485,21 +1492,36 @@ export abstract class BaseReporter implements Reporter {
}

if (previous) {
previous[1].push(task)
previous.sources.push({ task, attempt })
} else {
errorsQueue.push([error, [task]])
errorsQueue.push({ error, sources: [{ task, attempt }] })
}
})
}
}

for (const [error, tasks] of errorsQueue) {
for (const task of tasks) {
for (const { error, sources } of errorsQueue) {
const tasks = sources.map(({ task }) => task)
const attemptsByTask = new Map<Task, Set<TaskResultAttempt | undefined>>()
for (const { task, attempt } of sources) {
attemptsByTask.set(task, (attemptsByTask.get(task) ?? new Set()).add(attempt))
}
const screenshotPaths = [...attemptsByTask].flatMap(([task, attempts]) =>
[...attempts].flatMap((attempt) =>
attempt ? getFailureScreenshotPaths(task, attempt) : [],
),
)
for (const [task, attempts] of attemptsByTask) {
const filepath = (task as File)?.filepath || ''
const projectName = (task as File)?.projectName || task.file?.projectName || ''
const project = this.ctx.projects.find((p) => p.name === projectName)

let name = this.getFullName(task, separator)

const attemptLabel = formatErrorAttempts(task, attempts)
if (attemptLabel) {
name += c.dim(` (${attemptLabel})`)
}

if (filepath) {
name += c.dim(` [ ${this.relative(filepath)} ]`)
}
Expand All @@ -1510,20 +1532,6 @@ export abstract class BaseReporter implements Reporter {
)
}

const screenshotPaths = tasks.reduce<string[]>((paths, t) => {
if (t.type === 'test') {
for (const artifact of t.artifacts) {
if (artifact.type === 'internal:failureScreenshot') {
if (artifact.attachments.length) {
paths.push(artifact.attachments[0].originalPath)
}
}
}
}

return paths
}, [])

this.ctx.logger.printError(error, {
project: this.ctx.getProjectByName(tasks[0].file.projectName || ''),
verbose: this.verbose,
Expand Down Expand Up @@ -1551,6 +1559,52 @@ export abstract class BaseReporter implements Reporter {
}
}

function formatErrorAttempts(
task: Task,
attempts: Set<TaskResultAttempt | undefined>,
): string | undefined {
const allAttempts = task.result?.attempts ?? []
if (allAttempts.length <= 1) {
return
}
const hasRepeats = allAttempts.some((attempt) => attempt.repeatIndex > 0)
const labels = [...attempts].flatMap((attempt) => {
if (!attempt) {
return []
}
return hasRepeats
? `repeat ${attempt.repeatIndex} retry ${attempt.retryIndex}`
: String(attempt.retryIndex)
})
return hasRepeats ? labels.join(', ') : `retry ${labels.join(', ')}`
}

function getErrorsWithAttempt(task: Task): { error: TestError; attempt?: TaskResultAttempt }[] {
const attempts = task.result?.attempts
if (!attempts?.length) {
return (task.result?.errors ?? []).map((error) => ({ error }))
}
return attempts.flatMap((attempt) => (attempt.errors ?? []).map((error) => ({ error, attempt })))
}

function getFailureScreenshotPaths(task: Task, attempt: TaskResultAttempt): string[] {
if (task.type !== 'test') {
return []
}
const paths: string[] = []
for (const artifact of task.artifacts) {
if (
artifact.type === 'internal:failureScreenshot' &&
artifact.attachments.length &&
artifact.retryIndex === attempt.retryIndex &&
artifact.repeatIndex === attempt.repeatIndex
) {
paths.push(artifact.attachments[0].originalPath)
}
}
return paths
}

function deepEqual(a: any, b: any): boolean {
if (a === b) {
return true
Expand Down
4 changes: 4 additions & 0 deletions packages/vitest/src/runtime/runner/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1517,6 +1517,10 @@ interface FailureScreenshotArtifactAttachment extends TestAttachment {
export interface FailureScreenshotArtifact extends TestArtifactBase {
type: 'internal:failureScreenshot'
attachments: [FailureScreenshotArtifactAttachment] | []
/** The zero-based retry index of the attempt that took the screenshot. */
retryIndex?: number
/** The zero-based repeat index of the attempt that took the screenshot. */
repeatIndex?: number
}

/**
Expand Down
82 changes: 82 additions & 0 deletions test/browser/specs/failure-screenshot.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import type { TestFsStructure } from '../../test-utils'
import { readFileSync } from 'node:fs'
import { resolve } from 'pathe'
import { describe, expect, test } from 'vitest'
import { runInlineTests } from '../../test-utils'
import utilsContent from '../fixtures/expect-dom/utils?raw'
Expand Down Expand Up @@ -105,4 +107,84 @@ describe('failure screenshots', () => {
)
})
})

describe('retry and repeats', () => {
function extractFailureLines(stderr: string, root: string) {
return stderr
.split('\n')
.filter((line) => /FAIL|AssertionError|Failure screenshot|\.png/.test(line))
.map((line) => line.replace(root.split('/').at(-1)!, '<root>'))
.join('\n')
}

function readReportedScreenshots(stderr: string) {
const paths = [...new Set(stderr.match(/vitest-test-\S+?\.png/g))]
return paths.map((path) => readFileSync(resolve(path)).toString('base64'))
}

test('saves and reports one screenshot per attempt', async () => {
const { stderr, root } = await runBrowserTests({
[testFilename]: /* ts */ `
import { expect, test } from 'vitest'
import { page } from 'vitest/browser'

let attempt = 0
test('repeated', { repeats: 1, retry: 1 }, async () => {
attempt++
document.body.innerHTML = \`<button>attempt \${attempt}</button>\`
await page.getByText(\`attempt \${attempt}\`).click()
expect(attempt).toBe(10)
})
`,
})

expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(`
" FAIL |chromium| basic.test.ts > repeated (repeat 0 retry 0)
AssertionError: expected 1 to be 10 // Object.is equality
Failure screenshot:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-0.png
FAIL |chromium| basic.test.ts > repeated (repeat 0 retry 1)
AssertionError: expected 2 to be 10 // Object.is equality
Failure screenshot:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-1.png
FAIL |chromium| basic.test.ts > repeated (repeat 1 retry 0)
AssertionError: expected 3 to be 10 // Object.is equality
Failure screenshot:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-0.png
FAIL |chromium| basic.test.ts > repeated (repeat 1 retry 1)
AssertionError: expected 4 to be 10 // Object.is equality
Failure screenshot:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-1.png"
`)
expect(new Set(readReportedScreenshots(stderr)).size).toBe(4)
})

test('lists screenshots of every attempt that produced the error', async () => {
const { stderr, root } = await runBrowserTests({
[testFilename]: /* ts */ `
import { expect, test } from 'vitest'

test('soft', { retry: 1 }, () => {
document.body.innerHTML = '<button>soft</button>'
expect.soft(1).toBe(2)
expect.soft(2).toBe(3)
})
`,
})

// screenshots are taken per attempt, so each soft error lists the same ones
expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(`
" FAIL |chromium| basic.test.ts > soft (retry 0, 1)
AssertionError: expected 1 to be 2 // Object.is equality
Failure screenshots:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-1.png
FAIL |chromium| basic.test.ts > soft (retry 0, 1)
AssertionError: expected 2 to be 3 // Object.is equality
Failure screenshots:
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-1.png"
`)
})
})
})
Loading
Loading