Skip to content

Commit

Permalink
fix(runner): make the default value of retry and repeats 0 (#3638)
Browse files Browse the repository at this point in the history
  • Loading branch information
Dunqing authored Jun 22, 2023
1 parent 5c0ac4a commit 6d146d1
Show file tree
Hide file tree
Showing 9 changed files with 43 additions and 33 deletions.
4 changes: 4 additions & 0 deletions docs/api/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@ interface TestOptions {
timeout?: number
/**
* Will retry the test specific number of times if it fails
*
* @default 0
*/
retry?: number
/**
* Will repeat the same test several times even if it fails each time
* If you have "retry" option and it fails, it will use every retry in each cycle
* Useful for debugging random failings
*
* @default 0
*/
repeats?: number
}
Expand Down
15 changes: 7 additions & 8 deletions packages/runner/src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,22 +127,20 @@ export async function runTest(test: Test, runner: VitestRunner) {
test.result = {
state: 'run',
startTime: start,
retryCount: 0,
}
updateTask(test, runner)

setCurrentTest(test)

const repeats = typeof test.repeats === 'number' ? test.repeats : 1

for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
const retry = test.retry || 1

for (let retryCount = 0; retryCount < retry; retryCount++) {
const repeats = test.repeats ?? 0
for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) {
const retry = test.retry ?? 0
for (let retryCount = 0; retryCount <= retry; retryCount++) {
let beforeEachCleanups: HookCleanupCallback[] = []
try {
await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount })

test.result.retryCount = retryCount
test.result.repeatCount = repeatCount

beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
Expand Down Expand Up @@ -188,9 +186,10 @@ export async function runTest(test: Test, runner: VitestRunner) {
if (test.result.state === 'pass')
break

if (retryCount < retry - 1) {
if (retryCount < retry) {
// reset state when retry test
test.result.state = 'run'
test.result.retryCount = (test.result.retryCount ?? 0) + 1
}

// update retry info
Expand Down
5 changes: 2 additions & 3 deletions packages/runner/src/types/tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,15 +166,14 @@ export interface TestOptions {
* Times to retry the test if fails. Useful for making flaky tests more stable.
* When retries is up, the last test error will be thrown.
*
* @default 1
* @default 0
*/
retry?: number
/**
* How many times the test will run.
* Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default.
*
* @default 1
*
* @default 0
*/
repeats?: number
}
Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/node/reporters/renderers/listRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
if (level === 0 && task.type === 'suite' && task.projectName)
prefix += formatProjectName(task.projectName)

if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1)
if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 0)
suffix += c.yellow(` (retry x${task.result.retryCount})`)

if (task.type === 'suite' && !task.meta?.typecheck) {
Expand All @@ -111,7 +111,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
if (task.mode === 'skip' || task.mode === 'todo')
suffix += ` ${c.dim(c.gray('[skipped]'))}`

if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 0)
suffix += c.yellow(` (repeat x${task.result.repeatCount})`)

if (task.result?.duration != null) {
Expand Down
2 changes: 1 addition & 1 deletion test/core/test/propagate-options-nested-suite.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,5 @@ describe(
})
})
},
{ retry: 10 },
{ retry: 9 },
)
34 changes: 21 additions & 13 deletions test/core/test/repeats.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { getCurrentTest } from '@vitest/runner'
import { afterAll, describe, expect, test } from 'vitest'

const testNumbers: number[] = []
Expand All @@ -7,16 +8,16 @@ describe('testing it/test', () => {

test('test 1', () => {
testNumbers.push(1)
}, { repeats: 5 })
}, { repeats: 4 })

test('test 2', () => {
testNumbers.push(2)
}, { repeats: 3 })
}, { repeats: 2 })

test.fails('test 3', () => {
testNumbers.push(3)
expect(testNumbers).toStrictEqual(result)
}, { repeats: 1 })
}, { repeats: 0 })

afterAll(() => {
result.push(3)
Expand All @@ -30,7 +31,7 @@ describe('testing describe', () => {
test('test 1', () => {
describeNumbers.push(1)
})
}, { repeats: 3 })
}, { repeats: 2 })

afterAll(() => {
expect(describeNumbers).toStrictEqual([1, 1, 1])
Expand All @@ -39,14 +40,21 @@ afterAll(() => {
const retryNumbers: number[] = []

describe('testing repeats with retry', () => {
const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
test('test 1', () => {
retryNumbers.push(1)
}, { repeats: 5, retry: 2 })

afterAll(() => {
expect(retryNumbers).toStrictEqual(result)
describe('normal test', () => {
const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
test.fails('test 1', () => {
retryNumbers.push(1)
expect(1).toBe(2)
}, { repeats: 4, retry: 1 })

afterAll(() => {
expect(retryNumbers).toStrictEqual(result)
})
})

test('should not reset retry count', () => {
expect(getCurrentTest()!.result?.retryCount).toBe(3)
}, { repeats: 2, retry: 1 })
})

const nestedDescribeNumbers: number[] = []
Expand All @@ -71,10 +79,10 @@ describe('testing nested describe', () => {
nestedDescribeNumbers.push(4)
})
}, 100)
}, { repeats: 3 })
}, { repeats: 2 })
})

afterAll(() => {
expect(nestedDescribeNumbers).toStrictEqual([1, 1, 2, 2, 3, 3, 3, 4, 4, 4])
})
}, { repeats: 2 })
}, { repeats: 1 })
4 changes: 2 additions & 2 deletions test/core/test/retry-only.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ describe.only('description.only retry', () => {
it('test should not inherit options from the description block if exists', () => {
count5 += 1
expect(count5).toBe(5)
}, { retry: 5 })
}, { retry: 2 })
}, { retry: 4 })
}, { retry: 1 })
4 changes: 2 additions & 2 deletions test/core/test/retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ let count1 = 0
it('retry test', () => {
count1 += 1
expect(count1).toBe(3)
}, { retry: 3 })
}, { retry: 2 })

let count2 = 0
it.fails('retry test fails', () => {
count2 += 1
expect(count2).toBe(3)
}, { retry: 2 })
}, { retry: 1 })

let count3 = 0
it('retry test fails', () => {
Expand Down
4 changes: 2 additions & 2 deletions test/failing/fixtures/expects/soft.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,13 @@ test('retry will passed', () => {
expect.soft(num += 1).toBe(3)
expect.soft(num += 1).toBe(4)
}, {
retry: 2,
retry: 1,
})

num = 0
test('retry will failed', () => {
expect.soft(num += 1).toBe(4)
expect.soft(num += 1).toBe(5)
}, {
retry: 2,
retry: 1,
})

0 comments on commit 6d146d1

Please sign in to comment.