diff --git a/src/loader.js b/src/loader.js index d7003217..c9698495 100644 --- a/src/loader.js +++ b/src/loader.js @@ -241,7 +241,12 @@ async function loadTests(config, globPattern) { } else if (typeof tc.run === 'function') { // ESM modules are readonly, so we need to create our own writable // object. - testCases.push({...tc, name: t.name, fileName: t.fileName}); + testCases.push({ + ...tc, + name: t.name, + fileName: t.fileName, + retryTimes: tc.retryTimes || 0 + }); } else { output.log(config, output.color(config, 'red', `No tests found in file "${t.fileName}", skipping.`)); } diff --git a/src/runner.js b/src/runner.js index c3dc3ded..b3facf86 100644 --- a/src/runner.js +++ b/src/runner.js @@ -271,13 +271,13 @@ function update_results(config, state, task) { assert(result); let status = task.status; - if (config.repeatFlaky > 0 && status === 'success' || status === 'error') { + if ((config.repeatFlaky > 0 || task.tc.retryTimes > 0) && status === 'success' || status === 'error') { const runs = flakyCounts.get(group) || 1; // Not flaky if the first run was successful if (!(runs === 1 && task.status === 'success')) { // If task is failing, but we haven't reached the limit, then // we are still trying to determine if the test is flaky. - if (runs < config.repeatFlaky && task.status === 'error') { + if ((runs < config.repeatFlaky || runs < task.tc.retryTimes) && task.status === 'error') { status = 'todo'; } else if (task.status === 'success') { // At this point the test was run more than 1 time. This means @@ -336,7 +336,7 @@ async function run_one(config, state, task) { await run_task(config, task); const repeat = config.repeat || 1; - if (count < config.repeatFlaky - 1 && task.status === 'error' && !task.expectedToFail) { + if ((count < config.repeatFlaky - 1 || count < task.tc.retryTimes - 1) && task.status === 'error' && !task.expectedToFail) { output.logVerbose(config, `[runner] Retrying task for flaky detection. Retry count: ${count + 1} (${task.id})`); const tcName = task.tc.name; state.tasks.push({ diff --git a/tests/selftest_flaky_local.js b/tests/selftest_flaky_local.js new file mode 100644 index 00000000..a5ae0dbf --- /dev/null +++ b/tests/selftest_flaky_local.js @@ -0,0 +1,68 @@ +const assert = require('assert').strict; +const runner = require('../src/runner'); +const render = require('../src/render'); + +/** + * @param {import('../src/runner').TaskConfig} config + */ +async function run(config) { + let output = []; + const runnerConfig = { + ...config, + colors: false, + logFunc: (_, message) => output.push(message), + quiet: false, + }; + + + function createTests() { + let i = 0; + /** @type {import('../src/runner').TestCase[]} */ + return [ + { + name: 'foo', + retryTimes: 3, + run: async () => { + i++; + if (i < 3) { + throw new Error('fail'); + } + }, + }, + ]; + } + + function assertResult(test_info) { + const result = render.craftResults(config, test_info); + const formatted = result.tests.map(t => { + return { + id: t.id, + status: t.status, + runs: t.taskResults.length, + }; + }); + + assert.deepEqual(formatted, [ + {id: 'foo', status: 'flaky', runs: 3}, + ]); + } + + // Sequential run + let tests = createTests(); + output = []; + let result = await runner.run({...runnerConfig, concurrency: 1}, tests); + assertResult(result); + assert(output[output.length-1].includes('1 flaky (foo)'), 'Summary did not include flaky tests'); + + // Parallel run + tests = createTests(); + output = []; + result = await runner.run({...runnerConfig, concurrency: 1}, tests); + assertResult(result); + assert(output[output.length-1].includes('1 flaky (foo)'), 'Summary did not include flaky tests'); +} + +module.exports = { + run, + descripton: 'Rerun task max 3 times', +};