Allow setting retryTimes per test case

src/loader.js

@@ -241,7 +241,12 @@ async function loadTests(config, globPattern) {
             } else if (typeof tc.run === 'function') {
                 // ESM modules are readonly, so we need to create our own writable
                 // object.
-                testCases.push({...tc, name: t.name, fileName: t.fileName});
+                testCases.push({
+                    ...tc,
+                    name: t.name,
+                    fileName: t.fileName,
+                    retryTimes: tc.retryTimes || 0
+                });
             } else {
                 output.log(config, output.color(config, 'red', `No tests found in file "${t.fileName}", skipping.`));
             }

src/runner.js

@@ -271,13 +271,13 @@ function update_results(config, state, task) {
     assert(result);
 
     let status = task.status;
-    if (config.repeatFlaky > 0 && status === 'success' || status === 'error') {
+    if ((config.repeatFlaky > 0 || task.tc.retryTimes > 0) && status === 'success' || status === 'error') {
         const runs = flakyCounts.get(group) || 1;
         // Not flaky if the first run was successful
         if (!(runs === 1 && task.status === 'success')) {
             // If task is failing, but we haven't reached the limit, then
             // we are still trying to determine if the test is flaky.
-            if (runs < config.repeatFlaky && task.status === 'error') {
+            if ((runs < config.repeatFlaky || runs < task.tc.retryTimes) && task.status === 'error') {
                 status = 'todo';
             } else if (task.status === 'success') {
                 // At this point the test was run more than 1 time. This means
@@ -336,7 +336,7 @@ async function run_one(config, state, task) {
     await run_task(config, task);
 
     const repeat = config.repeat || 1;
-    if (count < config.repeatFlaky - 1 && task.status === 'error' && !task.expectedToFail) {
+    if ((count < config.repeatFlaky - 1 || count < task.tc.retryTimes - 1) && task.status === 'error' && !task.expectedToFail) {
         output.logVerbose(config, `[runner] Retrying task for flaky detection. Retry count: ${count + 1} (${task.id})`);
         const tcName = task.tc.name;
         state.tasks.push({

tests/selftest_flaky_local.js

@@ -0,0 +1,68 @@
+const assert = require('assert').strict;
+const runner = require('../src/runner');
+const render = require('../src/render');
+
+/**
+ * @param {import('../src/runner').TaskConfig} config
+ */
+async function run(config) {
+    let output = [];
+    const runnerConfig = {
+        ...config,
+        colors: false,
+        logFunc: (_, message) => output.push(message),
+        quiet: false,
+    };
+
+
+    function createTests() {
+        let i = 0;
+        /** @type {import('../src/runner').TestCase[]} */
+        return [
+            {
+                name: 'foo',
+                retryTimes: 3,
+                run: async () => {
+                    i++;
+                    if (i < 3) {
+                        throw new Error('fail');
+                    }
+                },
+            },
+        ];
+    }
+
+    function assertResult(test_info) {
+        const result = render.craftResults(config, test_info);
+        const formatted = result.tests.map(t => {
+            return {
+                id: t.id,
+                status: t.status,
+                runs: t.taskResults.length,
+            };
+        });
+
+        assert.deepEqual(formatted, [
+            {id: 'foo', status: 'flaky', runs: 3},
+        ]);
+    }
+
+    // Sequential run
+    let tests = createTests();
+    output = [];
+    let result = await runner.run({...runnerConfig, concurrency: 1}, tests);
+    assertResult(result);
+    assert(output[output.length-1].includes('1 flaky (foo)'), 'Summary did not include flaky tests');
+
+    // Parallel run
+    tests = createTests();
+    output = [];
+    result = await runner.run({...runnerConfig, concurrency: 1}, tests);
+    assertResult(result);
+    assert(output[output.length-1].includes('1 flaky (foo)'), 'Summary did not include flaky tests');
+}
+
+module.exports = {
+    run,
+    descripton: 'Rerun task max 3 times',
+};