CS-10684: Fix test run instances being overwritten across factory iterations

packages/software-factory/scripts/lib/factory-implement.ts

@@ -463,6 +463,7 @@ function buildTestRunner(
   toolCallLog: ToolCallEntry[],
   runConfig: TestRunnerConfig,
 ): TestRunner {
+  let lastSequenceNumber = 0;
   return async (): Promise<TestResult> => {
     let wroteTestFiles = toolCallLog.some(
       (entry) =>
@@ -526,8 +527,15 @@ function buildTestRunner(
         realmServerUrl: runConfig.realmServerUrl,
         hostAppUrl: runConfig.hostAppUrl,
         forceNew: true,
+        lastSequenceNumber,
       });
 
+      // Track the sequence number so the next iteration doesn't reuse it
+      // even if the realm search index hasn't caught up yet.
+      if (handle.sequenceNumber != null) {
+        lastSequenceNumber = handle.sequenceNumber;
+      }
+
       let durationMs = Date.now() - start;
       console.error(
         `[factory-implement] Test run complete: status=${handle.status} (${durationMs}ms)`,

packages/software-factory/scripts/lib/factory-tool-builder.ts

@@ -423,6 +423,7 @@ function buildCreateCatalogSpecTool(config: ToolBuilderConfig): FactoryTool {
 }
 
 function buildRunTestsTool(config: ToolBuilderConfig): FactoryTool {
+  let lastSequenceBySlug = new Map<string, number>();
   return {
     name: 'run_tests',
     description: 'Execute QUnit card tests against the target realm',
@@ -449,6 +450,7 @@ function buildRunTestsTool(config: ToolBuilderConfig): FactoryTool {
       required: ['slug'],
     },
     execute: async (args) => {
+      let slug = args.slug as string;
       let targetRealmUrl = config.targetRealmUrl;
       let authorization = resolveAuthForUrl(config, targetRealmUrl);
       let testResultsModuleUrl =
@@ -459,16 +461,23 @@ function buildRunTestsTool(config: ToolBuilderConfig): FactoryTool {
       let result = await executeFn({
         targetRealmUrl,
         testResultsModuleUrl,
-        slug: args.slug as string,
+        slug,
         hostAppUrl: config.hostAppUrl ?? config.realmServerUrl,
         testNames: (args.testNames as string[]) ?? [],
         authorization,
         fetch: config.fetch,
         projectCardUrl: args.projectCardUrl as string | undefined,
         realmServerUrl: config.realmServerUrl,
         forceNew: true,
+        lastSequenceNumber: lastSequenceBySlug.get(slug) ?? 0,
       });
 
+      // Track the sequence number per slug so subsequent calls don't
+      // reuse it even if the realm search index hasn't caught up yet.
+      if (result.sequenceNumber != null) {
+        lastSequenceBySlug.set(slug, result.sequenceNumber);
+      }
+
       return result;
     },
   };

packages/software-factory/scripts/lib/test-run-execution.ts

@@ -45,13 +45,17 @@ export async function resolveTestRun(
   if (resumeResult) {
     return {
       testRunId: resumeResult.testRunId,
+      sequenceNumber: resumeResult.sequenceNumber,
       status: 'running',
       resumed: true,
       pendingTests: resumeResult.pendingTests,
     };
   }
 
-  let sequenceNumber = await getNextSequenceNumber(realmOptions);
+  let sequenceNumber = await getNextSequenceNumber(
+    realmOptions,
+    options.lastSequenceNumber,
+  );
 
   let createResult = await createTestRun(options.slug, options.testNames, {
     ...realmOptions,
@@ -63,6 +67,7 @@ export async function resolveTestRun(
   if (!createResult.created) {
     return {
       testRunId: createResult.testRunId,
+      sequenceNumber,
       status: 'error',
       errorMessage: `Failed to create TestRun: ${createResult.error}`,
       resumed: false,
@@ -71,6 +76,7 @@ export async function resolveTestRun(
 
   return {
     testRunId: createResult.testRunId,
+    sequenceNumber,
     status: 'running',
     resumed: false,
   };
@@ -133,6 +139,7 @@ async function findResumableTestRun(
 
 async function getNextSequenceNumber(
   options: TestRunRealmOptions,
+  minSequenceNumber = 0,
 ): Promise<number> {
   let result = await searchRealm(
     options.testRealmUrl,
@@ -151,7 +158,8 @@ async function getNextSequenceNumber(
         | { attributes?: { sequenceNumber?: number } }
         | undefined)
     : undefined;
-  return (latest?.attributes?.sequenceNumber ?? 0) + 1;
+  let fromIndex = latest?.attributes?.sequenceNumber ?? 0;
+  return Math.max(fromIndex, minSequenceNumber) + 1;
 }
 
 // ---------------------------------------------------------------------------
@@ -418,6 +426,7 @@ export async function executeTestRunFromRealm(
     return resolved;
   }
   let testRunId = resolved.testRunId;
+  let sequenceNumber = resolved.sequenceNumber;
 
   // Step 2: Serve a custom QUnit test page and navigate Playwright to it.
   let start = Date.now();
@@ -513,6 +522,7 @@ export async function executeTestRunFromRealm(
 
     return {
       testRunId,
+      sequenceNumber,
       status: attrs.status,
       ...(attrs.errorMessage ? { errorMessage: attrs.errorMessage } : {}),
       ...(completeResult.error ? { error: completeResult.error } : {}),
@@ -539,7 +549,7 @@ export async function executeTestRunFromRealm(
     } catch {
       // Best-effort
     }
-    return { testRunId, status: 'error', errorMessage };
+    return { testRunId, sequenceNumber, status: 'error', errorMessage };
   } finally {
     if (browser) {
       await browser.close().catch(() => {});

packages/software-factory/scripts/lib/test-run-types.ts

@@ -51,6 +51,8 @@ export interface TestRunHandle {
   testRunId: string;
   status: 'running' | 'passed' | 'failed' | 'error';
   errorMessage?: string;
+  /** The sequence number assigned to this TestRun. */
+  sequenceNumber?: number;
 }
 
 /**
@@ -140,4 +142,11 @@ export interface ExecuteTestRunOptions {
   hostDistDir?: string;
   /** Log browser console output for debugging. */
   debug?: boolean;
+  /**
+   * Floor for the next sequence number. When the realm search index is stale
+   * (hasn't indexed the most recent TestRun yet), getNextSequenceNumber may
+   * return a number that was already used. Passing the last-used sequence
+   * number here guarantees the new TestRun gets at least lastSequenceNumber + 1.
+   */
+  lastSequenceNumber?: number;
 }

packages/software-factory/tests/factory-test-realm.test.ts

@@ -681,6 +681,64 @@ module('factory-test-realm > resolveTestRun', function () {
       'each iteration gets its own TestRun',
     );
   });
+
+  test('lastSequenceNumber prevents reuse when realm index is stale', async function (assert) {
+    // Simulates the real-world bug: the realm search index hasn't indexed
+    // the TestRun created in the previous iteration, so the search returns
+    // stale data. Without lastSequenceNumber, getNextSequenceNumber would
+    // return 1 again and overwrite the first TestRun.
+    let handle1 = await resolveTestRun({
+      ...testRealmOptions,
+      targetRealmUrl: 'https://realms.example.test/user/personal/',
+      slug: 'my-ticket',
+      testNames: ['test A'],
+      forceNew: true,
+      realmServerUrl: 'https://realms.example.test/',
+      hostAppUrl: 'https://realms.example.test/',
+      fetch: buildMockSearchFetch([]),
+    });
+
+    assert.strictEqual(handle1.testRunId, 'Test Runs/my-ticket-1');
+
+    // Second call — search index is STALE (still returns empty), but
+    // lastSequenceNumber=1 prevents reusing sequence 1.
+    let handle2 = await resolveTestRun({
+      ...testRealmOptions,
+      targetRealmUrl: 'https://realms.example.test/user/personal/',
+      slug: 'my-ticket',
+      testNames: ['test A'],
+      forceNew: true,
+      lastSequenceNumber: 1,
+      realmServerUrl: 'https://realms.example.test/',
+      hostAppUrl: 'https://realms.example.test/',
+      fetch: buildMockSearchFetch([]),
+    });
+
+    assert.strictEqual(
+      handle2.testRunId,
+      'Test Runs/my-ticket-2',
+      'uses lastSequenceNumber as floor even when index returns nothing',
+    );
+
+    // Third call — index still stale, lastSequenceNumber=2
+    let handle3 = await resolveTestRun({
+      ...testRealmOptions,
+      targetRealmUrl: 'https://realms.example.test/user/personal/',
+      slug: 'my-ticket',
+      testNames: ['test A'],
+      forceNew: true,
+      lastSequenceNumber: 2,
+      realmServerUrl: 'https://realms.example.test/',
+      hostAppUrl: 'https://realms.example.test/',
+      fetch: buildMockSearchFetch([]),
+    });
+
+    assert.strictEqual(
+      handle3.testRunId,
+      'Test Runs/my-ticket-3',
+      'continues incrementing from lastSequenceNumber floor',
+    );
+  });
 });
 
 // ---------------------------------------------------------------------------