Stabilize scratch iteration 005 aggregate audit

src/Web/StellaOps.Web/scripts/live-full-core-audit.mjs

@@ -135,6 +135,12 @@ const arrayFailureKeys = new Set([
   'warnings',
 ]);
 
+const runtimeOnlyFailurePaths = new Set([
+  'runtimeIssueCount',
+  'runtimeIssues',
+  'runtimeErrors',
+]);
+
 function collectFailureSignals(value) {
   const signals = [];
 
@@ -170,6 +176,22 @@ function collectFailureSignals(value) {
   return signals;
 }
 
+function isRuntimeOnlyFailure(execution, report, failureSignals) {
+  if ((execution.exitCode ?? 1) === 0 && failureSignals.length === 0 && !report.reportReadFailed) {
+    return false;
+  }
+
+  if (report.reportReadFailed || failureSignals.length === 0) {
+    return false;
+  }
+
+  return failureSignals.every((signal) => runtimeOnlyFailurePaths.has(signal.path));
+}
+
+function wait(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 async function readReport(reportPath) {
   try {
     const content = await readFile(reportPath, 'utf8');
@@ -211,14 +233,52 @@ async function main() {
     baseUrl: process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local',
     suiteCount: suites.length,
     suites: [],
+    retriedSuiteCount: 0,
+    stabilizedAfterRetryCount: 0,
   };
 
   for (const suite of suites) {
     process.stdout.write(`[live-full-core-audit] START ${suite.name}\n`);
-    const execution = await runSuite(suite);
-    const report = await readReport(suite.reportPath);
-    const failureSignals = collectFailureSignals(report);
+    let execution = await runSuite(suite);
+    let report = await readReport(suite.reportPath);
+    let failureSignals = collectFailureSignals(report);
+    let retry = null;
+
+    if (isRuntimeOnlyFailure(execution, report, failureSignals)) {
+      summary.retriedSuiteCount += 1;
+      retry = {
+        reason: 'runtime-only-first-pass-failure',
+        firstAttempt: {
+          exitCode: execution.exitCode,
+          signal: execution.signal,
+          durationMs: execution.durationMs,
+          failureSignals,
+          report,
+        },
+      };
+
+      process.stdout.write(
+        `[live-full-core-audit] RETRY ${suite.name} reason=runtime-only-first-pass-failure\n`,
+      );
+      await wait(2_000);
+
+      execution = await runSuite(suite);
+      report = await readReport(suite.reportPath);
+      failureSignals = collectFailureSignals(report);
+      retry.secondAttempt = {
+        exitCode: execution.exitCode,
+        signal: execution.signal,
+        durationMs: execution.durationMs,
+        failureSignals,
+      };
+    }
+
     const ok = execution.exitCode === 0 && failureSignals.length === 0 && !report.reportReadFailed;
+    const stabilizedAfterRetry = Boolean(retry) && ok;
+
+    if (stabilizedAfterRetry) {
+      summary.stabilizedAfterRetryCount += 1;
+    }
 
     const result = {
       ...execution,
@@ -226,12 +286,16 @@ async function main() {
       ok,
       failureSignals,
       report,
+      retried: Boolean(retry),
+      stabilizedAfterRetry,
+      retry,
     };
 
     summary.suites.push(result);
     process.stdout.write(
       `[live-full-core-audit] DONE ${suite.name} ok=${ok} exitCode=${execution.exitCode ?? 'null'} ` +
-      `signals=${failureSignals.length} durationMs=${execution.durationMs}\n`,
+      `signals=${failureSignals.length} durationMs=${execution.durationMs}` +
+      `${stabilizedAfterRetry ? ' stabilizedAfterRetry=true' : ''}\n`,
     );
   }
 
@@ -245,6 +309,8 @@ async function main() {
       signal: suite.signal,
       failureSignals: suite.failureSignals,
       reportPath: suite.reportPath,
+      retried: suite.retried,
+      stabilizedAfterRetry: suite.stabilizedAfterRetry,
     }));
 
   await writeFile(resultPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');