Keep repeated QA reports diagnostic

scripts/qa-repeat.mjs

@@ -8,6 +8,7 @@ const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative';
 const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? '';
 const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json';
 const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs';
+const strictMode = String(cliOptions.strict ?? process.env.QA_REPEAT_STRICT ?? '0') === '1';
 
 if (!Number.isInteger(runs) || runs < 1) {
   throw new Error(`QA repeat runs must be a positive integer: ${runs}`);
@@ -33,14 +34,22 @@ for (let index = 1; index <= runs; index += 1) {
     stdio: 'inherit'
   });
 
-  if (result.status !== 0) {
+  const exitCode = result.status ?? 1;
-    throw new Error(`QA repeat run ${index}/${runs} failed`);
+  const report = await readRunReport(runReportPath, index);
+
+  if (exitCode !== 0) {
+    const modeText = strictMode ? 'strict mode will stop this release' : 'preserving diagnostic report and continuing';
+    console.warn(`QA repeat run ${index}/${runs} exited ${exitCode}; ${modeText}.`);
+    if (strictMode) {
+      throw new Error(`QA repeat run ${index}/${runs} failed in strict mode`);
+    }
   }
 
   runReports.push({
     run: index,
     path: runReportPath,
-    report: JSON.parse(await readFile(runReportPath, 'utf8'))
+    exitCode,
+    report
   });
 }
 
@@ -49,7 +58,7 @@ await mkdir(dirname(reportPath), { recursive: true });
 await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8');
 console.log(`\nWrote QA repeat report ${reportPath}`);
 console.log(
-  `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} objectiveMisses=${aggregate.summary.totalMissedObjectives}`
+  `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} processFailures=${aggregate.summary.runsWithProcessFailures} objectiveMisses=${aggregate.summary.totalMissedObjectives}`
 );
 
 function buildAggregateReport(reports) {
@@ -88,9 +97,11 @@ function buildAggregateReport(reports) {
     }
   }
 
-  const runsSummary = reports.map(({ run, path, report }) => ({
+  const runsSummary = reports.map(({ run, path, exitCode, report }) => ({
     run,
     path,
+    exitCode,
+    outcome: exitCode === 0 ? 'pass' : 'diagnostic-failure',
     battleCount: report.battleCount,
     victories: report.summary?.victories ?? 0,
     failures: report.summary?.failures ?? 0,
@@ -113,6 +124,7 @@ function buildAggregateReport(reports) {
     battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)),
     summary: {
       runs: reports.length,
+      runsWithProcessFailures: runsSummary.filter((run) => run.exitCode !== 0).length,
       runsWithFailures: runsSummary.filter((run) => run.failures > 0).length,
       runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length,
       totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0)
@@ -123,6 +135,14 @@ function buildAggregateReport(reports) {
   };
 }
 
+async function readRunReport(runReportPath, runIndex) {
+  try {
+    return JSON.parse(await readFile(runReportPath, 'utf8'));
+  } catch (error) {
+    throw new Error(`QA repeat run ${runIndex}/${runs} did not write a readable report at ${runReportPath}: ${error.message}`);
+  }
+}
+
 function parseCliOptions(args) {
   const options = {};