Add repeated early battle QA reporting

package.json

@@ -34,6 +34,7 @@
     "qa:representative": "node scripts/qa-representative-battles.mjs",
     "qa:smoke": "node scripts/qa-representative-battles.mjs --set=smoke",
     "qa:early": "node scripts/qa-representative-battles.mjs --set=early --battles=1,2,3,4,5,6,7",
+    "qa:early:repeat": "node scripts/qa-repeat.mjs --set=early --battles=1,2,3,4,5,6,7",
     "deploy:nas": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/deploy-nas.ps1 -Build",
     "deploy:nas:dist": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/deploy-nas.ps1",
     "ship:release": "node scripts/ship-release.mjs"

scripts/qa-repeat.mjs

@@ -0,0 +1,156 @@
+import { spawnSync } from 'node:child_process';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+const cliOptions = parseCliOptions(process.argv.slice(2));
+const runs = Number(cliOptions.runs ?? process.env.QA_REPEAT_RUNS ?? 3);
+const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative';
+const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? '';
+const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json';
+const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs';
+
+if (!Number.isInteger(runs) || runs < 1) {
+  throw new Error(`QA repeat runs must be a positive integer: ${runs}`);
+}
+
+await mkdir(runReportDir, { recursive: true });
+
+const runReports = [];
+for (let index = 1; index <= runs; index += 1) {
+  const runReportPath = join(runReportDir, `${qaSetName}-run-${String(index).padStart(2, '0')}.json`);
+  const args = ['scripts/qa-representative-battles.mjs', `--set=${qaSetName}`, `--report=${runReportPath}`];
+  if (battles) {
+    args.push(`--battles=${battles}`);
+  }
+
+  console.log(`\nQA repeat ${index}/${runs}: ${args.join(' ')}`);
+  const result = spawnSync(process.execPath, args, {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      QA_REPORT_PATH: runReportPath
+    },
+    stdio: 'inherit'
+  });
+
+  if (result.status !== 0) {
+    throw new Error(`QA repeat run ${index}/${runs} failed`);
+  }
+
+  runReports.push({
+    run: index,
+    path: runReportPath,
+    report: JSON.parse(await readFile(runReportPath, 'utf8'))
+  });
+}
+
+const aggregate = buildAggregateReport(runReports);
+await mkdir(dirname(reportPath), { recursive: true });
+await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8');
+console.log(`\nWrote QA repeat report ${reportPath}`);
+console.log(
+  `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} objectiveMisses=${aggregate.summary.totalMissedObjectives}`
+);
+
+function buildAggregateReport(reports) {
+  const missedObjectiveCounts = new Map();
+  const battleOutcomes = new Map();
+
+  for (const { run, report } of reports) {
+    for (const result of report.results ?? []) {
+      const outcomeKey = `${result.no}:${result.id}:${result.outcome}`;
+      battleOutcomes.set(outcomeKey, {
+        battleNo: result.no,
+        battleId: result.id,
+        outcome: result.outcome,
+        count: (battleOutcomes.get(outcomeKey)?.count ?? 0) + 1
+      });
+
+      for (const missed of result.missedObjectives ?? []) {
+        const key = `${result.no}:${missed.id}`;
+        const current = missedObjectiveCounts.get(key) ?? {
+          battleNo: result.no,
+          battleId: result.id,
+          objectiveId: missed.id,
+          category: missed.category,
+          count: 0,
+          examples: []
+        };
+        current.count += 1;
+        current.examples.push({
+          run,
+          summary: missed.summary,
+          detail: missed.detail,
+          failureReason: missed.failureReason ?? ''
+        });
+        missedObjectiveCounts.set(key, current);
+      }
+    }
+  }
+
+  const runsSummary = reports.map(({ run, path, report }) => ({
+    run,
+    path,
+    battleCount: report.battleCount,
+    victories: report.summary?.victories ?? 0,
+    failures: report.summary?.failures ?? 0,
+    missedObjectiveCount: report.summary?.missedObjectiveCount ?? 0,
+    battlesWithMissedObjectives: report.summary?.battlesWithMissedObjectives ?? []
+  }));
+
+  const missedObjectives = [...missedObjectiveCounts.values()]
+    .map((entry) => ({
+      ...entry,
+      missRate: entry.count / reports.length
+    }))
+    .sort((a, b) => b.count - a.count || a.battleNo - b.battleNo || a.objectiveId.localeCompare(b.objectiveId));
+
+  return {
+    app: 'heros_web',
+    generatedAt: new Date().toISOString(),
+    set: qaSetName,
+    requestedBattles: battles ? battles.split(',').map((entry) => entry.trim()).filter(Boolean) : [],
+    battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)),
+    summary: {
+      runs: reports.length,
+      runsWithFailures: runsSummary.filter((run) => run.failures > 0).length,
+      runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length,
+      totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0)
+    },
+    battleOutcomes: [...battleOutcomes.values()].sort((a, b) => a.battleNo - b.battleNo || a.outcome.localeCompare(b.outcome)),
+    missedObjectives,
+    runs: runsSummary
+  };
+}
+
+function parseCliOptions(args) {
+  const options = {};
+
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (!arg.startsWith('--')) {
+      continue;
+    }
+
+    const [rawKey, rawValue] = arg.slice(2).split('=', 2);
+    const key = rawKey.trim();
+    if (!key) {
+      continue;
+    }
+
+    if (rawValue !== undefined) {
+      options[key] = rawValue;
+      continue;
+    }
+
+    const next = args[index + 1];
+    if (next && !next.startsWith('--')) {
+      options[key] = next;
+      index += 1;
+    } else {
+      options[key] = '1';
+    }
+  }
+
+  return options;
+}

scripts/ship-release.mjs

@@ -6,11 +6,19 @@ const qaVersion = process.env.PUBLIC_QA_VERSION ?? makeQaVersion();
 const releaseCommit = gitValue(['rev-parse', 'HEAD'], 'unknown');
 const releaseShortCommit = gitValue(['rev-parse', '--short', 'HEAD'], 'unknown');
 const qaReportFile = 'qa-early-report.json';
+const qaRepeatReportFile = 'qa-early-repeat-report.json';
+const qaRepeatRuns = Number(process.env.PUBLIC_QA_REPEAT_RUNS ?? 2);
 
 assertNoTrackedChanges();
 runPnpmScript('verify:local-release');
 runPnpmScript('qa:early', { QA_REPORT_PATH: join('dist', qaReportFile) });
-writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit, qaReportFile);
+if (qaRepeatRuns > 1) {
+  runPnpmScript('qa:early:repeat', {
+    QA_REPEAT_RUNS: String(qaRepeatRuns),
+    QA_REPEAT_REPORT_PATH: join('dist', qaRepeatReportFile)
+  });
+}
+writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit, qaReportFile, qaRepeatRuns > 1 ? qaRepeatReportFile : undefined);
 runPnpmScript('deploy:nas:dist');
 runPnpmScript('verify:public-deploy', { PUBLIC_QA_VERSION: qaVersion, PUBLIC_QA_COMMIT: releaseCommit });
 
@@ -36,7 +44,7 @@ function assertNoTrackedChanges() {
   }
 }
 
-function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport) {
+function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport, qaRepeatReport) {
   const manifest = {
     app: 'heros_web',
     qaVersion,
@@ -45,7 +53,8 @@ function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport) {
     branch: gitValue(['branch', '--show-current'], 'unknown'),
     generatedAt: new Date().toISOString(),
     qaReport,
-    verification: ['verify:local-release', 'qa:early', 'verify:public-deploy']
+    qaRepeatReport,
+    verification: ['verify:local-release', 'qa:early', ...(qaRepeatReport ? ['qa:early:repeat'] : []), 'verify:public-deploy']
   };
 
   mkdirSync('dist', { recursive: true });

scripts/verify-public-deploy.mjs

@@ -124,6 +124,16 @@ try {
     assert(typeof qaReport.generatedAt === 'string' && qaReport.generatedAt.length > 0, `Expected deployed QA report timestamp: ${JSON.stringify(qaReport)}`);
     console.log(`Verified QA report ${releaseManifest.qaReport} with ${qaReport.results.length} battles`);
   }
+  if (releaseManifest.qaRepeatReport) {
+    const qaRepeatReport = await readJsonAsset(targetUrl, releaseManifest.qaRepeatReport);
+    assert(qaRepeatReport.app === 'heros_web', `Expected deployed QA repeat report app id: ${JSON.stringify(qaRepeatReport)}`);
+    assert(Array.isArray(qaRepeatReport.runs) && qaRepeatReport.runs.length > 0, `Expected deployed QA repeat report runs: ${JSON.stringify(qaRepeatReport)}`);
+    assert(
+      typeof qaRepeatReport.summary?.runs === 'number' && qaRepeatReport.summary.runs === qaRepeatReport.runs.length,
+      `Expected deployed QA repeat report run summary: ${JSON.stringify(qaRepeatReport)}`
+    );
+    console.log(`Verified QA repeat report ${releaseManifest.qaRepeatReport} with ${qaRepeatReport.runs.length} runs`);
+  }
 
   const relevantLogs = consoleMessages.filter(
     (message) => message.type === 'error' || (isWarning(message.type) && relevantLogPattern.test(message.text))