import { spawnSync } from 'node:child_process'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; const cliOptions = parseCliOptions(process.argv.slice(2)); const runs = Number(cliOptions.runs ?? process.env.QA_REPEAT_RUNS ?? 3); const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative'; const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? ''; const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json'; const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs'; const strictMode = String(cliOptions.strict ?? process.env.QA_REPEAT_STRICT ?? '0') === '1'; if (!Number.isInteger(runs) || runs < 1) { throw new Error(`QA repeat runs must be a positive integer: ${runs}`); } await mkdir(runReportDir, { recursive: true }); const runReports = []; for (let index = 1; index <= runs; index += 1) { const runReportPath = join(runReportDir, `${qaSetName}-run-${String(index).padStart(2, '0')}.json`); const args = ['scripts/qa-representative-battles.mjs', `--set=${qaSetName}`, `--report=${runReportPath}`]; if (battles) { args.push(`--battles=${battles}`); } console.log(`\nQA repeat ${index}/${runs}: ${args.join(' ')}`); const result = spawnSync(process.execPath, args, { cwd: process.cwd(), env: { ...process.env, QA_REPORT_PATH: runReportPath }, stdio: 'inherit' }); const exitCode = result.status ?? 1; const report = await readRunReport(runReportPath, index); if (exitCode !== 0) { const modeText = strictMode ? 'strict mode will stop this release' : 'preserving diagnostic report and continuing'; console.warn(`QA repeat run ${index}/${runs} exited ${exitCode}; ${modeText}.`); if (strictMode) { throw new Error(`QA repeat run ${index}/${runs} failed in strict mode`); } } runReports.push({ run: index, path: runReportPath, exitCode, report }); } const aggregate = buildAggregateReport(runReports); await mkdir(dirname(reportPath), { recursive: true }); await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8'); console.log(`\nWrote QA repeat report ${reportPath}`); console.log( `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} processFailures=${aggregate.summary.runsWithProcessFailures} objectiveMisses=${aggregate.summary.totalMissedObjectives}` ); function buildAggregateReport(reports) { const missedObjectiveCounts = new Map(); const battleOutcomes = new Map(); for (const { run, report } of reports) { for (const result of report.results ?? []) { const outcome = result.outcome ?? 'unknown'; const outcomeKey = `${result.no}:${result.id}:${outcome}`; battleOutcomes.set(outcomeKey, { battleNo: result.no, battleId: result.id, outcome, count: (battleOutcomes.get(outcomeKey)?.count ?? 0) + 1 }); for (const missed of result.missedObjectives ?? []) { const key = `${result.no}:${missed.id}`; const current = missedObjectiveCounts.get(key) ?? { battleNo: result.no, battleId: result.id, objectiveId: missed.id, category: missed.category, count: 0, examples: [] }; current.count += 1; current.examples.push({ run, summary: missed.summary, detail: missed.detail, failureReason: missed.failureReason ?? '' }); missedObjectiveCounts.set(key, current); } } } const runsSummary = reports.map(({ run, path, exitCode, report }) => ({ run, path, exitCode, outcome: exitCode === 0 ? 'pass' : 'diagnostic-failure', battleCount: report.battleCount, victories: report.summary?.victories ?? 0, failures: report.summary?.failures ?? 0, missedObjectiveCount: report.summary?.missedObjectiveCount ?? 0, battlesWithMissedObjectives: report.summary?.battlesWithMissedObjectives ?? [] })); const missedObjectives = [...missedObjectiveCounts.values()] .map((entry) => ({ ...entry, missRate: entry.count / reports.length })) .sort((a, b) => b.count - a.count || a.battleNo - b.battleNo || a.objectiveId.localeCompare(b.objectiveId)); return { app: 'heros_web', generatedAt: new Date().toISOString(), set: qaSetName, requestedBattles: battles ? battles.split(',').map((entry) => entry.trim()).filter(Boolean) : [], battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)), summary: { runs: reports.length, runsWithProcessFailures: runsSummary.filter((run) => run.exitCode !== 0).length, runsWithFailures: runsSummary.filter((run) => run.failures > 0).length, runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length, totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0) }, battleOutcomes: [...battleOutcomes.values()].sort((a, b) => a.battleNo - b.battleNo || a.outcome.localeCompare(b.outcome)), missedObjectives, runs: runsSummary }; } async function readRunReport(runReportPath, runIndex) { try { return JSON.parse(await readFile(runReportPath, 'utf8')); } catch (error) { throw new Error(`QA repeat run ${runIndex}/${runs} did not write a readable report at ${runReportPath}: ${error.message}`); } } function parseCliOptions(args) { const options = {}; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (!arg.startsWith('--')) { continue; } const [rawKey, rawValue] = arg.slice(2).split('=', 2); const key = rawKey.trim(); if (!key) { continue; } if (rawValue !== undefined) { options[key] = rawValue; continue; } const next = args[index + 1]; if (next && !next.startsWith('--')) { options[key] = next; index += 1; } else { options[key] = '1'; } } return options; }