import { spawnSync } from 'node:child_process'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; const cliOptions = parseCliOptions(process.argv.slice(2)); const runs = Number(cliOptions.runs ?? process.env.QA_REPEAT_RUNS ?? 3); const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative'; const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? ''; const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json'; const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs'; if (!Number.isInteger(runs) || runs < 1) { throw new Error(`QA repeat runs must be a positive integer: ${runs}`); } await mkdir(runReportDir, { recursive: true }); const runReports = []; for (let index = 1; index <= runs; index += 1) { const runReportPath = join(runReportDir, `${qaSetName}-run-${String(index).padStart(2, '0')}.json`); const args = ['scripts/qa-representative-battles.mjs', `--set=${qaSetName}`, `--report=${runReportPath}`]; if (battles) { args.push(`--battles=${battles}`); } console.log(`\nQA repeat ${index}/${runs}: ${args.join(' ')}`); const result = spawnSync(process.execPath, args, { cwd: process.cwd(), env: { ...process.env, QA_REPORT_PATH: runReportPath }, stdio: 'inherit' }); if (result.status !== 0) { throw new Error(`QA repeat run ${index}/${runs} failed`); } runReports.push({ run: index, path: runReportPath, report: JSON.parse(await readFile(runReportPath, 'utf8')) }); } const aggregate = buildAggregateReport(runReports); await mkdir(dirname(reportPath), { recursive: true }); await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8'); console.log(`\nWrote QA repeat report ${reportPath}`); console.log( `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} objectiveMisses=${aggregate.summary.totalMissedObjectives}` ); function buildAggregateReport(reports) { const missedObjectiveCounts = new Map(); const battleOutcomes = new Map(); for (const { run, report } of reports) { for (const result of report.results ?? []) { const outcomeKey = `${result.no}:${result.id}:${result.outcome}`; battleOutcomes.set(outcomeKey, { battleNo: result.no, battleId: result.id, outcome: result.outcome, count: (battleOutcomes.get(outcomeKey)?.count ?? 0) + 1 }); for (const missed of result.missedObjectives ?? []) { const key = `${result.no}:${missed.id}`; const current = missedObjectiveCounts.get(key) ?? { battleNo: result.no, battleId: result.id, objectiveId: missed.id, category: missed.category, count: 0, examples: [] }; current.count += 1; current.examples.push({ run, summary: missed.summary, detail: missed.detail, failureReason: missed.failureReason ?? '' }); missedObjectiveCounts.set(key, current); } } } const runsSummary = reports.map(({ run, path, report }) => ({ run, path, battleCount: report.battleCount, victories: report.summary?.victories ?? 0, failures: report.summary?.failures ?? 0, missedObjectiveCount: report.summary?.missedObjectiveCount ?? 0, battlesWithMissedObjectives: report.summary?.battlesWithMissedObjectives ?? [] })); const missedObjectives = [...missedObjectiveCounts.values()] .map((entry) => ({ ...entry, missRate: entry.count / reports.length })) .sort((a, b) => b.count - a.count || a.battleNo - b.battleNo || a.objectiveId.localeCompare(b.objectiveId)); return { app: 'heros_web', generatedAt: new Date().toISOString(), set: qaSetName, requestedBattles: battles ? battles.split(',').map((entry) => entry.trim()).filter(Boolean) : [], battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)), summary: { runs: reports.length, runsWithFailures: runsSummary.filter((run) => run.failures > 0).length, runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length, totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0) }, battleOutcomes: [...battleOutcomes.values()].sort((a, b) => a.battleNo - b.battleNo || a.outcome.localeCompare(b.outcome)), missedObjectives, runs: runsSummary }; } function parseCliOptions(args) { const options = {}; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (!arg.startsWith('--')) { continue; } const [rawKey, rawValue] = arg.slice(2).split('=', 2); const key = rawKey.trim(); if (!key) { continue; } if (rawValue !== undefined) { options[key] = rawValue; continue; } const next = args[index + 1]; if (next && !next.startsWith('--')) { options[key] = next; index += 1; } else { options[key] = '1'; } } return options; }