diff --git a/package.json b/package.json index 83d1cd7..eb9f8e6 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "qa:representative": "node scripts/qa-representative-battles.mjs", "qa:smoke": "node scripts/qa-representative-battles.mjs --set=smoke", "qa:early": "node scripts/qa-representative-battles.mjs --set=early --battles=1,2,3,4,5,6,7", + "qa:early:repeat": "node scripts/qa-repeat.mjs --set=early --battles=1,2,3,4,5,6,7", "deploy:nas": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/deploy-nas.ps1 -Build", "deploy:nas:dist": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts/deploy-nas.ps1", "ship:release": "node scripts/ship-release.mjs" diff --git a/scripts/qa-repeat.mjs b/scripts/qa-repeat.mjs new file mode 100644 index 0000000..0c157f8 --- /dev/null +++ b/scripts/qa-repeat.mjs @@ -0,0 +1,156 @@ +import { spawnSync } from 'node:child_process'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +const cliOptions = parseCliOptions(process.argv.slice(2)); +const runs = Number(cliOptions.runs ?? process.env.QA_REPEAT_RUNS ?? 3); +const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative'; +const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? ''; +const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json'; +const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs'; + +if (!Number.isInteger(runs) || runs < 1) { + throw new Error(`QA repeat runs must be a positive integer: ${runs}`); +} + +await mkdir(runReportDir, { recursive: true }); + +const runReports = []; +for (let index = 1; index <= runs; index += 1) { + const runReportPath = join(runReportDir, `${qaSetName}-run-${String(index).padStart(2, '0')}.json`); + const args = ['scripts/qa-representative-battles.mjs', `--set=${qaSetName}`, `--report=${runReportPath}`]; + if (battles) { + args.push(`--battles=${battles}`); + } + + console.log(`\nQA repeat ${index}/${runs}: ${args.join(' ')}`); + const result = spawnSync(process.execPath, args, { + cwd: process.cwd(), + env: { + ...process.env, + QA_REPORT_PATH: runReportPath + }, + stdio: 'inherit' + }); + + if (result.status !== 0) { + throw new Error(`QA repeat run ${index}/${runs} failed`); + } + + runReports.push({ + run: index, + path: runReportPath, + report: JSON.parse(await readFile(runReportPath, 'utf8')) + }); +} + +const aggregate = buildAggregateReport(runReports); +await mkdir(dirname(reportPath), { recursive: true }); +await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8'); +console.log(`\nWrote QA repeat report ${reportPath}`); +console.log( + `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} objectiveMisses=${aggregate.summary.totalMissedObjectives}` +); + +function buildAggregateReport(reports) { + const missedObjectiveCounts = new Map(); + const battleOutcomes = new Map(); + + for (const { run, report } of reports) { + for (const result of report.results ?? []) { + const outcomeKey = `${result.no}:${result.id}:${result.outcome}`; + battleOutcomes.set(outcomeKey, { + battleNo: result.no, + battleId: result.id, + outcome: result.outcome, + count: (battleOutcomes.get(outcomeKey)?.count ?? 0) + 1 + }); + + for (const missed of result.missedObjectives ?? []) { + const key = `${result.no}:${missed.id}`; + const current = missedObjectiveCounts.get(key) ?? { + battleNo: result.no, + battleId: result.id, + objectiveId: missed.id, + category: missed.category, + count: 0, + examples: [] + }; + current.count += 1; + current.examples.push({ + run, + summary: missed.summary, + detail: missed.detail, + failureReason: missed.failureReason ?? '' + }); + missedObjectiveCounts.set(key, current); + } + } + } + + const runsSummary = reports.map(({ run, path, report }) => ({ + run, + path, + battleCount: report.battleCount, + victories: report.summary?.victories ?? 0, + failures: report.summary?.failures ?? 0, + missedObjectiveCount: report.summary?.missedObjectiveCount ?? 0, + battlesWithMissedObjectives: report.summary?.battlesWithMissedObjectives ?? [] + })); + + const missedObjectives = [...missedObjectiveCounts.values()] + .map((entry) => ({ + ...entry, + missRate: entry.count / reports.length + })) + .sort((a, b) => b.count - a.count || a.battleNo - b.battleNo || a.objectiveId.localeCompare(b.objectiveId)); + + return { + app: 'heros_web', + generatedAt: new Date().toISOString(), + set: qaSetName, + requestedBattles: battles ? battles.split(',').map((entry) => entry.trim()).filter(Boolean) : [], + battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)), + summary: { + runs: reports.length, + runsWithFailures: runsSummary.filter((run) => run.failures > 0).length, + runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length, + totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0) + }, + battleOutcomes: [...battleOutcomes.values()].sort((a, b) => a.battleNo - b.battleNo || a.outcome.localeCompare(b.outcome)), + missedObjectives, + runs: runsSummary + }; +} + +function parseCliOptions(args) { + const options = {}; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (!arg.startsWith('--')) { + continue; + } + + const [rawKey, rawValue] = arg.slice(2).split('=', 2); + const key = rawKey.trim(); + if (!key) { + continue; + } + + if (rawValue !== undefined) { + options[key] = rawValue; + continue; + } + + const next = args[index + 1]; + if (next && !next.startsWith('--')) { + options[key] = next; + index += 1; + } else { + options[key] = '1'; + } + } + + return options; +} diff --git a/scripts/ship-release.mjs b/scripts/ship-release.mjs index c2dbac2..d632acb 100644 --- a/scripts/ship-release.mjs +++ b/scripts/ship-release.mjs @@ -6,11 +6,19 @@ const qaVersion = process.env.PUBLIC_QA_VERSION ?? makeQaVersion(); const releaseCommit = gitValue(['rev-parse', 'HEAD'], 'unknown'); const releaseShortCommit = gitValue(['rev-parse', '--short', 'HEAD'], 'unknown'); const qaReportFile = 'qa-early-report.json'; +const qaRepeatReportFile = 'qa-early-repeat-report.json'; +const qaRepeatRuns = Number(process.env.PUBLIC_QA_REPEAT_RUNS ?? 2); assertNoTrackedChanges(); runPnpmScript('verify:local-release'); runPnpmScript('qa:early', { QA_REPORT_PATH: join('dist', qaReportFile) }); -writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit, qaReportFile); +if (qaRepeatRuns > 1) { + runPnpmScript('qa:early:repeat', { + QA_REPEAT_RUNS: String(qaRepeatRuns), + QA_REPEAT_REPORT_PATH: join('dist', qaRepeatReportFile) + }); +} +writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit, qaReportFile, qaRepeatRuns > 1 ? qaRepeatReportFile : undefined); runPnpmScript('deploy:nas:dist'); runPnpmScript('verify:public-deploy', { PUBLIC_QA_VERSION: qaVersion, PUBLIC_QA_COMMIT: releaseCommit }); @@ -36,7 +44,7 @@ function assertNoTrackedChanges() { } } -function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport) { +function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport, qaRepeatReport) { const manifest = { app: 'heros_web', qaVersion, @@ -45,7 +53,8 @@ function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport) { branch: gitValue(['branch', '--show-current'], 'unknown'), generatedAt: new Date().toISOString(), qaReport, - verification: ['verify:local-release', 'qa:early', 'verify:public-deploy'] + qaRepeatReport, + verification: ['verify:local-release', 'qa:early', ...(qaRepeatReport ? ['qa:early:repeat'] : []), 'verify:public-deploy'] }; mkdirSync('dist', { recursive: true }); diff --git a/scripts/verify-public-deploy.mjs b/scripts/verify-public-deploy.mjs index c83024a..b2edc44 100644 --- a/scripts/verify-public-deploy.mjs +++ b/scripts/verify-public-deploy.mjs @@ -124,6 +124,16 @@ try { assert(typeof qaReport.generatedAt === 'string' && qaReport.generatedAt.length > 0, `Expected deployed QA report timestamp: ${JSON.stringify(qaReport)}`); console.log(`Verified QA report ${releaseManifest.qaReport} with ${qaReport.results.length} battles`); } + if (releaseManifest.qaRepeatReport) { + const qaRepeatReport = await readJsonAsset(targetUrl, releaseManifest.qaRepeatReport); + assert(qaRepeatReport.app === 'heros_web', `Expected deployed QA repeat report app id: ${JSON.stringify(qaRepeatReport)}`); + assert(Array.isArray(qaRepeatReport.runs) && qaRepeatReport.runs.length > 0, `Expected deployed QA repeat report runs: ${JSON.stringify(qaRepeatReport)}`); + assert( + typeof qaRepeatReport.summary?.runs === 'number' && qaRepeatReport.summary.runs === qaRepeatReport.runs.length, + `Expected deployed QA repeat report run summary: ${JSON.stringify(qaRepeatReport)}` + ); + console.log(`Verified QA repeat report ${releaseManifest.qaRepeatReport} with ${qaRepeatReport.runs.length} runs`); + } const relevantLogs = consoleMessages.filter( (message) => message.type === 'error' || (isWarning(message.type) && relevantLogPattern.test(message.text))