diff --git a/scripts/qa-repeat.mjs b/scripts/qa-repeat.mjs index 0c157f8..9f68bfe 100644 --- a/scripts/qa-repeat.mjs +++ b/scripts/qa-repeat.mjs @@ -8,6 +8,7 @@ const qaSetName = cliOptions.set ?? process.env.QA_SET ?? 'representative'; const battles = cliOptions.battles ?? process.env.QA_BATTLES ?? ''; const reportPath = cliOptions.report ?? process.env.QA_REPEAT_REPORT_PATH ?? 'dist/qa-repeat-report.json'; const runReportDir = cliOptions.runReportDir ?? process.env.QA_REPEAT_RUN_REPORT_DIR ?? 'dist/qa-repeat-runs'; +const strictMode = String(cliOptions.strict ?? process.env.QA_REPEAT_STRICT ?? '0') === '1'; if (!Number.isInteger(runs) || runs < 1) { throw new Error(`QA repeat runs must be a positive integer: ${runs}`); @@ -33,14 +34,22 @@ for (let index = 1; index <= runs; index += 1) { stdio: 'inherit' }); - if (result.status !== 0) { - throw new Error(`QA repeat run ${index}/${runs} failed`); + const exitCode = result.status ?? 1; + const report = await readRunReport(runReportPath, index); + + if (exitCode !== 0) { + const modeText = strictMode ? 'strict mode will stop this release' : 'preserving diagnostic report and continuing'; + console.warn(`QA repeat run ${index}/${runs} exited ${exitCode}; ${modeText}.`); + if (strictMode) { + throw new Error(`QA repeat run ${index}/${runs} failed in strict mode`); + } } runReports.push({ run: index, path: runReportPath, - report: JSON.parse(await readFile(runReportPath, 'utf8')) + exitCode, + report }); } @@ -49,7 +58,7 @@ await mkdir(dirname(reportPath), { recursive: true }); await writeFile(reportPath, `${JSON.stringify(aggregate, null, 2)}\n`, 'utf8'); console.log(`\nWrote QA repeat report ${reportPath}`); console.log( - `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} objectiveMisses=${aggregate.summary.totalMissedObjectives}` + `QA repeat summary: runs=${aggregate.runs.length} battles=${aggregate.battleCount} processFailures=${aggregate.summary.runsWithProcessFailures} objectiveMisses=${aggregate.summary.totalMissedObjectives}` ); function buildAggregateReport(reports) { @@ -88,9 +97,11 @@ function buildAggregateReport(reports) { } } - const runsSummary = reports.map(({ run, path, report }) => ({ + const runsSummary = reports.map(({ run, path, exitCode, report }) => ({ run, path, + exitCode, + outcome: exitCode === 0 ? 'pass' : 'diagnostic-failure', battleCount: report.battleCount, victories: report.summary?.victories ?? 0, failures: report.summary?.failures ?? 0, @@ -113,6 +124,7 @@ function buildAggregateReport(reports) { battleCount: Math.max(...reports.map(({ report }) => report.battleCount ?? 0)), summary: { runs: reports.length, + runsWithProcessFailures: runsSummary.filter((run) => run.exitCode !== 0).length, runsWithFailures: runsSummary.filter((run) => run.failures > 0).length, runsWithMissedObjectives: runsSummary.filter((run) => run.missedObjectiveCount > 0).length, totalMissedObjectives: runsSummary.reduce((total, run) => total + run.missedObjectiveCount, 0) @@ -123,6 +135,14 @@ function buildAggregateReport(reports) { }; } +async function readRunReport(runReportPath, runIndex) { + try { + return JSON.parse(await readFile(runReportPath, 'utf8')); + } catch (error) { + throw new Error(`QA repeat run ${runIndex}/${runs} did not write a readable report at ${runReportPath}: ${error.message}`); + } +} + function parseCliOptions(args) { const options = {};