diff --git a/scripts/qa-representative-battles.mjs b/scripts/qa-representative-battles.mjs index 56b6ddb..8a692a1 100644 --- a/scripts/qa-representative-battles.mjs +++ b/scripts/qa-representative-battles.mjs @@ -1,6 +1,7 @@ import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; -import { readFile, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; import { chromium } from 'playwright'; const cliOptions = parseCliOptions(process.argv.slice(2)); @@ -9,6 +10,7 @@ const targetUrl = process.env.QA_URL ?? `http://127.0.0.1:${defaultQaPort}/`; const headless = process.env.QA_HEADLESS !== '0'; const maxRounds = Number(process.env.QA_MAX_ROUNDS ?? 80); const cumulativeMode = process.env.QA_CUMULATIVE === '1'; +const qaReportPath = cliOptions.report ?? process.env.QA_REPORT_PATH ?? 'dist/qa-representative-battles.json'; const campaignStorageKey = 'heros-web:campaign-state'; const campaignSlotStorageKey = 'heros-web:campaign-state:slot-1'; const campaignSnapshotPath = process.env.QA_CAMPAIGN_SNAPSHOT_PATH; @@ -733,6 +735,8 @@ try { ); } + await writeQaReport(qaReportPath, results, failed); + if (failed.length > 0) { console.dir( failed.map((result) => ({ @@ -1570,3 +1574,69 @@ function parseCliOptions(args) { return options; } + +async function writeQaReport(reportPath, results, failed) { + if (!reportPath) { + return; + } + + await mkdir(dirname(reportPath), { recursive: true }); + const report = buildQaReport(results, failed); + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + console.log(`Wrote QA report ${reportPath}`); +} + +function buildQaReport(results, failed) { + const missedObjectives = results.flatMap((result) => + (result.objectiveDetails ?? []) + .filter((objective) => !objective.achieved) + .map((objective) => ({ + battleNo: result.no, + battleId: result.id, + objectiveId: objective.id, + category: objective.category, + summary: objective.summary, + detail: objective.detail, + failureReason: objective.failureReason ?? '' + })) + ); + + return { + app: 'heros_web', + generatedAt: new Date().toISOString(), + set: qaSetName, + targetUrl, + headless, + maxRounds, + cumulativeMode, + requestedBattles: [...requestedBattles], + battleCount: results.length, + summary: { + victories: results.filter((result) => result.outcome === 'victory').length, + failures: failed.length, + missedObjectiveCount: missedObjectives.length, + battlesWithMissedObjectives: [...new Set(missedObjectives.map((objective) => objective.battleNo))] + }, + missedObjectives, + results: results.map((result) => ({ + no: result.no, + id: result.id, + outcome: result.outcome, + finalTurn: result.finalTurn, + alliesAlive: result.alliesAlive, + enemiesAlive: result.enemiesAlive, + itemsUsed: result.itemsUsed, + supports: result.supports, + objectivesAchieved: result.objectivesAchieved, + objectiveCount: result.objectiveCount, + missedObjectives: (result.objectiveDetails ?? []).filter((objective) => !objective.achieved), + protectedStatus: result.protectedStatus, + targetStatus: result.targetStatus, + allyStatus: result.allyStatus, + enemyStatus: result.enemyStatus, + battleLogTail: result.battleLogTail, + campSuppliesUsed: result.campSuppliesUsed, + campaign: result.campaign + })) + }; +} diff --git a/scripts/ship-release.mjs b/scripts/ship-release.mjs index 1c8b164..c2dbac2 100644 --- a/scripts/ship-release.mjs +++ b/scripts/ship-release.mjs @@ -5,11 +5,12 @@ import { join } from 'node:path'; const qaVersion = process.env.PUBLIC_QA_VERSION ?? makeQaVersion(); const releaseCommit = gitValue(['rev-parse', 'HEAD'], 'unknown'); const releaseShortCommit = gitValue(['rev-parse', '--short', 'HEAD'], 'unknown'); +const qaReportFile = 'qa-early-report.json'; assertNoTrackedChanges(); runPnpmScript('verify:local-release'); -runPnpmScript('qa:early'); -writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit); +runPnpmScript('qa:early', { QA_REPORT_PATH: join('dist', qaReportFile) }); +writeReleaseManifest(qaVersion, releaseCommit, releaseShortCommit, qaReportFile); runPnpmScript('deploy:nas:dist'); runPnpmScript('verify:public-deploy', { PUBLIC_QA_VERSION: qaVersion, PUBLIC_QA_COMMIT: releaseCommit }); @@ -35,7 +36,7 @@ function assertNoTrackedChanges() { } } -function writeReleaseManifest(qaVersion, commit, shortCommit) { +function writeReleaseManifest(qaVersion, commit, shortCommit, qaReport) { const manifest = { app: 'heros_web', qaVersion, @@ -43,6 +44,7 @@ function writeReleaseManifest(qaVersion, commit, shortCommit) { shortCommit, branch: gitValue(['branch', '--show-current'], 'unknown'), generatedAt: new Date().toISOString(), + qaReport, verification: ['verify:local-release', 'qa:early', 'verify:public-deploy'] }; diff --git a/scripts/verify-public-deploy.mjs b/scripts/verify-public-deploy.mjs index 4f45e1d..c83024a 100644 --- a/scripts/verify-public-deploy.mjs +++ b/scripts/verify-public-deploy.mjs @@ -117,6 +117,13 @@ try { `Expected deployed release manifest commit ${expectedCommit}: ${JSON.stringify(releaseManifest)}` ); } + if (releaseManifest.qaReport) { + const qaReport = await readJsonAsset(targetUrl, releaseManifest.qaReport); + assert(qaReport.app === 'heros_web', `Expected deployed QA report app id: ${JSON.stringify(qaReport)}`); + assert(Array.isArray(qaReport.results) && qaReport.results.length > 0, `Expected deployed QA report results: ${JSON.stringify(qaReport)}`); + assert(typeof qaReport.generatedAt === 'string' && qaReport.generatedAt.length > 0, `Expected deployed QA report timestamp: ${JSON.stringify(qaReport)}`); + console.log(`Verified QA report ${releaseManifest.qaReport} with ${qaReport.results.length} battles`); + } const relevantLogs = consoleMessages.filter( (message) => message.type === 'error' || (isWarning(message.type) && relevantLogPattern.test(message.text)) @@ -145,10 +152,14 @@ function isWarning(type) { } async function readReleaseManifest(pageUrl) { - const manifestUrl = new URL('release-manifest.json', pageUrl); - manifestUrl.searchParams.set('v', expectedQaVersion ?? String(Date.now())); - const response = await fetch(manifestUrl, { signal: AbortSignal.timeout(90000) }); - assert(response.ok, `Expected deployed release manifest at ${manifestUrl.href}: ${response.status} ${response.statusText}`); + return readJsonAsset(pageUrl, 'release-manifest.json'); +} + +async function readJsonAsset(pageUrl, assetPath) { + const assetUrl = new URL(assetPath, pageUrl); + assetUrl.searchParams.set('v', expectedQaVersion ?? String(Date.now())); + const response = await fetch(assetUrl, { signal: AbortSignal.timeout(90000) }); + assert(response.ok, `Expected deployed JSON asset at ${assetUrl.href}: ${response.status} ${response.statusText}`); return response.json(); }