import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve } from 'node:path'; import { campaignFlowSegmentSummary, readCampaignFlowSegments } from './campaign-flow-segments.mjs'; const allowedCliOptions = new Set([ 'segment', 'from', 'to', 'resume', 'output-dir', 'checkpoint-dir', 'screenshots', 'heartbeat-ms', 'segment-timeout-ms', 'list-segments' ]); const booleanCliOptions = new Set(['resume', 'list-segments']); const cliOptions = parseCliOptions(process.argv.slice(2)); if (cliOptions.segment && (cliOptions.from || cliOptions.to)) { throw new Error('--segment cannot be combined with --from or --to.'); } const { segments, manifestHash } = await readCampaignFlowSegments(); const selectedSegments = selectSegments(segments, cliOptions); const targetUrl = process.env.VERIFY_URL ?? 'http://127.0.0.1:41743/'; const outputDirectory = resolve(cliOptions['output-dir'] ?? process.env.VERIFY_FLOW_OUTPUT_DIR ?? 'dist/verify-flow'); const checkpointDirectory = resolve( cliOptions['checkpoint-dir'] ?? process.env.VERIFY_FLOW_CHECKPOINT_DIR ?? `${outputDirectory}/checkpoints` ); const resume = booleanOption(cliOptions.resume ?? process.env.VERIFY_FLOW_RESUME, 'resume'); const screenshots = cliOptions.screenshots ?? process.env.VERIFY_FLOW_SCREENSHOTS ?? 'failure'; const heartbeatMs = positiveNumber( cliOptions['heartbeat-ms'] ?? process.env.VERIFY_FLOW_HEARTBEAT_MS ?? '15000', 'heartbeat-ms' ); const segmentTimeoutMs = positiveNumber( cliOptions['segment-timeout-ms'] ?? process.env.VERIFY_FLOW_SEGMENT_TIMEOUT_MS ?? '720000', 'segment-timeout-ms' ); if (booleanOption(cliOptions['list-segments'], 'list-segments')) { for (const segment of segments) { console.log(campaignFlowSegmentSummary(segment)); } process.exit(0); } if (!['all', 'failure', 'none'].includes(screenshots)) { throw new Error(`Unknown screenshot mode "${screenshots}". Use all, failure, or none.`); } assertSafeOutputDirectory(outputDirectory); assertNestedDirectory(outputDirectory, checkpointDirectory, 'Flow checkpoint directory'); if (!resume) { if (selectedSegments[0]?.index !== 0) { throw new Error('A run starting after the first segment requires --resume=1 and an existing previous checkpoint.'); } await rm(outputDirectory, { recursive: true, force: true }); } await mkdir(checkpointDirectory, { recursive: true }); const runStartedAt = Date.now(); const summaryPath = resolve(outputDirectory, 'summary.json'); const results = []; let serverProcess; await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments); console.log(`[verify-flow] manifest ${manifestHash.slice(0, 12)} · ${selectedSegments.length}/${segments.length} segments`); try { serverProcess = await ensureLocalServer(targetUrl); for (const segment of selectedSegments) { const previousSegment = segments[segment.index - 1]; if (previousSegment) { const previousCheckpoint = resolve(checkpointDirectory, `${previousSegment.id}.json`); if (!existsSync(previousCheckpoint)) { throw new Error(`Missing prerequisite checkpoint for ${segment.id}: ${previousCheckpoint}`); } await validateSegmentCheckpoint(previousSegment, checkpointDirectory, manifestHash, segments); } const startedAt = Date.now(); await rm(resolve(outputDirectory, segment.id), { recursive: true, force: true }); console.log(`[verify-flow] segment start · ${campaignFlowSegmentSummary(segment)}`); const exitCode = await runSegmentProcess(segment, { checkpointDirectory, heartbeatMs, manifestHash, outputDirectory, screenshots, segmentTimeoutMs, targetUrl }); const result = { id: segment.id, label: segment.label, firstBattleOrdinal: segment.firstBattleOrdinal, lastBattleOrdinal: segment.lastBattleOrdinal, durationMs: Date.now() - startedAt, exitCode, status: exitCode === 0 ? 'passed' : 'failed' }; results.push(result); if (exitCode !== 0) { throw new Error(`Flow segment ${segment.id} failed with exit code ${exitCode}.`); } try { await validateSegmentArtifacts(segment, { checkpointDirectory, manifestHash, outputDirectory, segments }); result.artifactValidation = 'passed'; } catch (error) { result.status = 'failed'; result.artifactValidation = 'failed'; throw error; } await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments); console.log(`[verify-flow] segment pass · ${segment.id} · ${formatDuration(result.durationMs)}`); } console.log(`[verify-flow] all selected segments passed · ${formatDuration(Date.now() - runStartedAt)}`); } catch (error) { await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments, error); throw error; } finally { await terminateProcessTree(serverProcess); } async function runSegmentProcess(segment, options) { const progressPath = resolve(options.outputDirectory, segment.id, 'progress.jsonl'); await mkdir(resolve(options.outputDirectory, segment.id), { recursive: true }); const args = [ 'scripts/verify-flow.mjs', `--segment=${segment.id}`, `--checkpoint-dir=${options.checkpointDirectory}`, `--progress=${progressPath}`, `--screenshots=${options.screenshots}`, `--heartbeat-ms=${options.heartbeatMs}`, `--manifest-hash=${options.manifestHash}` ]; return new Promise((resolveExit, reject) => { const child = spawn(process.execPath, args, { cwd: process.cwd(), env: { ...process.env, VERIFY_URL: options.targetUrl }, stdio: 'inherit', windowsHide: true }); let timedOut = false; let settled = false; const settleExit = (code) => { if (settled) { return; } settled = true; clearTimeout(timeout); resolveExit(code); }; const settleError = (error) => { if (settled) { return; } settled = true; clearTimeout(timeout); reject(error); }; const timeout = setTimeout(() => { timedOut = true; console.error(`[verify-flow] segment timeout · ${segment.id} · ${formatDuration(options.segmentTimeoutMs)}`); void (async () => { await terminateProcessTree(child); await writeTimeoutFailure(segment, options); settleExit(124); })().catch(settleError); }, options.segmentTimeoutMs); child.once('error', (error) => { if (!timedOut) { settleError(error); } }); child.once('exit', (code, signal) => { if (timedOut) { return; } if (signal) { console.error(`[verify-flow] segment ${segment.id} ended by signal ${signal}.`); } settleExit(code ?? 1); }); }); } async function writeSummary(path, hash, segmentResults, startedAt, selectedSegments, manifestSegments, error) { const selectedSegmentIds = selectedSegments.map(({ id }) => id); const fullManifestSelected = selectedSegmentIds.length === manifestSegments.length && selectedSegmentIds.every((id, index) => id === manifestSegments[index]?.id); const allSelectedFinished = segmentResults.length === selectedSegments.length; const allSelectedPassed = allSelectedFinished && segmentResults.every(({ status }) => status === 'passed'); const status = error ? 'failed' : !allSelectedFinished ? 'running' : allSelectedPassed && fullManifestSelected ? 'full-manifest-passed' : allSelectedPassed ? 'partial-passed' : 'failed'; await writeTextAtomically(path, `${JSON.stringify({ version: 1, manifestHash: hash, status, startedAt: new Date(startedAt).toISOString(), updatedAt: new Date().toISOString(), durationMs: Date.now() - startedAt, manifestSegmentCount: manifestSegments.length, selectedSegmentCount: selectedSegments.length, selectedSegmentIds, fullManifestSelected, finishedSegments: segmentResults.length, passedSegments: segmentResults.filter(({ status: resultStatus }) => resultStatus === 'passed').length, results: segmentResults, error: error instanceof Error ? { name: error.name, message: error.message } : error ? { message: String(error) } : null }, null, 2)}\n`); } async function validateSegmentArtifacts(segment, options) { await validateSegmentCheckpoint(segment, options.checkpointDirectory, options.manifestHash, options.segments); const progressPath = resolve(options.outputDirectory, segment.id, 'progress.jsonl'); if (!existsSync(progressPath)) { throw new Error(`Flow segment ${segment.id} exited successfully without a progress log: ${progressPath}`); } const progressSource = await readFile(progressPath, 'utf8'); const events = progressSource .split(/\r?\n/) .filter(Boolean) .map((line, index) => { try { return JSON.parse(line); } catch (error) { throw new Error( `Flow segment ${segment.id} has invalid progress JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}` ); } }); const startEvent = events.find(({ type }) => type === 'start'); const checkpointEvent = events.findLast(({ type }) => type === 'checkpoint'); const passEvent = events.findLast(({ type }) => type === 'segment-pass'); if ( startEvent?.segment !== segment.id || startEvent.manifestHash !== options.manifestHash || checkpointEvent?.segment !== segment.id || checkpointEvent.campaignStep !== segment.checkpointStep || passEvent?.segment !== segment.id ) { throw new Error(`Flow segment ${segment.id} did not emit the required start/checkpoint/pass progress contract.`); } } async function validateSegmentCheckpoint(segment, checkpointDirectory, hash, allSegments) { const checkpointPath = resolve(checkpointDirectory, `${segment.id}.json`); if (!existsSync(checkpointPath)) { throw new Error(`Flow segment ${segment.id} exited successfully without a checkpoint: ${checkpointPath}`); } let checkpoint; try { checkpoint = JSON.parse(await readFile(checkpointPath, 'utf8')); } catch (error) { throw new Error( `Flow checkpoint ${segment.id} is not valid JSON: ${error instanceof Error ? error.message : String(error)}` ); } if ( checkpoint?.version !== 1 || checkpoint.manifestHash !== hash || checkpoint.segment?.id !== segment.id || checkpoint.segment?.lastBattleOrdinal !== segment.lastBattleOrdinal || checkpoint.segment?.lastBattleId !== segment.lastBattleId || checkpoint.segment?.checkpointStep !== segment.checkpointStep || !checkpoint.localStorage || typeof checkpoint.localStorage !== 'object' ) { throw new Error(`Flow checkpoint ${segment.id} does not match the active manifest.`); } const globalRaw = checkpoint.localStorage['heros-web:campaign-state']; const slotRaw = checkpoint.localStorage['heros-web:campaign-state:slot-1']; if (!globalRaw || !slotRaw || globalRaw !== slotRaw) { throw new Error(`Flow checkpoint ${segment.id} must contain identical global and slot-1 campaign saves.`); } let campaign; try { campaign = JSON.parse(slotRaw); } catch (error) { throw new Error( `Flow checkpoint ${segment.id} campaign save is not valid JSON: ${error instanceof Error ? error.message : String(error)}` ); } const expectedBattleIds = allSegments .slice(0, segment.index + 1) .flatMap((candidate) => candidate.battleIds); const completedBattleIds = Object.keys(campaign?.battleHistory ?? {}); const exactBattlePrefix = expectedBattleIds.length === completedBattleIds.length && expectedBattleIds.every((battleId, index) => completedBattleIds[index] === battleId); const allBattlesWon = expectedBattleIds.every( (battleId) => campaign?.battleHistory?.[battleId]?.outcome === 'victory' ); if ( campaign?.version !== 1 || campaign?.activeSaveSlot !== 1 || campaign?.step !== segment.checkpointStep || campaign?.latestBattleId !== segment.lastBattleId || !exactBattlePrefix || !allBattlesWon ) { throw new Error(`Flow checkpoint ${segment.id} campaign position or battle history is invalid.`); } } async function writeTimeoutFailure(segment, options) { const segmentDirectory = resolve(options.outputDirectory, segment.id); const progressPath = resolve(segmentDirectory, 'progress.jsonl'); let lastProgressEvent = null; try { const lines = (await readFile(progressPath, 'utf8')).split(/\r?\n/).filter(Boolean); if (lines.length > 0) { lastProgressEvent = JSON.parse(lines.at(-1)); } } catch { lastProgressEvent = null; } await writeTextAtomically(resolve(segmentDirectory, 'timeout-state.json'), `${JSON.stringify({ version: 1, manifestHash: options.manifestHash, timestamp: new Date().toISOString(), segment: segment.id, timeoutMs: options.segmentTimeoutMs, lastProgressEvent }, null, 2)}\n`); } async function writeTextAtomically(path, contents) { await mkdir(dirname(path), { recursive: true }); const temporaryPath = `${path}.${process.pid}.${Date.now()}.tmp`; try { await writeFile(temporaryPath, contents, 'utf8'); await rename(temporaryPath, path); } catch (error) { await rm(temporaryPath, { force: true }).catch(() => {}); throw error; } } function selectSegments(allSegments, options) { const onlyId = options.segment; const fromId = options.from; const toId = options.to; if (onlyId) { const segment = allSegments.find((candidate) => candidate.id === onlyId); if (!segment) { throw new Error(`Unknown flow segment "${onlyId}". Use one of: ${allSegments.map(({ id }) => id).join(', ')}`); } return [segment]; } const fromIndex = fromId ? allSegments.findIndex(({ id }) => id === fromId) : 0; const toIndex = toId ? allSegments.findIndex(({ id }) => id === toId) : allSegments.length - 1; if (fromIndex < 0 || toIndex < 0 || fromIndex > toIndex) { throw new Error(`Invalid segment range: ${fromId ?? allSegments[0].id}..${toId ?? allSegments.at(-1).id}`); } return allSegments.slice(fromIndex, toIndex + 1); } async function ensureLocalServer(url) { if (await canReach(url)) { return undefined; } const parsed = new URL(url); if (!['localhost', '127.0.0.1', '0.0.0.0'].includes(parsed.hostname)) { throw new Error(`No server responded at ${url}`); } const child = spawn( process.execPath, ['node_modules/vite/bin/vite.js', '--host', '127.0.0.1', '--port', parsed.port || '41743', '--strictPort'], { cwd: process.cwd(), env: process.env, stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true } ); let startupError; let stderr = ''; child.once('error', (error) => { startupError = error; }); child.stderr?.on('data', (chunk) => { stderr = `${stderr}${chunk}`.slice(-6000); }); for (let attempt = 0; attempt < 120; attempt += 1) { if (startupError) { throw new Error(`Could not start the segmented flow server: ${startupError.message}`); } if (child.exitCode !== null || child.signalCode !== null) { throw new Error( `Segmented flow server exited before becoming ready (${child.exitCode ?? child.signalCode}).${stderr ? `\n${stderr.trim()}` : ''}` ); } if (await canReach(url)) { return child; } await delay(500); } await terminateProcessTree(child); throw new Error(`Could not start the segmented flow server at ${url}`); } async function terminateProcessTree(child) { if (!child || child.exitCode !== null || child.signalCode !== null) { return; } if (process.platform === 'win32' && child.pid) { await runProcessTreeTerminator(child.pid); } else { try { child.kill('SIGTERM'); } catch { // The process may have exited between the state check and the signal. } } await waitForProcessExit(child, 3000); if (child.exitCode === null && child.signalCode === null) { try { child.kill('SIGKILL'); } catch { // The process may have exited between the state check and the signal. } await waitForProcessExit(child, 2000); } if (child.exitCode === null && child.signalCode === null) { throw new Error(`Could not terminate process tree for PID ${child.pid ?? 'unknown'}.`); } } async function runProcessTreeTerminator(pid) { await new Promise((resolveTermination) => { const terminator = spawn('taskkill.exe', ['/PID', String(pid), '/T', '/F'], { stdio: 'ignore', windowsHide: true }); let settled = false; const finish = () => { if (settled) { return; } settled = true; clearTimeout(timeout); resolveTermination(); }; const timeout = setTimeout(() => { terminator.kill(); finish(); }, 5000); terminator.once('error', finish); terminator.once('exit', finish); }); } async function waitForProcessExit(child, timeoutMs) { if (!child || child.exitCode !== null || child.signalCode !== null) { return; } await new Promise((resolveExit) => { let settled = false; const finish = () => { if (settled) { return; } settled = true; clearTimeout(timeout); child.off('exit', finish); child.off('error', finish); resolveExit(); }; const timeout = setTimeout(finish, timeoutMs); child.once('exit', finish); child.once('error', finish); }); } async function canReach(url) { try { const response = await fetch(url, { signal: AbortSignal.timeout(1000) }); return response.ok; } catch { return false; } } function parseCliOptions(args) { const options = {}; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (!arg.startsWith('--')) { throw new Error(`Unexpected flow runner argument: ${arg}`); } const option = arg.slice(2); const equalsIndex = option.indexOf('='); const key = equalsIndex >= 0 ? option.slice(0, equalsIndex) : option; const inlineValue = equalsIndex >= 0 ? option.slice(equalsIndex + 1) : undefined; if (!allowedCliOptions.has(key)) { throw new Error(`Unknown flow runner option: --${key}`); } if (Object.hasOwn(options, key)) { throw new Error(`Flow runner option was provided more than once: --${key}`); } if (inlineValue !== undefined) { if (!inlineValue && !booleanCliOptions.has(key)) { throw new Error(`Flow runner option --${key} requires a value.`); } options[key] = inlineValue; continue; } const next = args[index + 1]; if (next && !next.startsWith('--')) { options[key] = next; index += 1; } else if (booleanCliOptions.has(key)) { options[key] = '1'; } else { throw new Error(`Flow runner option --${key} requires a value.`); } } return options; } function positiveNumber(value, label) { const number = Math.round(Number(value)); if (!Number.isFinite(number) || number <= 0) { throw new Error(`${label} must be a positive number; received ${value}.`); } return number; } function booleanOption(value, label) { if (value === undefined) { return false; } const normalized = String(value).trim().toLowerCase(); if (['1', 'true', 'yes', 'on'].includes(normalized)) { return true; } if (['0', 'false', 'no', 'off'].includes(normalized)) { return false; } throw new Error(`${label} must be a boolean value; received ${value}.`); } function assertSafeOutputDirectory(path) { const distRoot = resolve('dist'); const relativePath = relative(distRoot, path); if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) { throw new Error(`Flow output directory must be a dedicated subdirectory of dist: ${path}`); } } function assertNestedDirectory(parent, path, label) { const relativePath = relative(parent, path); if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) { throw new Error(`${label} must be a dedicated subdirectory of ${parent}: ${path}`); } } function formatDuration(ms) { const totalSeconds = Math.round(ms / 1000); const minutes = Math.floor(totalSeconds / 60); const seconds = totalSeconds % 60; return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`; } function delay(ms) { return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); }