heros_web/scripts/verify-flow-segments.mjs

import { spawn } from 'node:child_process';
import { existsSync } from 'node:fs';
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { dirname, isAbsolute, relative, resolve } from 'node:path';
import {
  campaignFlowSegmentSummary,
  readCampaignFlowSegments
} from './campaign-flow-segments.mjs';

const allowedCliOptions = new Set([
  'segment',
  'from',
  'to',
  'resume',
  'output-dir',
  'checkpoint-dir',
  'screenshots',
  'heartbeat-ms',
  'segment-timeout-ms',
  'list-segments'
]);
const booleanCliOptions = new Set(['resume', 'list-segments']);
const cliOptions = parseCliOptions(process.argv.slice(2));
if (cliOptions.segment && (cliOptions.from || cliOptions.to)) {
  throw new Error('--segment cannot be combined with --from or --to.');
}
const { segments, manifestHash } = await readCampaignFlowSegments();
const selectedSegments = selectSegments(segments, cliOptions);
const targetUrl = process.env.VERIFY_URL ?? 'http://127.0.0.1:41743/';
const outputDirectory = resolve(cliOptions['output-dir'] ?? process.env.VERIFY_FLOW_OUTPUT_DIR ?? 'dist/verify-flow');
const checkpointDirectory = resolve(
  cliOptions['checkpoint-dir'] ?? process.env.VERIFY_FLOW_CHECKPOINT_DIR ?? `${outputDirectory}/checkpoints`
);
const resume = booleanOption(cliOptions.resume ?? process.env.VERIFY_FLOW_RESUME, 'resume');
const screenshots = cliOptions.screenshots ?? process.env.VERIFY_FLOW_SCREENSHOTS ?? 'failure';
const heartbeatMs = positiveNumber(
  cliOptions['heartbeat-ms'] ?? process.env.VERIFY_FLOW_HEARTBEAT_MS ?? '15000',
  'heartbeat-ms'
);
const segmentTimeoutMs = positiveNumber(
  cliOptions['segment-timeout-ms'] ?? process.env.VERIFY_FLOW_SEGMENT_TIMEOUT_MS ?? '720000',
  'segment-timeout-ms'
);

if (booleanOption(cliOptions['list-segments'], 'list-segments')) {
  for (const segment of segments) {
    console.log(campaignFlowSegmentSummary(segment));
  }
  process.exit(0);
}

if (!['all', 'failure', 'none'].includes(screenshots)) {
  throw new Error(`Unknown screenshot mode "${screenshots}". Use all, failure, or none.`);
}

assertSafeOutputDirectory(outputDirectory);
assertNestedDirectory(outputDirectory, checkpointDirectory, 'Flow checkpoint directory');

if (!resume) {
  if (selectedSegments[0]?.index !== 0) {
    throw new Error('A run starting after the first segment requires --resume=1 and an existing previous checkpoint.');
  }
  await rm(outputDirectory, { recursive: true, force: true });
}
await mkdir(checkpointDirectory, { recursive: true });

const runStartedAt = Date.now();
const summaryPath = resolve(outputDirectory, 'summary.json');
const results = [];
let serverProcess;

await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments);
console.log(`[verify-flow] manifest ${manifestHash.slice(0, 12)} · ${selectedSegments.length}/${segments.length} segments`);

try {
  serverProcess = await ensureLocalServer(targetUrl);
  for (const segment of selectedSegments) {
    const previousSegment = segments[segment.index - 1];
    if (previousSegment) {
      const previousCheckpoint = resolve(checkpointDirectory, `${previousSegment.id}.json`);
      if (!existsSync(previousCheckpoint)) {
        throw new Error(`Missing prerequisite checkpoint for ${segment.id}: ${previousCheckpoint}`);
      }
      await validateSegmentCheckpoint(previousSegment, checkpointDirectory, manifestHash, segments);
    }

    const startedAt = Date.now();
    await rm(resolve(outputDirectory, segment.id), { recursive: true, force: true });
    console.log(`[verify-flow] segment start · ${campaignFlowSegmentSummary(segment)}`);
    const exitCode = await runSegmentProcess(segment, {
      checkpointDirectory,
      heartbeatMs,
      manifestHash,
      outputDirectory,
      screenshots,
      segmentTimeoutMs,
      targetUrl
    });
    const result = {
      id: segment.id,
      label: segment.label,
      firstBattleOrdinal: segment.firstBattleOrdinal,
      lastBattleOrdinal: segment.lastBattleOrdinal,
      durationMs: Date.now() - startedAt,
      exitCode,
      status: exitCode === 0 ? 'passed' : 'failed'
    };
    results.push(result);
    if (exitCode !== 0) {
      throw new Error(`Flow segment ${segment.id} failed with exit code ${exitCode}.`);
    }
    try {
      await validateSegmentArtifacts(segment, {
        checkpointDirectory,
        manifestHash,
        outputDirectory,
        segments
      });
      result.artifactValidation = 'passed';
    } catch (error) {
      result.status = 'failed';
      result.artifactValidation = 'failed';
      throw error;
    }
    await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments);
    console.log(`[verify-flow] segment pass · ${segment.id} · ${formatDuration(result.durationMs)}`);
  }

  console.log(`[verify-flow] all selected segments passed · ${formatDuration(Date.now() - runStartedAt)}`);
} catch (error) {
  await writeSummary(summaryPath, manifestHash, results, runStartedAt, selectedSegments, segments, error);
  throw error;
} finally {
  await terminateProcessTree(serverProcess);
}

async function runSegmentProcess(segment, options) {
  const progressPath = resolve(options.outputDirectory, segment.id, 'progress.jsonl');
  await mkdir(resolve(options.outputDirectory, segment.id), { recursive: true });
  const args = [
    'scripts/verify-flow.mjs',
    `--segment=${segment.id}`,
    `--checkpoint-dir=${options.checkpointDirectory}`,
    `--progress=${progressPath}`,
    `--screenshots=${options.screenshots}`,
    `--heartbeat-ms=${options.heartbeatMs}`,
    `--manifest-hash=${options.manifestHash}`
  ];

  return new Promise((resolveExit, reject) => {
    const child = spawn(process.execPath, args, {
      cwd: process.cwd(),
      env: { ...process.env, VERIFY_URL: options.targetUrl },
      stdio: 'inherit',
      windowsHide: true
    });
    let timedOut = false;
    let settled = false;
    const settleExit = (code) => {
      if (settled) {
        return;
      }
      settled = true;
      clearTimeout(timeout);
      resolveExit(code);
    };
    const settleError = (error) => {
      if (settled) {
        return;
      }
      settled = true;
      clearTimeout(timeout);
      reject(error);
    };
    const timeout = setTimeout(() => {
      timedOut = true;
      console.error(`[verify-flow] segment timeout · ${segment.id} · ${formatDuration(options.segmentTimeoutMs)}`);
      void (async () => {
        await terminateProcessTree(child);
        await writeTimeoutFailure(segment, options);
        settleExit(124);
      })().catch(settleError);
    }, options.segmentTimeoutMs);

    child.once('error', (error) => {
      if (!timedOut) {
        settleError(error);
      }
    });
    child.once('exit', (code, signal) => {
      if (timedOut) {
        return;
      }
      if (signal) {
        console.error(`[verify-flow] segment ${segment.id} ended by signal ${signal}.`);
      }
      settleExit(code ?? 1);
    });
  });
}

async function writeSummary(path, hash, segmentResults, startedAt, selectedSegments, manifestSegments, error) {
  const selectedSegmentIds = selectedSegments.map(({ id }) => id);
  const fullManifestSelected = selectedSegmentIds.length === manifestSegments.length &&
    selectedSegmentIds.every((id, index) => id === manifestSegments[index]?.id);
  const allSelectedFinished = segmentResults.length === selectedSegments.length;
  const allSelectedPassed = allSelectedFinished && segmentResults.every(({ status }) => status === 'passed');
  const status = error
    ? 'failed'
    : !allSelectedFinished
      ? 'running'
      : allSelectedPassed && fullManifestSelected
        ? 'full-manifest-passed'
        : allSelectedPassed
          ? 'partial-passed'
          : 'failed';
  await writeTextAtomically(path, `${JSON.stringify({
    version: 1,
    manifestHash: hash,
    status,
    startedAt: new Date(startedAt).toISOString(),
    updatedAt: new Date().toISOString(),
    durationMs: Date.now() - startedAt,
    manifestSegmentCount: manifestSegments.length,
    selectedSegmentCount: selectedSegments.length,
    selectedSegmentIds,
    fullManifestSelected,
    finishedSegments: segmentResults.length,
    passedSegments: segmentResults.filter(({ status: resultStatus }) => resultStatus === 'passed').length,
    results: segmentResults,
    error: error instanceof Error ? { name: error.name, message: error.message } : error ? { message: String(error) } : null
  }, null, 2)}\n`);
}

async function validateSegmentArtifacts(segment, options) {
  await validateSegmentCheckpoint(segment, options.checkpointDirectory, options.manifestHash, options.segments);
  const progressPath = resolve(options.outputDirectory, segment.id, 'progress.jsonl');
  if (!existsSync(progressPath)) {
    throw new Error(`Flow segment ${segment.id} exited successfully without a progress log: ${progressPath}`);
  }
  const progressSource = await readFile(progressPath, 'utf8');
  const events = progressSource
    .split(/\r?\n/)
    .filter(Boolean)
    .map((line, index) => {
      try {
        return JSON.parse(line);
      } catch (error) {
        throw new Error(
          `Flow segment ${segment.id} has invalid progress JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`
        );
      }
    });
  const startEvent = events.find(({ type }) => type === 'start');
  const checkpointEvent = events.findLast(({ type }) => type === 'checkpoint');
  const passEvent = events.findLast(({ type }) => type === 'segment-pass');
  if (
    startEvent?.segment !== segment.id ||
    startEvent.manifestHash !== options.manifestHash ||
    checkpointEvent?.segment !== segment.id ||
    checkpointEvent.campaignStep !== segment.checkpointStep ||
    passEvent?.segment !== segment.id
  ) {
    throw new Error(`Flow segment ${segment.id} did not emit the required start/checkpoint/pass progress contract.`);
  }
}

async function validateSegmentCheckpoint(segment, checkpointDirectory, hash, allSegments) {
  const checkpointPath = resolve(checkpointDirectory, `${segment.id}.json`);
  if (!existsSync(checkpointPath)) {
    throw new Error(`Flow segment ${segment.id} exited successfully without a checkpoint: ${checkpointPath}`);
  }
  let checkpoint;
  try {
    checkpoint = JSON.parse(await readFile(checkpointPath, 'utf8'));
  } catch (error) {
    throw new Error(
      `Flow checkpoint ${segment.id} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
    );
  }
  if (
    checkpoint?.version !== 1 ||
    checkpoint.manifestHash !== hash ||
    checkpoint.segment?.id !== segment.id ||
    checkpoint.segment?.lastBattleOrdinal !== segment.lastBattleOrdinal ||
    checkpoint.segment?.lastBattleId !== segment.lastBattleId ||
    checkpoint.segment?.checkpointStep !== segment.checkpointStep ||
    !checkpoint.localStorage ||
    typeof checkpoint.localStorage !== 'object'
  ) {
    throw new Error(`Flow checkpoint ${segment.id} does not match the active manifest.`);
  }
  const globalRaw = checkpoint.localStorage['heros-web:campaign-state'];
  const slotRaw = checkpoint.localStorage['heros-web:campaign-state:slot-1'];
  if (!globalRaw || !slotRaw || globalRaw !== slotRaw) {
    throw new Error(`Flow checkpoint ${segment.id} must contain identical global and slot-1 campaign saves.`);
  }
  let campaign;
  try {
    campaign = JSON.parse(slotRaw);
  } catch (error) {
    throw new Error(
      `Flow checkpoint ${segment.id} campaign save is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
    );
  }
  const expectedBattleIds = allSegments
    .slice(0, segment.index + 1)
    .flatMap((candidate) => candidate.battleIds);
  const completedBattleIds = Object.keys(campaign?.battleHistory ?? {});
  const exactBattlePrefix = expectedBattleIds.length === completedBattleIds.length &&
    expectedBattleIds.every((battleId, index) => completedBattleIds[index] === battleId);
  const allBattlesWon = expectedBattleIds.every(
    (battleId) => campaign?.battleHistory?.[battleId]?.outcome === 'victory'
  );
  if (
    campaign?.version !== 1 ||
    campaign?.activeSaveSlot !== 1 ||
    campaign?.step !== segment.checkpointStep ||
    campaign?.latestBattleId !== segment.lastBattleId ||
    !exactBattlePrefix ||
    !allBattlesWon
  ) {
    throw new Error(`Flow checkpoint ${segment.id} campaign position or battle history is invalid.`);
  }
}

async function writeTimeoutFailure(segment, options) {
  const segmentDirectory = resolve(options.outputDirectory, segment.id);
  const progressPath = resolve(segmentDirectory, 'progress.jsonl');
  let lastProgressEvent = null;
  try {
    const lines = (await readFile(progressPath, 'utf8')).split(/\r?\n/).filter(Boolean);
    if (lines.length > 0) {
      lastProgressEvent = JSON.parse(lines.at(-1));
    }
  } catch {
    lastProgressEvent = null;
  }
  await writeTextAtomically(resolve(segmentDirectory, 'timeout-state.json'), `${JSON.stringify({
    version: 1,
    manifestHash: options.manifestHash,
    timestamp: new Date().toISOString(),
    segment: segment.id,
    timeoutMs: options.segmentTimeoutMs,
    lastProgressEvent
  }, null, 2)}\n`);
}

async function writeTextAtomically(path, contents) {
  await mkdir(dirname(path), { recursive: true });
  const temporaryPath = `${path}.${process.pid}.${Date.now()}.tmp`;
  try {
    await writeFile(temporaryPath, contents, 'utf8');
    await rename(temporaryPath, path);
  } catch (error) {
    await rm(temporaryPath, { force: true }).catch(() => {});
    throw error;
  }
}

function selectSegments(allSegments, options) {
  const onlyId = options.segment;
  const fromId = options.from;
  const toId = options.to;
  if (onlyId) {
    const segment = allSegments.find((candidate) => candidate.id === onlyId);
    if (!segment) {
      throw new Error(`Unknown flow segment "${onlyId}". Use one of: ${allSegments.map(({ id }) => id).join(', ')}`);
    }
    return [segment];
  }

  const fromIndex = fromId ? allSegments.findIndex(({ id }) => id === fromId) : 0;
  const toIndex = toId ? allSegments.findIndex(({ id }) => id === toId) : allSegments.length - 1;
  if (fromIndex < 0 || toIndex < 0 || fromIndex > toIndex) {
    throw new Error(`Invalid segment range: ${fromId ?? allSegments[0].id}..${toId ?? allSegments.at(-1).id}`);
  }
  return allSegments.slice(fromIndex, toIndex + 1);
}

async function ensureLocalServer(url) {
  if (await canReach(url)) {
    return undefined;
  }
  const parsed = new URL(url);
  if (!['localhost', '127.0.0.1', '0.0.0.0'].includes(parsed.hostname)) {
    throw new Error(`No server responded at ${url}`);
  }
  const child = spawn(
    process.execPath,
    ['node_modules/vite/bin/vite.js', '--host', '127.0.0.1', '--port', parsed.port || '41743', '--strictPort'],
    { cwd: process.cwd(), env: process.env, stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true }
  );
  let startupError;
  let stderr = '';
  child.once('error', (error) => {
    startupError = error;
  });
  child.stderr?.on('data', (chunk) => {
    stderr = `${stderr}${chunk}`.slice(-6000);
  });
  for (let attempt = 0; attempt < 120; attempt += 1) {
    if (startupError) {
      throw new Error(`Could not start the segmented flow server: ${startupError.message}`);
    }
    if (child.exitCode !== null || child.signalCode !== null) {
      throw new Error(
        `Segmented flow server exited before becoming ready (${child.exitCode ?? child.signalCode}).${stderr ? `\n${stderr.trim()}` : ''}`
      );
    }
    if (await canReach(url)) {
      return child;
    }
    await delay(500);
  }
  await terminateProcessTree(child);
  throw new Error(`Could not start the segmented flow server at ${url}`);
}

async function terminateProcessTree(child) {
  if (!child || child.exitCode !== null || child.signalCode !== null) {
    return;
  }
  if (process.platform === 'win32' && child.pid) {
    await runProcessTreeTerminator(child.pid);
  } else {
    try {
      child.kill('SIGTERM');
    } catch {
      // The process may have exited between the state check and the signal.
    }
  }
  await waitForProcessExit(child, 3000);
  if (child.exitCode === null && child.signalCode === null) {
    try {
      child.kill('SIGKILL');
    } catch {
      // The process may have exited between the state check and the signal.
    }
    await waitForProcessExit(child, 2000);
  }
  if (child.exitCode === null && child.signalCode === null) {
    throw new Error(`Could not terminate process tree for PID ${child.pid ?? 'unknown'}.`);
  }
}

async function runProcessTreeTerminator(pid) {
  await new Promise((resolveTermination) => {
    const terminator = spawn('taskkill.exe', ['/PID', String(pid), '/T', '/F'], {
      stdio: 'ignore',
      windowsHide: true
    });
    let settled = false;
    const finish = () => {
      if (settled) {
        return;
      }
      settled = true;
      clearTimeout(timeout);
      resolveTermination();
    };
    const timeout = setTimeout(() => {
      terminator.kill();
      finish();
    }, 5000);
    terminator.once('error', finish);
    terminator.once('exit', finish);
  });
}

async function waitForProcessExit(child, timeoutMs) {
  if (!child || child.exitCode !== null || child.signalCode !== null) {
    return;
  }
  await new Promise((resolveExit) => {
    let settled = false;
    const finish = () => {
      if (settled) {
        return;
      }
      settled = true;
      clearTimeout(timeout);
      child.off('exit', finish);
      child.off('error', finish);
      resolveExit();
    };
    const timeout = setTimeout(finish, timeoutMs);
    child.once('exit', finish);
    child.once('error', finish);
  });
}

async function canReach(url) {
  try {
    const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
    return response.ok;
  } catch {
    return false;
  }
}

function parseCliOptions(args) {
  const options = {};
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (!arg.startsWith('--')) {
      throw new Error(`Unexpected flow runner argument: ${arg}`);
    }
    const option = arg.slice(2);
    const equalsIndex = option.indexOf('=');
    const key = equalsIndex >= 0 ? option.slice(0, equalsIndex) : option;
    const inlineValue = equalsIndex >= 0 ? option.slice(equalsIndex + 1) : undefined;
    if (!allowedCliOptions.has(key)) {
      throw new Error(`Unknown flow runner option: --${key}`);
    }
    if (Object.hasOwn(options, key)) {
      throw new Error(`Flow runner option was provided more than once: --${key}`);
    }
    if (inlineValue !== undefined) {
      if (!inlineValue && !booleanCliOptions.has(key)) {
        throw new Error(`Flow runner option --${key} requires a value.`);
      }
      options[key] = inlineValue;
      continue;
    }
    const next = args[index + 1];
    if (next && !next.startsWith('--')) {
      options[key] = next;
      index += 1;
    } else if (booleanCliOptions.has(key)) {
      options[key] = '1';
    } else {
      throw new Error(`Flow runner option --${key} requires a value.`);
    }
  }
  return options;
}

function positiveNumber(value, label) {
  const number = Math.round(Number(value));
  if (!Number.isFinite(number) || number <= 0) {
    throw new Error(`${label} must be a positive number; received ${value}.`);
  }
  return number;
}

function booleanOption(value, label) {
  if (value === undefined) {
    return false;
  }
  const normalized = String(value).trim().toLowerCase();
  if (['1', 'true', 'yes', 'on'].includes(normalized)) {
    return true;
  }
  if (['0', 'false', 'no', 'off'].includes(normalized)) {
    return false;
  }
  throw new Error(`${label} must be a boolean value; received ${value}.`);
}

function assertSafeOutputDirectory(path) {
  const distRoot = resolve('dist');
  const relativePath = relative(distRoot, path);
  if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) {
    throw new Error(`Flow output directory must be a dedicated subdirectory of dist: ${path}`);
  }
}

function assertNestedDirectory(parent, path, label) {
  const relativePath = relative(parent, path);
  if (!relativePath || relativePath.startsWith('..') || isAbsolute(relativePath)) {
    throw new Error(`${label} must be a dedicated subdirectory of ${parent}: ${path}`);
  }
}

function formatDuration(ms) {
  const totalSeconds = Math.round(ms / 1000);
  const minutes = Math.floor(totalSeconds / 60);
  const seconds = totalSeconds % 60;
  return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
}

function delay(ms) {
  return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
}