PIG-Demo/scripts/conformance.mjs

#!/usr/bin/env node
/**
 * The cross-language conformance gate.
 *
 * Every environment under `envs/` is the reference implementation, and the
 * matching `src/demos/<slug>/engine.ts` is a port of it. The site's entire
 * argument is that the board you play in the browser is the same environment
 * the model was evaluated in, so the two implementations agreeing is not a
 * nice-to-have — it is the claim. Each environment commits the digest(s) its
 * Python computes to `envs/<pkg>/CONFORMANCE.txt`; this script recomputes them
 * from the TypeScript and fails if any differ.
 *
 * ── wordle ──────────────────────────────────────────────────────────────────
 * A hand-written vector file would only ever catch the cases somebody thought
 * of, and the case nobody thinks of is always the same one: repeated letters,
 * where a green must claim its letter before any yellow is assigned. So the
 * gate is exhaustive. For every answer A in `words/answers.json`, in list
 * order, concatenate `score(G, A)` for every guess G in that same list, in that
 * same order, and stream the whole thing through SHA-256. Both the list order
 * and the two nested loop orders are part of the definition.
 *
 * ── alert-triage ────────────────────────────────────────────────────────────
 * Three digests, because three things can drift:
 *   world     canonical JSON of the generated world for seeds 0–4095, with the
 *             reference minutes the shipped policies compute. One RNG draw out
 *             of order anywhere in the generator changes every seed after it.
 *   scorer    `scoreExact` over a fixed grid of synthetic episodes, as exact
 *             fractions so no float formatting is hashed.
 *   protocol  parse + engine over a committed corpus of reply strings —
 *             accepted actions, rejection reasons and the minute meter after
 *             every turn. JSON-in-prose with fences and a brace scan is where
 *             two runtimes disagree, and the rejection reasons quote the
 *             offending value with Python's `repr`, so this is the digest that
 *             catches a port that "mostly works".
 *
 * Usage:
 *   node scripts/conformance.mjs                 every environment, compared to its CONFORMANCE.txt
 *   node scripts/conformance.mjs --only wordle   one environment
 *   node scripts/conformance.mjs --limit 50      wordle: first 50 answers; alert-triage: first 50 seeds
 *                                                of the world digest. No comparison.
 *
 * An environment whose `engine.ts` does not exist yet is noted and skipped
 * (exit 0) — the TypeScript port and this gate are allowed to arrive in
 * either order. An environment whose CONFORMANCE.txt does not exist yet has
 * its digests printed with the command that writes the file.
 */

import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';

import { abs, cyan, die, dim, exists, green, read, red, rel } from './_lib.mjs';

/* ------------------------------------------------------------------ flags */

const argv = process.argv.slice(2);
const limitFlag = argv.indexOf('--limit');
const limit = limitFlag === -1 ? 0 : Number(argv[limitFlag + 1] ?? 0);
if (limitFlag !== -1 && (!Number.isInteger(limit) || limit <= 0)) {
  die('--limit takes a positive integer, e.g. `--limit 50`.');
}
const onlyFlag = argv.indexOf('--only');
const only = onlyFlag === -1 ? null : argv[onlyFlag + 1];

/* ----------------------------------------------------------- environments */

/**
 * @typedef {{
 *   name: string,
 *   engine: string,
 *   expected: string,
 *   args: string[],
 *   probe: () => string,
 *   writeHint: string,
 *   diagnose: string,
 *   describe: (result: any, seconds: string) => string[],
 * }} Environment
 */

/** @type {Environment[]} */
const ENVIRONMENTS = [
  {
    name: 'wordle',
    engine: abs('src', 'demos', 'wordle', 'engine.ts'),
    expected: abs('envs', 'wordle_five', 'CONFORMANCE.txt'),
    args: [abs('envs', 'wordle_five', 'words', 'answers.json')],
    probe: wordleProbeSource,
    writeHint: 'python -c "from wordle_five.engine import conformance_digest; print(conformance_digest())"',
    diagnose:
      '      The usual culprit is the two-pass scoring: every green must claim its letter out of the pool\n' +
      '      BEFORE any yellow is assigned, or a guess like SASSY against BASIS marks an S yellow that the\n' +
      '      greens have already spent.',
    describe: (result, seconds) => [
      `  scored ${result.pairs.toLocaleString('en-US')} pairs from ${result.answers.toLocaleString('en-US')} answers in ${seconds}s`,
      `  via ${cyan(result.via)}`,
    ],
  },
  {
    name: 'alert-triage',
    engine: abs('src', 'demos', 'alert-triage', 'engine.ts'),
    expected: abs('envs', 'alert_triage', 'CONFORMANCE.txt'),
    args: [abs('envs', 'alert_triage', 'conformance', 'replies.json')],
    probe: triageProbeSource,
    writeHint: 'uv run python -m alert_triage.conformance --write   (from envs/alert_triage)',
    diagnose:
      '      world:    an RNG draw made out of order, or a `//` that became a `/`. Bisect by seed: dump\n' +
      '                canonicalJson(worldForSeed(s)) on both sides and diff the first seed that differs.\n' +
      '      scorer:   scoreExact() — check the fraction forms and the best-alternate tie-break (first wins).\n' +
      '      protocol: the parse rules (fence wins; first balanced span; no rescanning) or a rejection\n' +
      '                reason whose repr() of the offending value differs from Python.',
    describe: (result, seconds) => [
      `  hashed ${result.seeds.toLocaleString('en-US')} worlds, ${result.grid.toLocaleString('en-US')} grid episodes and ${result.cases} corpus cases in ${seconds}s`,
    ],
  },
];

/* ------------------------------------------------------------------ run it */

if (!exists(abs('node_modules', '.bin', 'tsx'))) die('tsx is not installed, and the engines are TypeScript. Run `pnpm install`.');

const selected = ENVIRONMENTS.filter((env) => only === null || env.name === only);
if (only !== null && selected.length === 0) {
  die(`--only ${only}: no such environment. Known: ${ENVIRONMENTS.map((e) => e.name).join(', ')}.`);
}

let failed = false;
for (const env of selected) {
  if (!runEnvironment(env)) failed = true;
  console.log('');
}
process.exit(failed ? 1 : 0);

/** @param {Environment} env */
function runEnvironment(env) {
  if (!exists(env.engine)) {
    console.log(dim(`conformance[${env.name}]: engine not present yet — no ${rel(env.engine)} to check.`));
    console.log(dim('The TypeScript port and this gate may land in either order; re-run once it exists.'));
    return true;
  }
  for (const arg of env.args) {
    if (!exists(arg)) die(`${rel(arg)} does not exist. It is part of the definition of the ${env.name} digest.`);
  }

  const probeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pig-conformance-'));
  const probe = path.join(probeDir, 'digest.mts');
  fs.writeFileSync(probe, env.probe(), 'utf8');

  const started = Date.now();
  console.log(dim(`conformance[${env.name}]: hashing through ${rel(env.engine)}...`));

  // The tsx bin is a POSIX shell shim, not a JS file, so it is the executable
  // here rather than an argument to node.
  const run = spawnSync(abs('node_modules', '.bin', 'tsx'), [probe, env.engine, ...env.args, String(limit)], {
    cwd: abs('.'),
    encoding: 'utf8',
    maxBuffer: 16 * 1024 * 1024,
    timeout: 15 * 60_000,
    // The probe lives in a temp directory, where tsx would never find the
    // repo's tsconfig — and without it the `@/` alias in an engine does not
    // resolve and the import fails for a reason that looks nothing like the cause.
    env: { ...process.env, TSX_TSCONFIG_PATH: abs('tsconfig.json') },
    stdio: ['ignore', 'pipe', 'inherit'],
  });
  fs.rmSync(probeDir, { recursive: true, force: true });

  if (run.error) die(`could not run the ${env.name} digest probe: ${run.error.message}`);
  if (run.status !== 0) die(`the ${env.name} digest probe exited ${run.status}. See the error above.`);

  let result;
  try {
    result = JSON.parse(String(run.stdout).trim().split('\n').pop());
  } catch {
    die(`the ${env.name} digest probe produced no parseable result. Output was:\n${run.stdout}`);
  }
  if (result.error) {
    die(`${rel(env.engine)}: ${result.error}`);
  }

  const seconds = ((Date.now() - started) / 1000).toFixed(1);
  for (const line of env.describe(result, seconds)) console.log(line);
  /** @type {Record<string, string>} */
  const digests = result.digests;
  const names = Object.keys(digests);
  for (const name of names) console.log(`  ${names.length > 1 ? name.padEnd(9) : 'digest'} ${cyan(digests[name])}`);

  if (limit) {
    console.log('');
    console.log(dim(`Partial run (--limit ${limit}): not compared to ${rel(env.expected)}. Run without --limit for the real gate.`));
    return true;
  }

  /* ----------------------------------------------------------- compare */

  if (!exists(env.expected)) {
    console.log('');
    console.log(green(`conformance[${env.name}]: computed, but there is nothing to compare against yet.`));
    console.log(`Write the digest(s) to ${rel(env.expected)} once the Python agrees:`);
    console.log(dim(`  ${env.writeHint}`));
    return true;
  }

  const expected = parseExpected(read(env.expected), names);
  if (!expected) {
    die(`${rel(env.expected)} contains no 64-character hex digest${names.length > 1 ? ` for each of: ${names.join(', ')}` : ''}.`);
  }

  const mismatched = names.filter((name) => expected[name] !== digests[name]);
  if (mismatched.length) {
    console.error('');
    console.error(`${red('FAIL')}  the TypeScript engine and ${rel(env.expected)} disagree${names.length > 1 ? ` on: ${mismatched.join(', ')}` : ''}.`);
    for (const name of mismatched) {
      const label = names.length > 1 ? ` (${name})` : '';
      console.error(`      expected${label}  ${expected[name]}`);
      console.error(`      got${label}       ${digests[name]}`);
    }
    console.error('');
    console.error(
      '      One of the two implementations is wrong, and the board on the site is therefore not the\n' +
        '      environment the model was evaluated in.\n' +
        env.diagnose +
        '\n      If the Python changed on purpose, re-run its own digest and update the file:\n' +
        `      ${env.writeHint}`,
    );
    return false;
  }

  console.log('');
  console.log(green(`conformance[${env.name}]: the TypeScript engine matches ${rel(env.expected)} exactly${names.length > 1 ? ` (${names.join(', ')})` : ''}.`));
  return true;
}

/**
 * Reads the committed digests. A single-digest file is one hex string on a
 * line; a multi-digest file is `hex  name  description` per line, and every
 * name the probe produced must be present.
 *
 * @param {string} text
 * @param {string[]} names
 * @returns {Record<string, string> | null}
 */
function parseExpected(text, names) {
  /** @type {Record<string, string>} */
  const out = {};
  if (names.length === 1) {
    const hex = text.match(/\b[0-9a-f]{64}\b/)?.[0];
    if (!hex) return null;
    out[names[0]] = hex;
    return out;
  }
  for (const line of text.split('\n')) {
    const m = line.match(/^\s*([0-9a-f]{64})\s+(\S+)/);
    if (m) out[m[2]] = m[1];
  }
  return names.every((n) => n in out) ? out : null;
}

/* ------------------------------------------------------------------ probes */

/**
 * Each probe runs under tsx so it can import the TypeScript engine directly,
 * and prints one JSON line: `{ digests: { <name>: hex }, ...counts }` or
 * `{ error }`.
 *
 * Kept here as strings rather than as checked-in `.mts` files: they are
 * implementation details of this script, and a stray TypeScript file in
 * `scripts/` would be swept into a typecheck it is deliberately outside of.
 *
 * Function declarations, not `const`s, so they are hoisted above the table
 * that references them near the top of this file.
 */
function wordleProbeSource() {
  return `
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';

const [enginePath, answersPath, limitRaw] = process.argv.slice(2);
const limit = Number(limitRaw ?? 0);
const say = (value: unknown) => console.log(JSON.stringify(value));

let mod: any;
try {
  mod = await import(enginePath);
} catch (error: any) {
  say({ error: 'could not import it: ' + String(error?.message ?? error).split('\\n')[0] });
  process.exit(0);
}

const answers: string[] = JSON.parse(readFileSync(answersPath, 'utf8'));
const pool = limit > 0 ? answers.slice(0, limit) : answers;

// If the engine ships the digest itself, trust it over anything reconstructed
// here — it is the implementation's own statement of the definition.
if (typeof mod.conformanceDigest === 'function' && limit === 0) {
  const digest = await mod.conformanceDigest();
  say({ digests: { wordle: digest }, via: 'engine.conformanceDigest()', pairs: pool.length * pool.length, answers: pool.length });
  process.exit(0);
}

// Argument order is the Python's: score(guess, answer). Getting it backwards
// produces a valid-looking digest that matches nothing, so if this disagrees
// with CONFORMANCE.txt, check the signature before you check the algorithm.
const NAMES = ['scoreGuess', 'score', 'scorePattern', 'scoreWord', 'feedback', 'pattern'];
const name =
  NAMES.find((n) => typeof mod[n] === 'function' && mod[n].length >= 2) ??
  NAMES.find((n) => typeof mod[n] === 'function') ??
  (typeof mod.default === 'function' ? 'default' : null);

if (!name) {
  say({
    error:
      'no scoring function is exported. Found: ' + Object.keys(mod).join(', ') +
      '. The gate needs a scoring function it can recognise. Export one named scoreGuess (or score), ' +
      'taking (guess, answer) and returning the pattern — either a "GYXXG" string or an array of ' +
      "'exact' | 'present' | 'absent' tiles. Exporting conformanceDigest() directly also works.",
  });
  process.exit(0);
}
const score = mod[name];

/** Every spelling of green/yellow/grey this port might reasonably have chosen. */
const TILE: Record<string, string> = {
  g: 'G', y: 'Y', x: 'X',
  green: 'G', yellow: 'Y', grey: 'X', gray: 'X',
  exact: 'G', present: 'Y', absent: 'X',
  correct: 'G', misplaced: 'Y', miss: 'X', hit: 'G',
};

function normalise(value: any): string {
  if (typeof value === 'string') {
    const upper = value.toUpperCase();
    if (/^[GYX]+$/.test(upper)) return upper;
    const chars = [...value].map((c) => TILE[c.toLowerCase()]);
    if (chars.every(Boolean)) return chars.join('');
    throw new Error('unrecognised pattern string ' + JSON.stringify(value));
  }
  if (Array.isArray(value)) {
    return value
      .map((tile) => {
        const key = typeof tile === 'string' ? tile : tile?.state ?? tile?.tile ?? tile?.kind ?? tile?.result ?? tile?.status;
        const mapped = TILE[String(key).toLowerCase()];
        if (!mapped) throw new Error('unrecognised tile ' + JSON.stringify(tile));
        return mapped;
      })
      .join('');
  }
  throw new Error('unrecognised pattern ' + JSON.stringify(value));
}

let convert: (value: any) => string;
try {
  const sample = score(pool[0], pool[0]);
  // 21 million conversions: skip the whole normaliser when the engine already
  // speaks the reference alphabet, which is the case that actually ships.
  convert = typeof sample === 'string' && /^[GYX]+$/.test(sample) ? (v: any) => v : normalise;
  const selfScore = convert(sample);
  if (!/^G+$/.test(selfScore)) {
    say({ error: 'score(w, w) returned "' + selfScore + '"; a word scored against itself must be all-green' });
    process.exit(0);
  }
} catch (error: any) {
  say({ error: String(error?.message ?? error) });
  process.exit(0);
}

const hash = createHash('sha256');
const row = new Array<string>(pool.length);
for (let i = 0; i < pool.length; i += 1) {
  const answer = pool[i]!;
  for (let j = 0; j < pool.length; j += 1) row[j] = convert(score(pool[j]!, answer));
  hash.update(row.join(''));
  // Only on a TTY: written to a pipe or a CI log, a carriage return is not a
  // cursor move and the progress ends up inline with the result.
  if (i % 250 === 0 && process.stderr.isTTY) process.stderr.write('    ' + i + '/' + pool.length + ' answers\\r');
}
if (process.stderr.isTTY) process.stderr.write(' '.repeat(40) + '\\r');

say({ digests: { wordle: hash.digest('hex') }, via: 'engine.' + name + '(guess, answer)', pairs: pool.length * pool.length, answers: pool.length });
`;
}

/**
 * The alert-triage probe. Three digests, each SHA-256 over newline-terminated
 * canonical JSON lines, exactly as \`alert_triage/conformance.py\` defines
 * them. The engine exports the lines; the probe only hashes them, so the
 * engine stays free of node:crypto and browser-clean.
 */
function triageProbeSource() {
  return `
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';

const [enginePath, corpusPath, limitRaw] = process.argv.slice(2);
const limit = Number(limitRaw ?? 0);
const say = (value: unknown) => console.log(JSON.stringify(value));

let mod: any;
try {
  mod = await import(enginePath);
} catch (error: any) {
  say({ error: 'could not import it: ' + String(error?.message ?? error).split('\\n')[0] });
  process.exit(0);
}

for (const name of ['worldLines', 'scorerLines', 'protocolLines']) {
  if (typeof mod[name] !== 'function') {
    say({ error: 'does not export ' + name + '(). Found: ' + Object.keys(mod).join(', ') });
    process.exit(0);
  }
}

function digestOf(lines: Iterable<string>, label: string, total: number): [string, number] {
  const hash = createHash('sha256');
  let count = 0;
  for (const line of lines) {
    hash.update(line, 'utf8');
    hash.update('\\n');
    count += 1;
    if (count % 256 === 0 && process.stderr.isTTY) process.stderr.write('    ' + label + ' ' + count + '/' + total + '\\r');
  }
  if (process.stderr.isTTY) process.stderr.write(' '.repeat(40) + '\\r');
  return [hash.digest('hex'), count];
}

const SEEDS = 4096;
const seedCount = limit > 0 ? Math.min(limit, SEEDS) : SEEDS;
const seeds = Array.from({ length: seedCount }, (_, i) => i);
const corpus = JSON.parse(readFileSync(corpusPath, 'utf8'));

try {
  const [world] = digestOf(mod.worldLines(seeds), 'world', seedCount);
  const [scorer, grid] = digestOf(mod.scorerLines(), 'scorer', 0);
  const [protocol, cases] = digestOf(mod.protocolLines(corpus), 'protocol', corpus.length);
  say({ digests: { world, scorer, protocol }, seeds: seedCount, grid, cases });
} catch (error: any) {
  say({ error: String(error?.stack ?? error?.message ?? error).split('\\n').slice(0, 3).join(' | ') });
}
`;
}