PIG-Demo/scripts/conformance.mjs

#!/usr/bin/env node
/**
 * The cross-language conformance gate.
 *
 * `envs/wordle_five/wordle_five/engine.py` is the reference implementation and
 * `src/demos/wordle/engine.ts` is a port of it. The site's entire argument is
 * that the board you play in the browser is the same environment the model was
 * evaluated in, so the two implementations agreeing is not a nice-to-have — it
 * is the claim.
 *
 * A hand-written vector file would only ever catch the cases somebody thought
 * of, and the case nobody thinks of is always the same one: repeated letters,
 * where a green must claim its letter before any yellow is assigned. So the
 * gate is exhaustive instead. For every answer A in `words/answers.json`, in
 * list order, concatenate `score(G, A)` for every guess G in that same list, in
 * that same order, and stream the whole thing through SHA-256. That is the
 * digest `engine.py`'s `conformance_digest()` computes, and it is the digest
 * this script computes from the TypeScript.
 *
 * Both the answer list order and the two nested loop orders are part of the
 * definition. Sorting the list, deduplicating it, or swapping the loops all
 * produce a different, equally valid, completely useless number.
 *
 * Usage:
 *   node scripts/conformance.mjs            full digest, compared to CONFORMANCE.txt
 *   node scripts/conformance.mjs --limit 50 first 50 answers only, no comparison
 *
 * Exits 0 with a note (not an error) when `engine.ts` does not exist yet — the
 * TypeScript port and this script are allowed to arrive in either order.
 */

import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';

import { abs, cyan, die, dim, exists, green, read, red, rel } from './_lib.mjs';

const ANSWERS = abs('envs', 'wordle_five', 'words', 'answers.json');
const EXPECTED_FILE = abs('envs', 'wordle_five', 'CONFORMANCE.txt');

const limitFlag = process.argv.indexOf('--limit');
const limit = limitFlag === -1 ? 0 : Number(process.argv[limitFlag + 1] ?? 0);
if (limitFlag !== -1 && (!Number.isInteger(limit) || limit <= 0)) {
  die('--limit takes a positive integer, e.g. `--limit 50`.');
}

/* ------------------------------------------------------ find the TS engine */

function findEngine() {
  const preferred = abs('src', 'demos', 'wordle', 'engine.ts');
  if (exists(preferred)) return preferred;
  const demosDir = abs('src', 'demos');
  if (!exists(demosDir)) return null;
  const found = fs
    .readdirSync(demosDir, { withFileTypes: true })
    .filter((e) => e.isDirectory() && !e.name.startsWith('_'))
    .map((e) => path.join(demosDir, e.name, 'engine.ts'))
    .filter(exists);
  // More than one engine.ts and no `wordle` directory means there is nothing to
  // guess between; say so rather than silently conforming the wrong one.
  return found.length === 1 ? found[0] : null;
}

const engine = findEngine();
if (!engine) {
  console.log(dim('conformance: engine not present yet — no src/demos/wordle/engine.ts to check.'));
  console.log(dim('The TypeScript port and this gate may land in either order; re-run once it exists.'));
  process.exit(0);
}

if (!exists(ANSWERS)) die(`${rel(ANSWERS)} does not exist. It defines both the pairs and their order.`);
if (!exists(abs('node_modules', '.bin', 'tsx'))) die('tsx is not installed, and the engine is TypeScript. Run `pnpm install`.');

/* ------------------------------------------------------------------- run it */

const probeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pig-conformance-'));
const probe = path.join(probeDir, 'digest.mts');
fs.writeFileSync(probe, probeSource(), 'utf8');

const started = Date.now();
console.log(dim(`conformance: hashing every (guess, answer) pair through ${rel(engine)}...`));

// The tsx bin is a POSIX shell shim, not a JS file, so it is the executable
// here rather than an argument to node.
const run = spawnSync(abs('node_modules', '.bin', 'tsx'), [probe, engine, ANSWERS, String(limit)], {
  cwd: abs('.'),
  encoding: 'utf8',
  maxBuffer: 16 * 1024 * 1024,
  timeout: 15 * 60_000,
  // The probe lives in a temp directory, where tsx would never find the
  // repo's tsconfig — and without it the `@/` alias in engine.ts does not
  // resolve and the import fails for a reason that looks nothing like the cause.
  env: { ...process.env, TSX_TSCONFIG_PATH: abs('tsconfig.json') },
  stdio: ['ignore', 'pipe', 'inherit'],
});
fs.rmSync(probeDir, { recursive: true, force: true });

if (run.error) die(`could not run the digest probe: ${run.error.message}`);
if (run.status !== 0) die(`the digest probe exited ${run.status}. See the error above.`);

let result;
try {
  result = JSON.parse(String(run.stdout).trim().split('\n').pop());
} catch {
  die(`the digest probe produced no parseable result. Output was:\n${run.stdout}`);
}
if (result.error) {
  die(
    `${rel(engine)}: ${result.error}\n` +
      '      The gate needs a scoring function it can recognise. Export one named `scoreGuess` (or `score`), ' +
      'taking (guess, answer) and returning the pattern — either a "GYXXG" string or an array of ' +
      "'exact' | 'present' | 'absent' tiles. Exporting `conformanceDigest()` directly also works.",
  );
}

const seconds = ((Date.now() - started) / 1000).toFixed(1);
console.log(`  scored ${result.pairs.toLocaleString('en-US')} pairs from ${result.answers.toLocaleString('en-US')} answers in ${seconds}s`);
console.log(`  via ${cyan(result.via)}`);
console.log(`  digest ${cyan(result.digest)}`);

if (limit) {
  console.log('');
  console.log(dim(`Partial run (--limit ${limit}): not compared to ${rel(EXPECTED_FILE)}. Run without --limit for the real gate.`));
  process.exit(0);
}

/* ----------------------------------------------------------- compare it */

if (!exists(EXPECTED_FILE)) {
  console.log('');
  console.log(green('conformance: computed, but there is nothing to compare against yet.'));
  console.log(`Write the digest to ${rel(EXPECTED_FILE)} once the Python agrees:`);
  console.log(dim(`  python -c "from wordle_five.engine import conformance_digest; print(conformance_digest())"`));
  process.exit(0);
}

const expected = read(EXPECTED_FILE).match(/\b[0-9a-f]{64}\b/)?.[0];
if (!expected) {
  die(`${rel(EXPECTED_FILE)} contains no 64-character hex digest.`);
}

if (expected !== result.digest) {
  console.error('');
  console.error(`${red('FAIL')}  the TypeScript engine and ${rel(EXPECTED_FILE)} disagree.`);
  console.error(`      expected  ${expected}`);
  console.error(`      got       ${result.digest}`);
  console.error('');
  console.error(
    '      One of the two implementations is wrong, and the board on the site is therefore not the\n' +
      '      environment the model was evaluated in. The usual culprit is the two-pass scoring: every\n' +
      '      green must claim its letter out of the pool BEFORE any yellow is assigned, or a guess like\n' +
      '      SASSY against BASIS marks an S yellow that the greens have already spent.\n' +
      '      If the Python changed on purpose, re-run its own `conformance_digest()` and update the file.',
  );
  process.exit(1);
}

console.log('');
console.log(green(`conformance: the TypeScript engine matches ${rel(EXPECTED_FILE)} exactly.`));

/* ------------------------------------------------------------------ probe */

/**
 * Runs under tsx so it can import the TypeScript engine directly.
 *
 * Kept here as a string rather than as a checked-in `.mts`: it is an
 * implementation detail of this script, and a stray TypeScript file in
 * `scripts/` would be swept into a typecheck it is deliberately outside of.
 *
 * A function declaration, not a `const`, so it is hoisted above the code that
 * writes it to disk near the top of this file.
 */
function probeSource() {
  return `
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';

const [enginePath, answersPath, limitRaw] = process.argv.slice(2);
const limit = Number(limitRaw ?? 0);
const say = (value: unknown) => console.log(JSON.stringify(value));

let mod: any;
try {
  mod = await import(enginePath);
} catch (error: any) {
  say({ error: 'could not import it: ' + String(error?.message ?? error).split('\\n')[0] });
  process.exit(0);
}

const answers: string[] = JSON.parse(readFileSync(answersPath, 'utf8'));
const pool = limit > 0 ? answers.slice(0, limit) : answers;

// If the engine ships the digest itself, trust it over anything reconstructed
// here — it is the implementation's own statement of the definition.
if (typeof mod.conformanceDigest === 'function' && limit === 0) {
  const digest = await mod.conformanceDigest();
  say({ digest, via: 'engine.conformanceDigest()', pairs: pool.length * pool.length, answers: pool.length });
  process.exit(0);
}

// Argument order is the Python's: score(guess, answer). Getting it backwards
// produces a valid-looking digest that matches nothing, so if this disagrees
// with CONFORMANCE.txt, check the signature before you check the algorithm.
const NAMES = ['scoreGuess', 'score', 'scorePattern', 'scoreWord', 'feedback', 'pattern'];
const name =
  NAMES.find((n) => typeof mod[n] === 'function' && mod[n].length >= 2) ??
  NAMES.find((n) => typeof mod[n] === 'function') ??
  (typeof mod.default === 'function' ? 'default' : null);

if (!name) {
  say({ error: 'no scoring function is exported. Found: ' + Object.keys(mod).join(', ') });
  process.exit(0);
}
const score = mod[name];

/** Every spelling of green/yellow/grey this port might reasonably have chosen. */
const TILE: Record<string, string> = {
  g: 'G', y: 'Y', x: 'X',
  green: 'G', yellow: 'Y', grey: 'X', gray: 'X',
  exact: 'G', present: 'Y', absent: 'X',
  correct: 'G', misplaced: 'Y', miss: 'X', hit: 'G',
};

function normalise(value: any): string {
  if (typeof value === 'string') {
    const upper = value.toUpperCase();
    if (/^[GYX]+$/.test(upper)) return upper;
    const chars = [...value].map((c) => TILE[c.toLowerCase()]);
    if (chars.every(Boolean)) return chars.join('');
    throw new Error('unrecognised pattern string ' + JSON.stringify(value));
  }
  if (Array.isArray(value)) {
    return value
      .map((tile) => {
        const key = typeof tile === 'string' ? tile : tile?.state ?? tile?.tile ?? tile?.kind ?? tile?.result ?? tile?.status;
        const mapped = TILE[String(key).toLowerCase()];
        if (!mapped) throw new Error('unrecognised tile ' + JSON.stringify(tile));
        return mapped;
      })
      .join('');
  }
  throw new Error('unrecognised pattern ' + JSON.stringify(value));
}

let convert: (value: any) => string;
try {
  const sample = score(pool[0], pool[0]);
  // 21 million conversions: skip the whole normaliser when the engine already
  // speaks the reference alphabet, which is the case that actually ships.
  convert = typeof sample === 'string' && /^[GYX]+$/.test(sample) ? (v: any) => v : normalise;
  const selfScore = convert(sample);
  if (!/^G+$/.test(selfScore)) {
    say({ error: 'score(w, w) returned "' + selfScore + '"; a word scored against itself must be all-green' });
    process.exit(0);
  }
} catch (error: any) {
  say({ error: String(error?.message ?? error) });
  process.exit(0);
}

const hash = createHash('sha256');
const row = new Array<string>(pool.length);
for (let i = 0; i < pool.length; i += 1) {
  const answer = pool[i]!;
  for (let j = 0; j < pool.length; j += 1) row[j] = convert(score(pool[j]!, answer));
  hash.update(row.join(''));
  // Only on a TTY: written to a pipe or a CI log, a carriage return is not a
  // cursor move and the progress ends up inline with the result.
  if (i % 250 === 0 && process.stderr.isTTY) process.stderr.write('    ' + i + '/' + pool.length + ' answers\\r');
}
if (process.stderr.isTTY) process.stderr.write(' '.repeat(40) + '\\r');

say({ digest: hash.digest('hex'), via: 'engine.' + name + '(guess, answer)', pairs: pool.length * pool.length, answers: pool.length });
`;
}