PIG-Demo/scripts/check-demos.mjs

#!/usr/bin/env node
/**
 * The demo contract validator.
 *
 * `src/lib/demo-kit/types.ts` says what a demo is. TypeScript enforces the
 * shapes; this enforces everything a type cannot: that the directory name
 * matches the slug, that a `spec` demo is a real published specification rather
 * than a coming-soon card, that a `live` demo's traces exist on disk, that the
 * shared shell has no idea any particular demo exists, and that the reward has
 * something pulling against its objective.
 *
 * Thirteen numbered rules, each reported with the file to open. Run it with
 * `pnpm check`.
 *
 * Most rules are checked by READING the TypeScript, not by running it — the
 * demos import React, `?raw` Python and the Vite `@/` alias, none of which
 * survive a bare `node` import. Rule 13 is the exception: it tries the adapter
 * for real under tsx first, and says so in the output when it had to settle for
 * reading the source.
 */

import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';

import {
  DEMOS_DIR,
  Report,
  abs,
  allDemoDirs,
  demoFiles,
  demoSlugs,
  die,
  exists,
  findInDemo,
  isUnresolved,
  loadManifest,
  loadMeta,
  parseStringUnion,
  read,
  rel,
  traceFile,
  walk,
} from './_lib.mjs';

const report = new Report('check-demos');

/* ------------------------------------------------ the contract, read once */

const TYPES_FILE = abs('src', 'lib', 'demo-kit', 'types.ts');
if (!exists(TYPES_FILE)) die(`${rel(TYPES_FILE)} is missing. It is the contract; there is nothing to check against.`);
const typesSrc = read(TYPES_FILE);

// Parsed out of types.ts rather than restated here. A second copy of this list
// would drift, and it would drift silently in the direction of passing.
const VERTICALS = parseStringUnion(typesSrc, 'Vertical');
const STATUSES = parseStringUnion(typesSrc, 'DemoStatus');
const ROLES = ['objective', 'counterweight', 'gate'];
if (!VERTICALS) die(`could not parse the \`Vertical\` union out of ${rel(TYPES_FILE)}.`);
if (!STATUSES) die(`could not parse the \`DemoStatus\` union out of ${rel(TYPES_FILE)}.`);

/**
 * Every icon name lucide-react actually exports, from its own type
 * declarations. "Plausible PascalCase" would accept `Grid3X3`, which compiles
 * to `undefined` and renders as a hole in the header.
 */
const LUCIDE_ICONS = (() => {
  const dts = abs('node_modules', 'lucide-react', 'dist', 'lucide-react.d.ts');
  if (!exists(dts)) return null;
  const names = new Set();
  for (const m of read(dts).matchAll(/declare const ([A-Za-z][A-Za-z0-9_]*)\s*:/g)) names.add(m[1]);
  return names.size > 100 ? names : null;
})();

const nonEmptyString = (v) => typeof v === 'string' && v.trim().length > 0;

/** field -> validator returning `true` or the reason it is wrong. */
const META_FIELDS = {
  slug: (v) => (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(String(v)) ? true : 'must be a non-empty kebab-case string'),
  title: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
  tagline: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
  vertical: (v) => (VERTICALS.includes(v) ? true : `must be one of the Vertical union: ${VERTICALS.join(', ')}`),
  status: (v) => (STATUSES.includes(v) ? true : `must be one of the DemoStatus union: ${STATUSES.join(', ')}`),
  order: (v) => (typeof v === 'number' && Number.isFinite(v) ? true : 'must be a finite number'),
  icon: (v) => {
    if (!nonEmptyString(v)) return 'must be a non-empty string';
    if (!/^[A-Z][A-Za-z0-9]*$/.test(v)) return `"${v}" is not PascalCase, so it is not a lucide export name`;
    if (LUCIDE_ICONS && !LUCIDE_ICONS.has(v)) return `lucide-react does not export "${v}" (check the exact casing at lucide.dev)`;
    return true;
  },
  persona: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
  rewardLine: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
  ogImage: (v) => (nonEmptyString(v) && v.startsWith('/') ? true : "must be a site-absolute path, e.g. '/og/wordle.png'"),
};

/* --------------------------------------------------------- literal anchors */

const COMPONENT_ANCHORS = [/\bcomponents\s*:\s*/];
const ANATOMY_ANCHORS = [/(?:export\s+)?const\s+anatomy\s*(?::\s*[^=]+)?=\s*/, /\banatomy\s*:\s*/];
const PROVENANCE_ANCHORS = [
  /(?:export\s+)?const\s+provenance\s*(?::\s*[^=]+)?=\s*/,
  /\bprovenance\s*:\s*/,
];

/* ------------------------------------------------------------- the checks */

const slugs = demoSlugs();
if (slugs.length === 0) die(`no demos found under ${rel(DEMOS_DIR)}. A repo with no demos has nothing to validate.`);

const manifest = loadManifest();

for (const slug of slugs) {
  const dir = path.join(DEMOS_DIR, slug);
  const found = loadMeta(slug);

  if ('error' in found) {
    report.fail(found.file, 'rule 2 (DemoMeta present)', found.error);
    continue;
  }
  const { meta, file: metaFile } = found;

  /* -- 1. directory name is the slug ------------------------------------- */
  report.check(
    meta.slug === slug,
    metaFile,
    'rule 1 (slug is the directory name)',
    `meta.slug is ${JSON.stringify(meta.slug)} but the directory is ${JSON.stringify(slug)}. ` +
      'The registry, the route and the OG card are all keyed on this; rename one of them.',
  );

  /* -- 2. every required field, correctly typed ---------------------------- */
  for (const [field, validate] of Object.entries(META_FIELDS)) {
    if (!(field in meta)) {
      report.fail(metaFile, 'rule 2 (DemoMeta complete)', `\`${field}\` is missing.`);
      continue;
    }
    const verdict = validate(meta[field]);
    report.check(verdict === true, metaFile, 'rule 2 (DemoMeta complete)', `\`${field}\`: ${verdict}`);
  }
  if (!LUCIDE_ICONS) {
    report.staticOnly(`${slug}: lucide-react is not installed, so \`icon\` was only checked for PascalCase shape.`);
  }

  /* -- 4. the social card exists ------------------------------------------ */
  if (nonEmptyString(meta.ogImage)) {
    const card = abs('public', String(meta.ogImage).replace(/^\/+/, ''));
    report.check(
      exists(card),
      metaFile,
      'rule 4 (ogImage exists)',
      `meta.ogImage points at ${meta.ogImage}, which is ${rel(card)} on disk, and that file does not exist. ` +
        'Generate it with `node scripts/og.mjs` (x86 only — it needs Playwright chromium).',
    );
  }

  /* -- 5. demo.tsx default-exports via defineDemo ------------------------- */
  const demoFile = path.join(dir, 'demo.tsx');
  if (!exists(demoFile)) {
    report.fail(rel(demoFile), 'rule 5 (defineDemo default export)', 'demo.tsx does not exist.');
  } else {
    const src = read(demoFile);
    report.check(
      /export\s+default\s+defineDemo\s*(?:<[^>]*>)?\s*\(/.test(src),
      rel(demoFile),
      'rule 5 (defineDemo default export)',
      'no `export default defineDemo(...)` found. The registry loads demos through defineDemo; ' +
        'a bare object default export skips whatever the kit validates.',
    );
    report.check(
      /\bdefineDemo\b[\s\S]*?from\s*'@\/lib\/demo-kit'/.test(src) || /from\s*'@\/lib\/demo-kit'[\s\S]*?\bdefineDemo\b/.test(src),
      rel(demoFile),
      'rule 5 (defineDemo default export)',
      "defineDemo must be imported from '@/lib/demo-kit'.",
    );
  }

  /* -- 6/10. the specification and the reward ----------------------------- */
  const anatomy = findInDemo(slug, ANATOMY_ANCHORS, '{', 'anatomy', { lenient: true });
  const provenance = findInDemo(slug, PROVENANCE_ANCHORS, '{', 'provenance', { lenient: true });
  const components = findInDemo(slug, COMPONENT_ANCHORS, '[', 'reward.components', { lenient: true });

  const specRule = 'rule 6 (complete specification)';
  if (anatomy.error) {
    report.fail(anatomy.file ?? rel(dir), specRule, anatomy.error);
  } else {
    for (const field of ['task', 'actions', 'grader', 'score']) {
      report.check(
        nonEmptyString(anatomy.value?.[field]),
        anatomy.file,
        specRule,
        `anatomy.${field} is empty. A ${meta.status} demo publishes the four boxes in full; ` +
          'an empty one reads as a placeholder and undoes the page it sits on.',
      );
    }
  }

  if (provenance.error) {
    report.fail(provenance.file ?? rel(dir), specRule, provenance.error);
  } else {
    report.check(
      nonEmptyString(provenance.value?.command),
      provenance.file,
      specRule,
      'provenance.command is empty. The eval command is the reader\'s way to disprove the page; ' +
        'it is the one field that must never be aspirational.',
    );
  }

  const weightRule = 'rule 10 (reward weights)';
  if (components.error) {
    report.fail(components.file ?? rel(dir), weightRule, components.error);
  } else if (!Array.isArray(components.value) || components.value.length === 0) {
    report.fail(components.file, weightRule, 'reward.components is not a non-empty array.');
  } else {
    const list = components.value;
    let sum = 0;
    let readable = true;
    list.forEach((component, i) => {
      const where = `reward.components[${i}]${component?.key ? ` (${component.key})` : ''}`;
      for (const field of ['key', 'label', 'description']) {
        report.check(
          nonEmptyString(component?.[field]),
          components.file,
          weightRule,
          `${where}: \`${field}\` is empty.`,
        );
      }
      if (typeof component?.weight !== 'number' || !Number.isFinite(component.weight)) {
        readable = false;
        report.fail(
          components.file,
          weightRule,
          `${where}: \`weight\` is ${isUnresolved(component?.weight) ? 'an imported constant' : JSON.stringify(component?.weight)}, ` +
            'not a literal number. Weights are read off the page beside the code, so they are written out here.',
        );
      } else {
        sum += component.weight;
      }
      report.check(
        ROLES.includes(component?.role),
        components.file,
        weightRule,
        `${where}: \`role\` must be one of ${ROLES.join(', ')}.`,
      );
    });

    if (readable) {
      report.check(
        Math.abs(sum - 1) <= 1e-9,
        components.file,
        weightRule,
        `the weights sum to ${sum} and must sum to 1.0 (within 1e-9). ` +
          'A reward whose weights do not sum to one is not the reward the page shows you tuning.',
      );
    }
    report.check(
      list.some((c) => c?.role === 'counterweight'),
      components.file,
      'rule 10 (counterweight required)',
      'no component has role "counterweight". An objective with nothing pulling against it is a metric to ' +
        'game, and the demo exists to show the opposite. If the second term is one every good policy also ' +
        'scores 1.0 on, it is a gate — and the demo still needs a real counterweight.',
    );
    const counterweight = list.find((c) => c?.role === 'counterweight');
    if (counterweight) {
      report.check(
        nonEmptyString(counterweight.description),
        components.file,
        specRule,
        `the counterweight (${counterweight.key}) has no description. It is the half of the reward a reader ` +
          'does not expect, so it is the half that must be spelled out.',
      );
    }
  }

  /* -- 7/11/12. recorded runs --------------------------------------------- */
  if (meta.status === 'live') {
    const runRule = 'rule 7 (live demo has runs)';
    if (manifest.error) {
      report.fail(rel(abs('public', 'traces', 'manifest.json')), runRule, manifest.error);
    } else {
      const entry = manifest.byDemo.get(slug);
      const runs = entry?.runs ?? [];
      if (runs.length === 0) {
        report.fail(
          rel(abs('public', 'traces', 'manifest.json')),
          runRule,
          `demo "${slug}" is status "live" but has no runs in the manifest. ` +
            'A live demo asserts recorded evidence; without a run there is nothing to replay.',
        );
      } else {
        const episodes = [];
        for (const run of runs) {
          const label = `run ${run?.id ?? '(no id)'}`;
          if (!nonEmptyString(run?.path)) {
            report.fail(rel(abs('public', 'traces', 'manifest.json')), runRule, `${label}: \`path\` is empty.`);
            continue;
          }
          const file = traceFile(run.path);
          if (!report.check(exists(file), rel(file), runRule, `${label} points at ${run.path}, which does not exist on disk.`)) {
            continue;
          }
          try {
            const raw = JSON.parse(read(file));
            for (const ep of Array.isArray(raw) ? raw : Array.isArray(raw?.episodes) ? raw.episodes : [raw]) {
              episodes.push({ run, file, episode: ep });
            }
          } catch (error) {
            report.fail(rel(file), runRule, `not valid JSON: ${error.message}`);
          }

          /* -- 12. an intervened run must say what was done to it ---------- */
          if (run?.kind === 'intervened') {
            report.check(
              nonEmptyString(run.intervention),
              rel(abs('public', 'traces', 'manifest.json')),
              'rule 12 (intervened runs declare the intervention)',
              `${label} has kind "intervened" but no \`intervention\`. Without it a prompt change reads as a ` +
                'training result by omission, which is the single most misleading thing this page could do.',
            );
          }
        }

        /* -- 11. a failure, or a solve rate the page renders -------------- */
        const failedRun = episodes.find((e) => e.episode?.outcome === 'failed');
        if (!failedRun && episodes.length) {
          const rateSources = [entry?.extras ?? {}, ...runs];
          const rate = rateSources
            .map((s) => s?.solveRate ?? s?.solve_rate)
            .find((v) => typeof v === 'number' && Number.isFinite(v));
          const rendered = walk(abs('src'), (f) => /\.tsx?$/.test(f)).some((f) => /\bsolve[_R]?ate|solveRate|solve_rate/.test(read(f)));
          report.check(
            typeof rate === 'number' && rendered,
            rel(abs('public', 'traces', 'manifest.json')),
            'rule 11 (a clean sweep is reported, not hidden)',
            `demo "${slug}" ships no run with outcome "failed". That is allowed only if the page states the ` +
              'solve rate: add a numeric `solveRate` beside the runs in the manifest AND render it. ' +
              `Right now solveRate is ${typeof rate === 'number' ? rate : 'absent'} and a renderer for it was ` +
              `${rendered ? 'found' : 'not found'} under src/. Hunting for a losing seed to make the demo look ` +
              'honest is the failure mode this rule exists to block.',
          );
        }
      }
    }
  }

  /* -- 13. every step announces itself ------------------------------------ */
  checkAnnounce(slug, report, manifest);
}

/* ---------------------------------------- 8. the shell knows no demo names */

const SHELL_DIR = abs('src', 'components', 'demo');
for (const file of walk(SHELL_DIR, (f) => /\.(tsx?|css)$/.test(f))) {
  const lines = read(file).split('\n');
  for (const slug of allDemoDirs()) {
    if (slug.startsWith('_')) continue;
    // Word-boundary on both sides so a slug like `wordle` is not found inside
    // an unrelated identifier, and so `wordle-five` matches as one token.
    const needle = new RegExp(`(?<![\\w-])${slug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`);
    const hit = lines.findIndex((line) => needle.test(line));
    if (hit !== -1) {
      report.fail(
        `${rel(file)}:${hit + 1}`,
        'rule 8 (the shell is generic)',
        `mentions the demo slug "${slug}". Nothing under src/components/demo/ may know which demo it is ` +
          'rendering. If the shell needs to special-case something, that is a missing slot on DemoModule or a ' +
          'missing capability in demo-kit — fix it there and every future demo gets it too.',
      );
    } else {
      report.passed += 1;
    }
  }
}

/* --------------------------------- 9. demos reach only for the kit's barrel */

const FORBIDDEN_IMPORTS = [
  {
    test: (spec) => /^@\/lib\/demo-kit\/.+/.test(spec),
    why: "imports a file INSIDE demo-kit. Import '@/lib/demo-kit' — the barrel is the contract, and anything " +
      'you can only get by reaching past it is either private or belongs in the barrel.',
  },
  {
    test: (spec) => /^@\/components\/demo(\/|$)/.test(spec),
    why: 'imports the shared shell. The shell renders demos; demos never render the shell. This is the ' +
      'dependency that, once it exists, makes the shell impossible to change without opening every demo.',
  },
];

for (const slug of allDemoDirs()) {
  for (const file of demoFiles(slug)) {
    for (const spec of importSpecifiers(read(file))) {
      const normalised = normaliseSpecifier(spec, file);
      for (const rule of FORBIDDEN_IMPORTS) {
        if (rule.test(normalised)) {
          report.fail(
            rel(file),
            'rule 9 (demos import only the demo-kit barrel)',
            `\`${spec}\`${normalised === spec ? '' : ` (resolves to ${normalised})`} ${rule.why}`,
          );
        } else {
          report.passed += 1;
        }
      }
    }
  }
}

report.finish();

/* ------------------------------------------------------------- helpers */

/** Every module specifier a file imports, static, dynamic or side-effect. */
function importSpecifiers(src) {
  const out = new Set();
  for (const m of src.matchAll(/\bfrom\s*['"]([^'"]+)['"]/g)) out.add(m[1]);
  for (const m of src.matchAll(/\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g)) out.add(m[1]);
  for (const m of src.matchAll(/^\s*import\s+['"]([^'"]+)['"]/gm)) out.add(m[1]);
  return [...out];
}

/**
 * Rewrites a relative specifier into its `@/` form.
 *
 * Without this, `../../components/demo/Player` walks straight past a lint that
 * only looks at the alias — and it is the form an editor's auto-import
 * produces, so it is the form the violation actually arrives in.
 */
function normaliseSpecifier(spec, fromFile) {
  if (!spec.startsWith('.')) return spec;
  const resolved = path.resolve(path.dirname(fromFile), spec);
  const inSrc = path.relative(abs('src'), resolved);
  if (inSrc.startsWith('..')) return spec;
  return `@/${inSrc.split(path.sep).join('/')}`.replace(/\.(tsx?|jsx?)$/, '');
}

/**
 * Rule 13: every `DemoStep` an adapter produces sets a non-empty `announce`.
 *
 * Reduced motion clamps the tile flip to nothing, so for a screen-reader user
 * the announcement IS the result. A missing one is not a degraded experience,
 * it is a blank page that claims to be showing you something.
 *
 * Tried dynamically first, under tsx, against the demo's own recorded traces —
 * a static check cannot see through a helper that builds the step object. When
 * the module will not load in bare Node (a `?raw` import, a Vite-only alias,
 * JSX pulling in something browser-shaped) it falls back to reading the source
 * and SAYS SO in the report rather than quietly downgrading.
 */
function checkAnnounce(slug, report, manifest) {
  const rule = 'rule 13 (every step announces itself)';
  const dir = path.join(DEMOS_DIR, slug);
  const candidates = ['adapt.ts', 'adapter.ts', 'adapters.ts', 'adapt.tsx', 'demo.tsx']
    .map((n) => path.join(dir, n))
    .filter((p) => exists(p) && /\badapt\b\s*[:=(]|function\s+adapt\b/.test(read(p)));

  if (candidates.length === 0) {
    report.fail(
      rel(dir),
      rule,
      'no `adapt` implementation found. DemoModule.adapt is required; it is what turns a recorded episode ' +
        'into the steps the player renders.',
    );
    return;
  }

  const traces = (manifest.byDemo.get(slug)?.runs ?? [])
    .map((run) => (run?.path ? traceFile(run.path) : null))
    .filter((p) => p && exists(p));

  if (traces.length > 0) {
    for (const module of candidates) {
      const result = probeAdapter(module, traces);
      if (result.loaded) {
        if (result.ok) {
          report.passed += 1;
        } else {
          for (const problem of result.problems) report.fail(rel(module), rule, problem);
        }
        return;
      }
    }
  }

  // Static fallback. It cannot prove `announce` is set on every step, only that
  // the adapter sets it at all and never sets it to nothing.
  const module = candidates[0];
  const src = read(module);
  const assigns = /\bannounce\s*:/.test(src);
  report.check(
    assigns,
    rel(module),
    rule,
    'the adapter never assigns `announce`. Every DemoStep must carry one — with motion reduced, it is the ' +
      'only thing that reports the result.',
  );
  if (assigns) {
    const empty = /\bannounce\s*:\s*(?:''|""|``|null|undefined)\s*[,}\n]/.test(src);
    report.check(!empty, rel(module), rule, 'the adapter assigns an empty `announce` somewhere. An empty announcement is a missing one.');
  }
  report.staticOnly(
    `${slug}: rule 13 was checked by reading ${rel(module)}, not by running the adapter` +
      `${traces.length === 0 ? ' (no recorded traces to feed it)' : ' (the module would not load under tsx in bare Node)'}.`,
  );
}

/** Runs one adapter over real episodes under tsx. Never throws. */
function probeAdapter(modulePath, traces) {
  const tsx = abs('node_modules', '.bin', 'tsx');
  if (!exists(tsx)) return { loaded: false, reason: 'tsx is not installed' };

  const probe = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'pig-adapt-')), 'probe.mts');
  fs.writeFileSync(probe, adapterProbeSource(), 'utf8');
  try {
    const run = spawnSync(process.execPath, [tsx, probe, modulePath, ...traces], {
      cwd: abs('.'),
      encoding: 'utf8',
      timeout: 30_000,
      // tsx resolves the `@/` alias from a tsconfig, and the probe lives in
      // a temp dir where it would never find ours.
      env: { ...process.env, TSX_TSCONFIG_PATH: abs('tsconfig.json') },
    });
    if (run.status !== 0 || !run.stdout) return { loaded: false, reason: (run.stderr || '').trim().split('\n').slice(-1)[0] };
    const line = run.stdout.trim().split('\n').pop();
    const parsed = JSON.parse(line);
    return parsed.loaded === false ? { loaded: false, reason: parsed.reason } : { loaded: true, ...parsed };
  } catch {
    return { loaded: false, reason: 'the probe did not produce parseable output' };
  } finally {
    fs.rmSync(path.dirname(probe), { recursive: true, force: true });
  }
}

/**
 * The probe, written to a temp file and executed by tsx.
 *
 * It lives here as a string rather than as a checked-in `.mts` because it is an
 * implementation detail of this script, and a stray TypeScript file in
 * `scripts/` would get swept into the typecheck it is deliberately outside of.
 */
function adapterProbeSource() {
  return `
import { readFileSync } from 'node:fs';

const [modulePath, ...traces] = process.argv.slice(2);
const say = (o: unknown) => console.log(JSON.stringify(o));

let mod: any;
try {
  mod = await import(modulePath);
} catch (error: any) {
  say({ loaded: false, reason: String(error?.message ?? error).split('\\n')[0] });
  process.exit(0);
}

const adapt =
  typeof mod.adapt === 'function' ? mod.adapt :
  typeof mod.default?.adapt === 'function' ? mod.default.adapt :
  typeof mod.default === 'function' ? mod.default : null;

if (!adapt) {
  say({ loaded: false, reason: 'no callable \\\`adapt\\\` export' });
  process.exit(0);
}

const problems: string[] = [];
let steps = 0;

for (const trace of traces) {
  const raw = JSON.parse(readFileSync(trace, 'utf8'));
  const episodes = Array.isArray(raw) ? raw : Array.isArray(raw?.episodes) ? raw.episodes : [raw];
  for (const episode of episodes) {
    let produced: any;
    try {
      produced = adapt(episode);
    } catch (error: any) {
      problems.push(trace + ': adapt() threw ' + String(error?.message ?? error).split('\\n')[0]);
      continue;
    }
    if (!Array.isArray(produced)) {
      problems.push(trace + ': adapt() returned ' + typeof produced + ', not an array of DemoStep');
      continue;
    }
    produced.forEach((step: any, i: number) => {
      steps += 1;
      if (typeof step?.announce !== 'string' || step.announce.trim() === '') {
        problems.push(trace + ': step ' + i + ' has an empty \\\`announce\\\`');
      }
    });
  }
}

say({ loaded: true, ok: problems.length === 0, problems: problems.slice(0, 10), steps });
`;
}