Capture harness, fixture verification, CI, and the public README
The site does no live inference. Rollouts are captured once against spark-1 and replayed at their recorded wall-clock — a public demo with no auth cannot hold an API key, and a recorded run can be scrubbed, permalinked, blind-compared and verified in ways a live one cannot. What stops it being a video is that the browser re-derives every number from the recorded moves. verify_fixtures.py is the Python half of that: it replays every committed fixture through the engine and reproduces its own rewards. All 16 land at delta 0.0. A fixture that cannot be regenerated is a claim with no receipt. First real measurement, thinking off, 8 seeds: solved 0/8. The model repeats guesses it has already played, invents words (trape, slith, postt, boomy), and contradicts its own feedback — consistency 0.09 to 0.17. That is the published failure taxonomy showing up in our own data on the first run, and it is why `consistency` is a reward component rather than a footnote. A capture failure is recorded as a turn with a null reply, never dropped. A capture that silently discarded failed turns would be reporting a better model than the one that ran. CI gates both halves and four things that fail silently in production: the word lists must rebuild byte-identically, the prerendered routes must carry their own baked og tags (crawlers do not run JS, so without them every shared link previews as the homepage), no blob: URL may reach the bundle (the site's CSP has no worker-src, so it falls back to default-src 'self' and a blob worker is blocked with no error), and the conformance digest must match across languages. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
+58
-18
@@ -294,7 +294,7 @@ function codeSpans(src) {
|
||||
* @param {'{' | '['} open
|
||||
* @returns {string | null}
|
||||
*/
|
||||
export function literalAfter(src, anchor, open = '{') {
|
||||
export function literalsAfter(src, anchor, open = '{') {
|
||||
const flags = anchor.flags.includes('g') ? anchor.flags : `${anchor.flags}g`;
|
||||
const re = new RegExp(anchor.source, flags);
|
||||
const spans = codeSpans(src);
|
||||
@@ -302,18 +302,29 @@ export function literalAfter(src, anchor, open = '{') {
|
||||
const hit = spans.find((s) => offset >= s.start && offset < s.end);
|
||||
return hit ? hit.code : true;
|
||||
};
|
||||
const found = [];
|
||||
let match;
|
||||
while ((match = re.exec(src)) !== null) {
|
||||
// The anchor may legally match inside a comment or a string — a docblock
|
||||
// The anchor may legally match inside a comment or a string \u2014 a docblock
|
||||
// that quotes `const meta = {`. Only a match in real code counts.
|
||||
if (!isCode(match.index)) continue;
|
||||
const from = src.indexOf(open, match.index + Math.max(match[0].length - 1, 0));
|
||||
if (from === -1) continue;
|
||||
const between = src.slice(match.index + match[0].length, from);
|
||||
if (/[;=}]/.test(between)) continue;
|
||||
return src.slice(from, scanBalanced(src, from));
|
||||
const end = scanBalanced(src, from);
|
||||
found.push({ text: src.slice(from, end), start: from, end });
|
||||
// Skip past this literal so a nested anchor of the same name is not
|
||||
// reported as a second, overlapping hit.
|
||||
re.lastIndex = end;
|
||||
}
|
||||
return null;
|
||||
return found;
|
||||
}
|
||||
|
||||
/** First literal following `anchor`, or null. See {@link literalsAfter}. */
|
||||
export function literalAfter(src, anchor, open = '{') {
|
||||
const [first] = literalsAfter(src, anchor, open);
|
||||
return first ? first.text : null;
|
||||
}
|
||||
|
||||
/** Marker key on the stand-in an unresolvable identifier evaluates to. */
|
||||
@@ -501,7 +512,10 @@ const VERTICAL_ANCHORS = [
|
||||
const asVertical = (id, v) => ({
|
||||
id,
|
||||
title: v?.title ?? v?.label ?? v?.name ?? null,
|
||||
description: v?.description ?? v?.blurb ?? v?.tagline ?? v?.summary ?? null,
|
||||
// `anxiety` is this repo's field for the one line a vertical page leads
|
||||
// with, and it is the only prose in a VerticalEntry short enough to be a
|
||||
// meta description.
|
||||
description: v?.description ?? v?.blurb ?? v?.tagline ?? v?.summary ?? v?.anxiety ?? null,
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -559,8 +573,14 @@ export function loadManifest() {
|
||||
}
|
||||
/** @type {Map<string, {runs: any[], extras: Record<string, any>}>} */
|
||||
const byDemo = new Map();
|
||||
const container =
|
||||
raw && typeof raw === 'object' && !Array.isArray(raw) && raw.demos && typeof raw.demos === 'object' ? raw.demos : raw;
|
||||
// Unwrap the two envelopes the shell's own reader accepts. `{runs: [...]}`
|
||||
// has to be unwrapped BEFORE the record branch below, or "runs" is read as a
|
||||
// demo slug and every real demo reports zero runs.
|
||||
let container = raw;
|
||||
if (raw && typeof raw === 'object' && !Array.isArray(raw)) {
|
||||
if (raw.demos && typeof raw.demos === 'object') container = raw.demos;
|
||||
else if (Array.isArray(raw.runs)) container = raw.runs;
|
||||
}
|
||||
|
||||
if (Array.isArray(container)) {
|
||||
// A flat array of runs, each carrying its own `demo`/`slug`.
|
||||
@@ -706,12 +726,32 @@ export function discoverRoutePatterns() {
|
||||
return [...patterns];
|
||||
}
|
||||
|
||||
const DEMO_PARAMS = /^(slug|demo|demoSlug)$/;
|
||||
const VERTICAL_PARAMS = /^(vertical|verticalId|sector|category)$/;
|
||||
/**
|
||||
* Which set fills a pattern's one parameter.
|
||||
*
|
||||
* Decided by the STATIC segment in front of it, not by the parameter's name.
|
||||
* This router calls both of them `:slug` — `demos/:slug` and `verticals/:slug`
|
||||
* — so a name-based reader confidently prerenders `/verticals/wordle`, twelve
|
||||
* 404s, and no vertical pages at all.
|
||||
*/
|
||||
function fillFor(pattern) {
|
||||
const head = pattern.replace(/^\/+/, '').split('/')[0]?.toLowerCase() ?? '';
|
||||
if (/^(demos?|d)$/.test(head)) return 'demo';
|
||||
if (/^(verticals?|v|sectors?)$/.test(head)) return 'vertical';
|
||||
const param = pattern.match(/:([A-Za-z0-9_]+)/)?.[1] ?? '';
|
||||
if (/^(vertical|verticalId|sector|category)$/.test(param)) return 'vertical';
|
||||
if (/^(slug|demo|demoSlug|id)$/.test(param)) return 'demo';
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expands the router's patterns against the real data into concrete paths.
|
||||
*
|
||||
* Child paths are taken as written. This router nests everything exactly one
|
||||
* level under `/`, so a child's `path` is already the full path; a second level
|
||||
* of nesting would need the parent prefix joined on, and this would silently
|
||||
* emit the wrong routes rather than fail. If you nest deeper, fix this.
|
||||
*
|
||||
* @param {{metas: Map<string, any>, verticals: {id: string}[]}} data
|
||||
* @returns {{routes: {path: string, kind: string, slug?: string, id?: string}[], errors: string[]}}
|
||||
*/
|
||||
@@ -749,18 +789,18 @@ export function expandRoutes(data) {
|
||||
errors.push(`route pattern "${pattern}" has more than one parameter; prerender cannot expand it.`);
|
||||
continue;
|
||||
}
|
||||
const param = params[0];
|
||||
const fill = (value, record) => add(pattern.replace(/:[A-Za-z0-9_]+\??/, value), record);
|
||||
if (VERTICAL_PARAMS.test(param)) {
|
||||
if (!verticalIds.length) errors.push(`route "${pattern}" needs verticals, but none were readable from src/content/verticals.ts.`);
|
||||
for (const id of verticalIds) fill(id, { kind: 'vertical', id });
|
||||
} else if (DEMO_PARAMS.test(param) || param === 'id') {
|
||||
const fill = fillFor(pattern);
|
||||
const substitute = (value) => pattern.replace(/:[A-Za-z0-9_]+\??/, value);
|
||||
if (fill === 'vertical') {
|
||||
if (!verticalIds.length) errors.push(`route "${pattern}" needs verticals, but none were readable from ${rel(VERTICALS_FILE)}.`);
|
||||
for (const id of verticalIds) add(substitute(id), { kind: 'vertical', id });
|
||||
} else if (fill === 'demo') {
|
||||
if (!demoSlugList.length) errors.push(`route "${pattern}" needs demos, but no demo meta was readable under src/demos/.`);
|
||||
for (const slug of demoSlugList) fill(slug, { kind: 'demo', slug });
|
||||
for (const slug of demoSlugList) add(substitute(slug), { kind: 'demo', slug });
|
||||
} else {
|
||||
errors.push(
|
||||
`route pattern "${pattern}" uses parameter ":${param}", which prerender cannot fill. ` +
|
||||
'Name it :slug (a demo) or :vertical (a vertical), or teach scripts/_lib.mjs about it.',
|
||||
`route pattern "${pattern}" has a parameter prerender cannot fill. Put it under /demos/ or ` +
|
||||
'/verticals/, or teach fillFor() in scripts/_lib.mjs about it.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,597 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* The demo contract validator.
|
||||
*
|
||||
* `src/lib/demo-kit/types.ts` says what a demo is. TypeScript enforces the
|
||||
* shapes; this enforces everything a type cannot: that the directory name
|
||||
* matches the slug, that a `spec` demo is a real published specification rather
|
||||
* than a coming-soon card, that a `live` demo's traces exist on disk, that the
|
||||
* shared shell has no idea any particular demo exists, and that the reward has
|
||||
* something pulling against its objective.
|
||||
*
|
||||
* Thirteen numbered rules, each reported with the file to open. Run it with
|
||||
* `pnpm check`.
|
||||
*
|
||||
* Most rules are checked by READING the TypeScript, not by running it — the
|
||||
* demos import React, `?raw` Python and the Vite `@/` alias, none of which
|
||||
* survive a bare `node` import. Rule 13 is the exception: it tries the adapter
|
||||
* for real under tsx first, and says so in the output when it had to settle for
|
||||
* reading the source.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
import {
|
||||
DEMOS_DIR,
|
||||
Report,
|
||||
abs,
|
||||
allDemoDirs,
|
||||
demoFiles,
|
||||
demoSlugs,
|
||||
die,
|
||||
exists,
|
||||
findInDemo,
|
||||
isUnresolved,
|
||||
loadManifest,
|
||||
loadMeta,
|
||||
parseStringUnion,
|
||||
read,
|
||||
rel,
|
||||
traceFile,
|
||||
walk,
|
||||
} from './_lib.mjs';
|
||||
|
||||
const report = new Report('check-demos');
|
||||
|
||||
/* ------------------------------------------------ the contract, read once */
|
||||
|
||||
const TYPES_FILE = abs('src', 'lib', 'demo-kit', 'types.ts');
|
||||
if (!exists(TYPES_FILE)) die(`${rel(TYPES_FILE)} is missing. It is the contract; there is nothing to check against.`);
|
||||
const typesSrc = read(TYPES_FILE);
|
||||
|
||||
// Parsed out of types.ts rather than restated here. A second copy of this list
|
||||
// would drift, and it would drift silently in the direction of passing.
|
||||
const VERTICALS = parseStringUnion(typesSrc, 'Vertical');
|
||||
const STATUSES = parseStringUnion(typesSrc, 'DemoStatus');
|
||||
const ROLES = ['objective', 'counterweight', 'gate'];
|
||||
if (!VERTICALS) die(`could not parse the \`Vertical\` union out of ${rel(TYPES_FILE)}.`);
|
||||
if (!STATUSES) die(`could not parse the \`DemoStatus\` union out of ${rel(TYPES_FILE)}.`);
|
||||
|
||||
/**
|
||||
* Every icon name lucide-react actually exports, from its own type
|
||||
* declarations. "Plausible PascalCase" would accept `Grid3X3`, which compiles
|
||||
* to `undefined` and renders as a hole in the header.
|
||||
*/
|
||||
const LUCIDE_ICONS = (() => {
|
||||
const dts = abs('node_modules', 'lucide-react', 'dist', 'lucide-react.d.ts');
|
||||
if (!exists(dts)) return null;
|
||||
const names = new Set();
|
||||
for (const m of read(dts).matchAll(/declare const ([A-Za-z][A-Za-z0-9_]*)\s*:/g)) names.add(m[1]);
|
||||
return names.size > 100 ? names : null;
|
||||
})();
|
||||
|
||||
const nonEmptyString = (v) => typeof v === 'string' && v.trim().length > 0;
|
||||
|
||||
/** field -> validator returning `true` or the reason it is wrong. */
|
||||
const META_FIELDS = {
|
||||
slug: (v) => (/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(String(v)) ? true : 'must be a non-empty kebab-case string'),
|
||||
title: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
|
||||
tagline: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
|
||||
vertical: (v) => (VERTICALS.includes(v) ? true : `must be one of the Vertical union: ${VERTICALS.join(', ')}`),
|
||||
status: (v) => (STATUSES.includes(v) ? true : `must be one of the DemoStatus union: ${STATUSES.join(', ')}`),
|
||||
order: (v) => (typeof v === 'number' && Number.isFinite(v) ? true : 'must be a finite number'),
|
||||
icon: (v) => {
|
||||
if (!nonEmptyString(v)) return 'must be a non-empty string';
|
||||
if (!/^[A-Z][A-Za-z0-9]*$/.test(v)) return `"${v}" is not PascalCase, so it is not a lucide export name`;
|
||||
if (LUCIDE_ICONS && !LUCIDE_ICONS.has(v)) return `lucide-react does not export "${v}" (check the exact casing at lucide.dev)`;
|
||||
return true;
|
||||
},
|
||||
persona: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
|
||||
rewardLine: (v) => (nonEmptyString(v) ? true : 'must be a non-empty string'),
|
||||
ogImage: (v) => (nonEmptyString(v) && v.startsWith('/') ? true : "must be a site-absolute path, e.g. '/og/wordle.png'"),
|
||||
};
|
||||
|
||||
/* --------------------------------------------------------- literal anchors */
|
||||
|
||||
const COMPONENT_ANCHORS = [/\bcomponents\s*:\s*/];
|
||||
const ANATOMY_ANCHORS = [/(?:export\s+)?const\s+anatomy\s*(?::\s*[^=]+)?=\s*/, /\banatomy\s*:\s*/];
|
||||
const PROVENANCE_ANCHORS = [
|
||||
/(?:export\s+)?const\s+provenance\s*(?::\s*[^=]+)?=\s*/,
|
||||
/\bprovenance\s*:\s*/,
|
||||
];
|
||||
|
||||
/* ------------------------------------------------------------- the checks */
|
||||
|
||||
const slugs = demoSlugs();
|
||||
if (slugs.length === 0) die(`no demos found under ${rel(DEMOS_DIR)}. A repo with no demos has nothing to validate.`);
|
||||
|
||||
const manifest = loadManifest();
|
||||
|
||||
for (const slug of slugs) {
|
||||
const dir = path.join(DEMOS_DIR, slug);
|
||||
const found = loadMeta(slug);
|
||||
|
||||
if ('error' in found) {
|
||||
report.fail(found.file, 'rule 2 (DemoMeta present)', found.error);
|
||||
continue;
|
||||
}
|
||||
const { meta, file: metaFile } = found;
|
||||
|
||||
/* -- 1. directory name is the slug ------------------------------------- */
|
||||
report.check(
|
||||
meta.slug === slug,
|
||||
metaFile,
|
||||
'rule 1 (slug is the directory name)',
|
||||
`meta.slug is ${JSON.stringify(meta.slug)} but the directory is ${JSON.stringify(slug)}. ` +
|
||||
'The registry, the route and the OG card are all keyed on this; rename one of them.',
|
||||
);
|
||||
|
||||
/* -- 2. every required field, correctly typed ---------------------------- */
|
||||
for (const [field, validate] of Object.entries(META_FIELDS)) {
|
||||
if (!(field in meta)) {
|
||||
report.fail(metaFile, 'rule 2 (DemoMeta complete)', `\`${field}\` is missing.`);
|
||||
continue;
|
||||
}
|
||||
const verdict = validate(meta[field]);
|
||||
report.check(verdict === true, metaFile, 'rule 2 (DemoMeta complete)', `\`${field}\`: ${verdict}`);
|
||||
}
|
||||
if (!LUCIDE_ICONS) {
|
||||
report.staticOnly(`${slug}: lucide-react is not installed, so \`icon\` was only checked for PascalCase shape.`);
|
||||
}
|
||||
|
||||
/* -- 4. the social card exists ------------------------------------------ */
|
||||
if (nonEmptyString(meta.ogImage)) {
|
||||
const card = abs('public', String(meta.ogImage).replace(/^\/+/, ''));
|
||||
report.check(
|
||||
exists(card),
|
||||
metaFile,
|
||||
'rule 4 (ogImage exists)',
|
||||
`meta.ogImage points at ${meta.ogImage}, which is ${rel(card)} on disk, and that file does not exist. ` +
|
||||
'Generate it with `node scripts/og.mjs` (x86 only — it needs Playwright chromium).',
|
||||
);
|
||||
}
|
||||
|
||||
/* -- 5. demo.tsx default-exports via defineDemo ------------------------- */
|
||||
const demoFile = path.join(dir, 'demo.tsx');
|
||||
if (!exists(demoFile)) {
|
||||
report.fail(rel(demoFile), 'rule 5 (defineDemo default export)', 'demo.tsx does not exist.');
|
||||
} else {
|
||||
const src = read(demoFile);
|
||||
report.check(
|
||||
/export\s+default\s+defineDemo\s*(?:<[^>]*>)?\s*\(/.test(src),
|
||||
rel(demoFile),
|
||||
'rule 5 (defineDemo default export)',
|
||||
'no `export default defineDemo(...)` found. The registry loads demos through defineDemo; ' +
|
||||
'a bare object default export skips whatever the kit validates.',
|
||||
);
|
||||
report.check(
|
||||
/\bdefineDemo\b[\s\S]*?from\s*'@\/lib\/demo-kit'/.test(src) || /from\s*'@\/lib\/demo-kit'[\s\S]*?\bdefineDemo\b/.test(src),
|
||||
rel(demoFile),
|
||||
'rule 5 (defineDemo default export)',
|
||||
"defineDemo must be imported from '@/lib/demo-kit'.",
|
||||
);
|
||||
}
|
||||
|
||||
/* -- 6/10. the specification and the reward ----------------------------- */
|
||||
const anatomy = findInDemo(slug, ANATOMY_ANCHORS, '{', 'anatomy', { lenient: true });
|
||||
const provenance = findInDemo(slug, PROVENANCE_ANCHORS, '{', 'provenance', { lenient: true });
|
||||
const components = findInDemo(slug, COMPONENT_ANCHORS, '[', 'reward.components', { lenient: true });
|
||||
|
||||
const specRule = 'rule 6 (complete specification)';
|
||||
if (anatomy.error) {
|
||||
report.fail(anatomy.file ?? rel(dir), specRule, anatomy.error);
|
||||
} else {
|
||||
for (const field of ['task', 'actions', 'grader', 'score']) {
|
||||
report.check(
|
||||
nonEmptyString(anatomy.value?.[field]),
|
||||
anatomy.file,
|
||||
specRule,
|
||||
`anatomy.${field} is empty. A ${meta.status} demo publishes the four boxes in full; ` +
|
||||
'an empty one reads as a placeholder and undoes the page it sits on.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (provenance.error) {
|
||||
report.fail(provenance.file ?? rel(dir), specRule, provenance.error);
|
||||
} else {
|
||||
report.check(
|
||||
nonEmptyString(provenance.value?.command),
|
||||
provenance.file,
|
||||
specRule,
|
||||
'provenance.command is empty. The eval command is the reader\'s way to disprove the page; ' +
|
||||
'it is the one field that must never be aspirational.',
|
||||
);
|
||||
}
|
||||
|
||||
const weightRule = 'rule 10 (reward weights)';
|
||||
if (components.error) {
|
||||
report.fail(components.file ?? rel(dir), weightRule, components.error);
|
||||
} else if (!Array.isArray(components.value) || components.value.length === 0) {
|
||||
report.fail(components.file, weightRule, 'reward.components is not a non-empty array.');
|
||||
} else {
|
||||
const list = components.value;
|
||||
let sum = 0;
|
||||
let readable = true;
|
||||
list.forEach((component, i) => {
|
||||
const where = `reward.components[${i}]${component?.key ? ` (${component.key})` : ''}`;
|
||||
for (const field of ['key', 'label', 'description']) {
|
||||
report.check(
|
||||
nonEmptyString(component?.[field]),
|
||||
components.file,
|
||||
weightRule,
|
||||
`${where}: \`${field}\` is empty.`,
|
||||
);
|
||||
}
|
||||
if (typeof component?.weight !== 'number' || !Number.isFinite(component.weight)) {
|
||||
readable = false;
|
||||
report.fail(
|
||||
components.file,
|
||||
weightRule,
|
||||
`${where}: \`weight\` is ${isUnresolved(component?.weight) ? 'an imported constant' : JSON.stringify(component?.weight)}, ` +
|
||||
'not a literal number. Weights are read off the page beside the code, so they are written out here.',
|
||||
);
|
||||
} else {
|
||||
sum += component.weight;
|
||||
}
|
||||
report.check(
|
||||
ROLES.includes(component?.role),
|
||||
components.file,
|
||||
weightRule,
|
||||
`${where}: \`role\` must be one of ${ROLES.join(', ')}.`,
|
||||
);
|
||||
});
|
||||
|
||||
if (readable) {
|
||||
report.check(
|
||||
Math.abs(sum - 1) <= 1e-9,
|
||||
components.file,
|
||||
weightRule,
|
||||
`the weights sum to ${sum} and must sum to 1.0 (within 1e-9). ` +
|
||||
'A reward whose weights do not sum to one is not the reward the page shows you tuning.',
|
||||
);
|
||||
}
|
||||
report.check(
|
||||
list.some((c) => c?.role === 'counterweight'),
|
||||
components.file,
|
||||
'rule 10 (counterweight required)',
|
||||
'no component has role "counterweight". An objective with nothing pulling against it is a metric to ' +
|
||||
'game, and the demo exists to show the opposite. If the second term is one every good policy also ' +
|
||||
'scores 1.0 on, it is a gate — and the demo still needs a real counterweight.',
|
||||
);
|
||||
const counterweight = list.find((c) => c?.role === 'counterweight');
|
||||
if (counterweight) {
|
||||
report.check(
|
||||
nonEmptyString(counterweight.description),
|
||||
components.file,
|
||||
specRule,
|
||||
`the counterweight (${counterweight.key}) has no description. It is the half of the reward a reader ` +
|
||||
'does not expect, so it is the half that must be spelled out.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/* -- 7/11/12. recorded runs --------------------------------------------- */
|
||||
if (meta.status === 'live') {
|
||||
const runRule = 'rule 7 (live demo has runs)';
|
||||
if (manifest.error) {
|
||||
report.fail(rel(abs('public', 'traces', 'manifest.json')), runRule, manifest.error);
|
||||
} else {
|
||||
const entry = manifest.byDemo.get(slug);
|
||||
const runs = entry?.runs ?? [];
|
||||
if (runs.length === 0) {
|
||||
report.fail(
|
||||
rel(abs('public', 'traces', 'manifest.json')),
|
||||
runRule,
|
||||
`demo "${slug}" is status "live" but has no runs in the manifest. ` +
|
||||
'A live demo asserts recorded evidence; without a run there is nothing to replay.',
|
||||
);
|
||||
} else {
|
||||
const episodes = [];
|
||||
for (const run of runs) {
|
||||
const label = `run ${run?.id ?? '(no id)'}`;
|
||||
if (!nonEmptyString(run?.path)) {
|
||||
report.fail(rel(abs('public', 'traces', 'manifest.json')), runRule, `${label}: \`path\` is empty.`);
|
||||
continue;
|
||||
}
|
||||
const file = traceFile(run.path);
|
||||
if (!report.check(exists(file), rel(file), runRule, `${label} points at ${run.path}, which does not exist on disk.`)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const raw = JSON.parse(read(file));
|
||||
for (const ep of Array.isArray(raw) ? raw : Array.isArray(raw?.episodes) ? raw.episodes : [raw]) {
|
||||
episodes.push({ run, file, episode: ep });
|
||||
}
|
||||
} catch (error) {
|
||||
report.fail(rel(file), runRule, `not valid JSON: ${error.message}`);
|
||||
}
|
||||
|
||||
/* -- 12. an intervened run must say what was done to it ---------- */
|
||||
if (run?.kind === 'intervened') {
|
||||
report.check(
|
||||
nonEmptyString(run.intervention),
|
||||
rel(abs('public', 'traces', 'manifest.json')),
|
||||
'rule 12 (intervened runs declare the intervention)',
|
||||
`${label} has kind "intervened" but no \`intervention\`. Without it a prompt change reads as a ` +
|
||||
'training result by omission, which is the single most misleading thing this page could do.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/* -- 11. a failure, or a solve rate the page renders -------------- */
|
||||
const failedRun = episodes.find((e) => e.episode?.outcome === 'failed');
|
||||
if (!failedRun && episodes.length) {
|
||||
const rateSources = [entry?.extras ?? {}, ...runs];
|
||||
const rate = rateSources
|
||||
.map((s) => s?.solveRate ?? s?.solve_rate)
|
||||
.find((v) => typeof v === 'number' && Number.isFinite(v));
|
||||
const rendered = walk(abs('src'), (f) => /\.tsx?$/.test(f)).some((f) => /\bsolve[_R]?ate|solveRate|solve_rate/.test(read(f)));
|
||||
report.check(
|
||||
typeof rate === 'number' && rendered,
|
||||
rel(abs('public', 'traces', 'manifest.json')),
|
||||
'rule 11 (a clean sweep is reported, not hidden)',
|
||||
`demo "${slug}" ships no run with outcome "failed". That is allowed only if the page states the ` +
|
||||
'solve rate: add a numeric `solveRate` beside the runs in the manifest AND render it. ' +
|
||||
`Right now solveRate is ${typeof rate === 'number' ? rate : 'absent'} and a renderer for it was ` +
|
||||
`${rendered ? 'found' : 'not found'} under src/. Hunting for a losing seed to make the demo look ` +
|
||||
'honest is the failure mode this rule exists to block.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* -- 13. every step announces itself ------------------------------------ */
|
||||
checkAnnounce(slug, report, manifest);
|
||||
}
|
||||
|
||||
/* ---------------------------------------- 8. the shell knows no demo names */
|
||||
|
||||
const SHELL_DIR = abs('src', 'components', 'demo');
|
||||
for (const file of walk(SHELL_DIR, (f) => /\.(tsx?|css)$/.test(f))) {
|
||||
const lines = read(file).split('\n');
|
||||
for (const slug of allDemoDirs()) {
|
||||
if (slug.startsWith('_')) continue;
|
||||
// Word-boundary on both sides so a slug like `wordle` is not found inside
|
||||
// an unrelated identifier, and so `wordle-five` matches as one token.
|
||||
const needle = new RegExp(`(?<![\\w-])${slug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`);
|
||||
const hit = lines.findIndex((line) => needle.test(line));
|
||||
if (hit !== -1) {
|
||||
report.fail(
|
||||
`${rel(file)}:${hit + 1}`,
|
||||
'rule 8 (the shell is generic)',
|
||||
`mentions the demo slug "${slug}". Nothing under src/components/demo/ may know which demo it is ` +
|
||||
'rendering. If the shell needs to special-case something, that is a missing slot on DemoModule or a ' +
|
||||
'missing capability in demo-kit — fix it there and every future demo gets it too.',
|
||||
);
|
||||
} else {
|
||||
report.passed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------- 9. demos reach only for the kit's barrel */
|
||||
|
||||
const FORBIDDEN_IMPORTS = [
|
||||
{
|
||||
test: (spec) => /^@\/lib\/demo-kit\/.+/.test(spec),
|
||||
why: "imports a file INSIDE demo-kit. Import '@/lib/demo-kit' — the barrel is the contract, and anything " +
|
||||
'you can only get by reaching past it is either private or belongs in the barrel.',
|
||||
},
|
||||
{
|
||||
test: (spec) => /^@\/components\/demo(\/|$)/.test(spec),
|
||||
why: 'imports the shared shell. The shell renders demos; demos never render the shell. This is the ' +
|
||||
'dependency that, once it exists, makes the shell impossible to change without opening every demo.',
|
||||
},
|
||||
];
|
||||
|
||||
for (const slug of allDemoDirs()) {
|
||||
for (const file of demoFiles(slug)) {
|
||||
for (const spec of importSpecifiers(read(file))) {
|
||||
const normalised = normaliseSpecifier(spec, file);
|
||||
for (const rule of FORBIDDEN_IMPORTS) {
|
||||
if (rule.test(normalised)) {
|
||||
report.fail(
|
||||
rel(file),
|
||||
'rule 9 (demos import only the demo-kit barrel)',
|
||||
`\`${spec}\`${normalised === spec ? '' : ` (resolves to ${normalised})`} ${rule.why}`,
|
||||
);
|
||||
} else {
|
||||
report.passed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.finish();
|
||||
|
||||
/* ------------------------------------------------------------- helpers */
|
||||
|
||||
/** Every module specifier a file imports, static, dynamic or side-effect. */
|
||||
function importSpecifiers(src) {
|
||||
const out = new Set();
|
||||
for (const m of src.matchAll(/\bfrom\s*['"]([^'"]+)['"]/g)) out.add(m[1]);
|
||||
for (const m of src.matchAll(/\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g)) out.add(m[1]);
|
||||
for (const m of src.matchAll(/^\s*import\s+['"]([^'"]+)['"]/gm)) out.add(m[1]);
|
||||
return [...out];
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrites a relative specifier into its `@/` form.
|
||||
*
|
||||
* Without this, `../../components/demo/Player` walks straight past a lint that
|
||||
* only looks at the alias — and it is the form an editor's auto-import
|
||||
* produces, so it is the form the violation actually arrives in.
|
||||
*/
|
||||
function normaliseSpecifier(spec, fromFile) {
|
||||
if (!spec.startsWith('.')) return spec;
|
||||
const resolved = path.resolve(path.dirname(fromFile), spec);
|
||||
const inSrc = path.relative(abs('src'), resolved);
|
||||
if (inSrc.startsWith('..')) return spec;
|
||||
return `@/${inSrc.split(path.sep).join('/')}`.replace(/\.(tsx?|jsx?)$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Rule 13: every `DemoStep` an adapter produces sets a non-empty `announce`.
|
||||
*
|
||||
* Reduced motion clamps the tile flip to nothing, so for a screen-reader user
|
||||
* the announcement IS the result. A missing one is not a degraded experience,
|
||||
* it is a blank page that claims to be showing you something.
|
||||
*
|
||||
* Tried dynamically first, under tsx, against the demo's own recorded traces —
|
||||
* a static check cannot see through a helper that builds the step object. When
|
||||
* the module will not load in bare Node (a `?raw` import, a Vite-only alias,
|
||||
* JSX pulling in something browser-shaped) it falls back to reading the source
|
||||
* and SAYS SO in the report rather than quietly downgrading.
|
||||
*/
|
||||
function checkAnnounce(slug, report, manifest) {
|
||||
const rule = 'rule 13 (every step announces itself)';
|
||||
const dir = path.join(DEMOS_DIR, slug);
|
||||
const candidates = ['adapt.ts', 'adapter.ts', 'adapters.ts', 'adapt.tsx', 'demo.tsx']
|
||||
.map((n) => path.join(dir, n))
|
||||
.filter((p) => exists(p) && /\badapt\b\s*[:=(]|function\s+adapt\b/.test(read(p)));
|
||||
|
||||
if (candidates.length === 0) {
|
||||
report.fail(
|
||||
rel(dir),
|
||||
rule,
|
||||
'no `adapt` implementation found. DemoModule.adapt is required; it is what turns a recorded episode ' +
|
||||
'into the steps the player renders.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const traces = (manifest.byDemo.get(slug)?.runs ?? [])
|
||||
.map((run) => (run?.path ? traceFile(run.path) : null))
|
||||
.filter((p) => p && exists(p));
|
||||
|
||||
if (traces.length > 0) {
|
||||
for (const module of candidates) {
|
||||
const result = probeAdapter(module, traces);
|
||||
if (result.loaded) {
|
||||
if (result.ok) {
|
||||
report.passed += 1;
|
||||
} else {
|
||||
for (const problem of result.problems) report.fail(rel(module), rule, problem);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Static fallback. It cannot prove `announce` is set on every step, only that
|
||||
// the adapter sets it at all and never sets it to nothing.
|
||||
const module = candidates[0];
|
||||
const src = read(module);
|
||||
const assigns = /\bannounce\s*:/.test(src);
|
||||
report.check(
|
||||
assigns,
|
||||
rel(module),
|
||||
rule,
|
||||
'the adapter never assigns `announce`. Every DemoStep must carry one — with motion reduced, it is the ' +
|
||||
'only thing that reports the result.',
|
||||
);
|
||||
if (assigns) {
|
||||
const empty = /\bannounce\s*:\s*(?:''|""|``|null|undefined)\s*[,}\n]/.test(src);
|
||||
report.check(!empty, rel(module), rule, 'the adapter assigns an empty `announce` somewhere. An empty announcement is a missing one.');
|
||||
}
|
||||
report.staticOnly(
|
||||
`${slug}: rule 13 was checked by reading ${rel(module)}, not by running the adapter` +
|
||||
`${traces.length === 0 ? ' (no recorded traces to feed it)' : ' (the module would not load under tsx in bare Node)'}.`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs one adapter over real episodes under tsx. Never throws. */
|
||||
function probeAdapter(modulePath, traces) {
|
||||
const tsx = abs('node_modules', '.bin', 'tsx');
|
||||
if (!exists(tsx)) return { loaded: false, reason: 'tsx is not installed' };
|
||||
|
||||
const probe = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'pig-adapt-')), 'probe.mts');
|
||||
fs.writeFileSync(probe, adapterProbeSource(), 'utf8');
|
||||
try {
|
||||
const run = spawnSync(process.execPath, [tsx, probe, modulePath, ...traces], {
|
||||
cwd: abs('.'),
|
||||
encoding: 'utf8',
|
||||
timeout: 30_000,
|
||||
// tsx resolves the `@/` alias from a tsconfig, and the probe lives in
|
||||
// a temp dir where it would never find ours.
|
||||
env: { ...process.env, TSX_TSCONFIG_PATH: abs('tsconfig.json') },
|
||||
});
|
||||
if (run.status !== 0 || !run.stdout) return { loaded: false, reason: (run.stderr || '').trim().split('\n').slice(-1)[0] };
|
||||
const line = run.stdout.trim().split('\n').pop();
|
||||
const parsed = JSON.parse(line);
|
||||
return parsed.loaded === false ? { loaded: false, reason: parsed.reason } : { loaded: true, ...parsed };
|
||||
} catch {
|
||||
return { loaded: false, reason: 'the probe did not produce parseable output' };
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(probe), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The probe, written to a temp file and executed by tsx.
|
||||
*
|
||||
* It lives here as a string rather than as a checked-in `.mts` because it is an
|
||||
* implementation detail of this script, and a stray TypeScript file in
|
||||
* `scripts/` would get swept into the typecheck it is deliberately outside of.
|
||||
*/
|
||||
function adapterProbeSource() {
|
||||
return `
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
const [modulePath, ...traces] = process.argv.slice(2);
|
||||
const say = (o: unknown) => console.log(JSON.stringify(o));
|
||||
|
||||
let mod: any;
|
||||
try {
|
||||
mod = await import(modulePath);
|
||||
} catch (error: any) {
|
||||
say({ loaded: false, reason: String(error?.message ?? error).split('\\n')[0] });
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const adapt =
|
||||
typeof mod.adapt === 'function' ? mod.adapt :
|
||||
typeof mod.default?.adapt === 'function' ? mod.default.adapt :
|
||||
typeof mod.default === 'function' ? mod.default : null;
|
||||
|
||||
if (!adapt) {
|
||||
say({ loaded: false, reason: 'no callable \\\`adapt\\\` export' });
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const problems: string[] = [];
|
||||
let steps = 0;
|
||||
|
||||
for (const trace of traces) {
|
||||
const raw = JSON.parse(readFileSync(trace, 'utf8'));
|
||||
const episodes = Array.isArray(raw) ? raw : Array.isArray(raw?.episodes) ? raw.episodes : [raw];
|
||||
for (const episode of episodes) {
|
||||
let produced: any;
|
||||
try {
|
||||
produced = adapt(episode);
|
||||
} catch (error: any) {
|
||||
problems.push(trace + ': adapt() threw ' + String(error?.message ?? error).split('\\n')[0]);
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(produced)) {
|
||||
problems.push(trace + ': adapt() returned ' + typeof produced + ', not an array of DemoStep');
|
||||
continue;
|
||||
}
|
||||
produced.forEach((step: any, i: number) => {
|
||||
steps += 1;
|
||||
if (typeof step?.announce !== 'string' || step.announce.trim() === '') {
|
||||
problems.push(trace + ': step ' + i + ' has an empty \\\`announce\\\`');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
say({ loaded: true, ok: problems.length === 0, problems: problems.slice(0, 10), steps });
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* The attribution checker.
|
||||
*
|
||||
* This repo is public and Apache-2.0, and almost none of what makes the demo
|
||||
* work is ours: the primitives are shadcn/ui over Radix, the answer list is
|
||||
* Wordnik's filtered through SCOWL, the typeface is Manrope under the SIL Open
|
||||
* Font License, the icons are lucide, the charts are recharts. Every one of
|
||||
* those licences is permissive, and every one of them requires attribution.
|
||||
*
|
||||
* So `NOTICE` is not paperwork, it is a build artifact with a test. This script
|
||||
* is that test: it fails if NOTICE is missing any source we actually ship, and
|
||||
* it fails if a source is named without its licence beside it — because
|
||||
* "uses Manrope" without "OFL-1.1" is not attribution, it is a mention.
|
||||
*
|
||||
* The OFL additionally requires that the licence text travel WITH the font, so
|
||||
* `public/fonts/OFL.txt` is checked for separately. Shipping the .woff2 out of
|
||||
* node_modules and leaving the licence behind is the single easiest way to
|
||||
* violate the one licence on this list that has teeth.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
|
||||
import { Report, abs, exists, read, rel } from './_lib.mjs';
|
||||
|
||||
const report = new Report('check-licenses');
|
||||
|
||||
const NOTICE = abs('NOTICE');
|
||||
const OFL = abs('public', 'fonts', 'OFL.txt');
|
||||
|
||||
/**
|
||||
* Every third-party source this site ships, and the licence each one must be
|
||||
* named with. `used` is a cheap sanity check in the other direction: an entry
|
||||
* whose artifact has left the repo is stale attribution, and stale attribution
|
||||
* quietly becomes wrong attribution.
|
||||
*/
|
||||
const SOURCES = [
|
||||
{
|
||||
label: 'shadcn/ui',
|
||||
match: /shadcn/i,
|
||||
licence: /\bMIT\b/,
|
||||
why: 'the UI primitives in src/components/ui are shadcn/ui components, hand-copied into this repo.',
|
||||
used: () => exists(abs('components.json')),
|
||||
},
|
||||
{
|
||||
label: 'Radix UI',
|
||||
match: /radix/i,
|
||||
licence: /\bMIT\b/,
|
||||
why: 'every primitive with behaviour — dialog, tabs, tooltip, slider — is Radix underneath.',
|
||||
used: () => exists(abs('node_modules', '@radix-ui')),
|
||||
},
|
||||
{
|
||||
label: 'the Wordnik word list',
|
||||
match: /wordnik/i,
|
||||
licence: /\bMIT\b/,
|
||||
why: 'the guess list is derived from Wordnik.',
|
||||
used: () => hasWordFile(/wordnik/i),
|
||||
},
|
||||
{
|
||||
label: 'SCOWL',
|
||||
match: /\bSCOWL\b/i,
|
||||
licence: /permissive|attribution|BSD|Kevin\s+Atkinson/i,
|
||||
why: 'the answer list is filtered through SCOWL.',
|
||||
used: () => hasWordFile(/scowl/i),
|
||||
},
|
||||
{
|
||||
label: 'Manrope',
|
||||
match: /manrope/i,
|
||||
// The OFL is version-specific and the version matters: OFL-1.1 is the one
|
||||
// Manrope ships under, and it is the one whose terms are quoted in OFL.txt.
|
||||
licence: /OFL[-\s]?1\.1|SIL\s+Open\s+Font\s+License/i,
|
||||
why: 'Manrope is the typeface, embedded as a variable woff2.',
|
||||
used: () => exists(abs('node_modules', '@fontsource-variable', 'manrope')),
|
||||
},
|
||||
{
|
||||
label: 'lucide',
|
||||
match: /lucide/i,
|
||||
licence: /\bISC\b/,
|
||||
why: 'every icon on the site is a lucide icon.',
|
||||
used: () => exists(abs('node_modules', 'lucide-react')),
|
||||
},
|
||||
{
|
||||
label: 'recharts',
|
||||
match: /recharts/i,
|
||||
licence: /\bMIT\b/,
|
||||
why: 'the reward and metric charts are recharts.',
|
||||
used: () => exists(abs('node_modules', 'recharts')),
|
||||
},
|
||||
{
|
||||
label: "PIG's own token layer",
|
||||
match: /\bPIG\b|Prime\s+Intellect\s+Growth/i,
|
||||
licence: /Apache[-\s]?2\.0/i,
|
||||
why: 'the colour and motion tokens in src/index.css come from PIG and ship under Apache-2.0.',
|
||||
used: () => exists(abs('src', 'index.css')),
|
||||
},
|
||||
];
|
||||
|
||||
/** True when the words directory still carries a file from this source. */
|
||||
function hasWordFile(pattern) {
|
||||
const dir = abs('envs', 'wordle_five', 'words');
|
||||
return exists(dir) && fs.readdirSync(dir).some((name) => pattern.test(name));
|
||||
}
|
||||
|
||||
if (!exists(NOTICE)) {
|
||||
report.fail(
|
||||
'NOTICE',
|
||||
'NOTICE exists',
|
||||
'there is no NOTICE file at the repo root. Every source below is shipped by this site and each of ' +
|
||||
`their licences requires attribution: ${SOURCES.map((s) => s.label).join(', ')}.`,
|
||||
);
|
||||
report.finish();
|
||||
}
|
||||
|
||||
const notice = read(NOTICE);
|
||||
const lines = notice.split('\n');
|
||||
|
||||
for (const source of SOURCES) {
|
||||
// Every line that names the source, not just the first. "PIG-Demo" in the
|
||||
// copyright header matches the PIG entry three dozen lines before its actual
|
||||
// attribution block, and a first-match-wins reader fails on a NOTICE that is
|
||||
// completely correct.
|
||||
const hits = [];
|
||||
lines.forEach((line, i) => {
|
||||
if (source.match.test(line)) hits.push(i);
|
||||
});
|
||||
|
||||
if (hits.length === 0) {
|
||||
report.fail(
|
||||
'NOTICE',
|
||||
'attribution complete',
|
||||
`${source.label} is not named. It is shipped by this site — ${source.why} — and its licence requires ` +
|
||||
'attribution. Add it with its licence identifier.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
report.passed += 1;
|
||||
|
||||
// The licence has to sit with the name, not merely somewhere in the file:
|
||||
// a NOTICE that says "MIT" once at the top and lists nine projects under it
|
||||
// is attributing all nine to whichever licence happens to be first.
|
||||
const near = (i) => lines.slice(Math.max(0, i - 2), i + 6).join('\n');
|
||||
report.check(
|
||||
hits.some((i) => source.licence.test(near(i))),
|
||||
`NOTICE:${hits[0] + 1}`,
|
||||
'attribution names the licence',
|
||||
`${source.label} is named on line(s) ${hits.map((i) => i + 1).join(', ')} but no matching licence appears ` +
|
||||
`beside any of them (looking for ${source.licence}). A name without a licence is a mention, not an ` +
|
||||
'attribution.',
|
||||
);
|
||||
|
||||
if (!source.used()) {
|
||||
report.warn(
|
||||
`NOTICE names ${source.label}, but nothing in the repo appears to use it any more. ` +
|
||||
'Stale attribution is how a NOTICE stops being trustworthy.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------- the OFL's own rule */
|
||||
|
||||
report.check(
|
||||
exists(OFL),
|
||||
rel(OFL),
|
||||
'OFL text ships with the font',
|
||||
'Manrope is under SIL OFL-1.1, which requires the licence text to travel with the font files. ' +
|
||||
'Copy node_modules/@fontsource-variable/manrope/LICENSE to public/fonts/OFL.txt. ' +
|
||||
'Shipping the woff2 and leaving the licence behind is the one violation on this list with teeth.',
|
||||
);
|
||||
|
||||
if (exists(OFL)) {
|
||||
const text = read(OFL);
|
||||
report.check(
|
||||
/SIL OPEN FONT LICENSE/i.test(text) && /Version 1\.1/i.test(text),
|
||||
rel(OFL),
|
||||
'OFL text ships with the font',
|
||||
'the file exists but does not look like the SIL Open Font License 1.1. It must be the licence text ' +
|
||||
'itself, not a pointer to it.',
|
||||
);
|
||||
}
|
||||
|
||||
report.finish();
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* The receipt checker.
|
||||
*
|
||||
* Every demo quotes the code that computes its reward, verbatim, next to the
|
||||
* number that code produced. That quote is the page's whole claim to being
|
||||
* evidence rather than marketing, so the thing it quotes has to exist and the
|
||||
* region it quotes has to resolve. A receipt panel that silently renders empty,
|
||||
* or renders the wrong forty lines because someone inserted a function above
|
||||
* the marker, is worse than no receipt at all.
|
||||
*
|
||||
* ── MARKER SYNTAX ──────────────────────────────────────────────────────────
|
||||
*
|
||||
* `RewardSpec.source` is `{ path, code, marker? }`. Without a `marker` the
|
||||
* whole file is the receipt. With one, the receipt is the region delimited by
|
||||
* a matched pair of comment lines in the source file itself:
|
||||
*
|
||||
* # region: pig-demo/<marker>
|
||||
* ... the quoted lines ...
|
||||
* # endregion: pig-demo/<marker>
|
||||
*
|
||||
* Rules, all enforced here:
|
||||
*
|
||||
* · The marker lines are EXCLUSIVE — neither appears in the quoted region.
|
||||
* · EXACTLY ONE pair per file per marker. Zero is a broken receipt; two or
|
||||
* more is ambiguous, and an ambiguous receipt silently quotes whichever
|
||||
* region the reader's implementation happened to find first. Both are
|
||||
* fatal.
|
||||
* · `region` must come before `endregion`, and the region must be non-empty
|
||||
* once the marker lines are removed.
|
||||
* · The comment token is `#` (Python) or `//` (TypeScript), then optional
|
||||
* whitespace. Everything else on the line is ignored, so
|
||||
* `# region: pig-demo/consistency (see PR #41)` is legal.
|
||||
* · The `pig-demo/` prefix is required. It is what makes these greppable and
|
||||
* stops an editor's own `#region` folding markers from being read as
|
||||
* receipts.
|
||||
*
|
||||
* Marker names are `[a-z0-9][a-z0-9-]*`: they end up in a grep, a CI message
|
||||
* and a code comment, and mixed case in all three is a bug factory.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
Report,
|
||||
abs,
|
||||
allDemoDirs,
|
||||
demoFiles,
|
||||
evalLiteral,
|
||||
exists,
|
||||
isUnresolved,
|
||||
literalsAfter,
|
||||
read,
|
||||
rel,
|
||||
} from './_lib.mjs';
|
||||
|
||||
const report = new Report('check-receipts');
|
||||
|
||||
const RULE_PATH = 'reward.source.path';
|
||||
const RULE_MARKER = 'reward.source.marker';
|
||||
|
||||
const MARKER_NAME = /^[a-z0-9][a-z0-9-]*$/;
|
||||
const markerLine = (kind, marker) =>
|
||||
new RegExp(`^[ \\t]*(?:#|//)[ \\t]*${kind}:[ \\t]*pig-demo/${marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\w-])`, 'm');
|
||||
|
||||
/** Every `source: { ... }` literal in a demo, wherever the demo chose to put it. */
|
||||
function receipts(slug) {
|
||||
const out = [];
|
||||
for (const file of demoFiles(slug)) {
|
||||
const src = read(file);
|
||||
for (const literal of literalsAfter(src, /\bsource\s*:\s*/, '{')) {
|
||||
// `code` is legitimately an identifier here — the Python arrives through
|
||||
// a Vite `?raw` import, which cannot resolve in plain Node — so the
|
||||
// literal is read leniently and only `path` and `marker` are trusted.
|
||||
const evaluated = evalLiteral(literal.text, 'RewardSpec.source', { lenient: true });
|
||||
if (!evaluated.ok) {
|
||||
out.push({ file, error: evaluated.error });
|
||||
continue;
|
||||
}
|
||||
const value = evaluated.value;
|
||||
if (!value || typeof value !== 'object' || !('path' in value)) continue; // some other `source:` key
|
||||
const line = src.slice(0, literal.start).split('\n').length;
|
||||
out.push({ file, line, value });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const slugs = allDemoDirs().filter((s) => !s.startsWith('_'));
|
||||
if (slugs.length === 0) {
|
||||
console.log('check-receipts: no demos to check.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
for (const slug of slugs) {
|
||||
const found = receipts(slug);
|
||||
|
||||
if (found.length === 0) {
|
||||
report.fail(
|
||||
rel(path.join(abs('src', 'demos'), slug)),
|
||||
RULE_PATH,
|
||||
'no `reward.source` found. Every demo quotes the code that computes its reward; without a source ' +
|
||||
'the receipt panel has nothing to show and the page is asserting its numbers rather than proving them.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const receipt of found) {
|
||||
const where = `${rel(receipt.file)}${receipt.line ? `:${receipt.line}` : ''}`;
|
||||
if (receipt.error) {
|
||||
report.fail(where, RULE_PATH, receipt.error);
|
||||
continue;
|
||||
}
|
||||
const { path: sourcePath, marker } = receipt.value;
|
||||
|
||||
if (typeof sourcePath !== 'string' || sourcePath.trim() === '') {
|
||||
report.fail(
|
||||
where,
|
||||
RULE_PATH,
|
||||
`\`path\` is ${isUnresolved(sourcePath) ? 'an imported constant' : JSON.stringify(sourcePath)}. ` +
|
||||
'It must be a literal repo-relative path, e.g. "envs/wordle_five/wordle_five/rubric.py", ' +
|
||||
'so a reader can open the same file this page quotes.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Repo-relative by contract. A leading slash would mean the site root,
|
||||
// which is not where the Python lives, and an absolute disk path would
|
||||
// only work on the machine that wrote it.
|
||||
const clean = sourcePath.replace(/^\.\//, '');
|
||||
if (path.isAbsolute(clean)) {
|
||||
report.fail(where, RULE_PATH, `\`path\` is absolute ("${sourcePath}"). Use a repo-relative path.`);
|
||||
continue;
|
||||
}
|
||||
const onDisk = abs(clean);
|
||||
if (!report.check(exists(onDisk), where, RULE_PATH, `\`path\` is "${sourcePath}", and ${rel(onDisk)} does not exist.`)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (marker === undefined || marker === null) continue; // whole file is the receipt
|
||||
|
||||
if (typeof marker !== 'string' || !MARKER_NAME.test(marker)) {
|
||||
report.fail(
|
||||
where,
|
||||
RULE_MARKER,
|
||||
`\`marker\` is ${JSON.stringify(marker)}. Markers are lower-kebab-case (${MARKER_NAME}) because they ` +
|
||||
'appear in a grep, a CI message and a code comment, and mixed case in all three is a bug factory.',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const body = read(onDisk);
|
||||
const lines = body.split('\n');
|
||||
const opens = [];
|
||||
const closes = [];
|
||||
lines.forEach((line, i) => {
|
||||
if (markerLine('region', marker).test(line)) opens.push(i);
|
||||
if (markerLine('endregion', marker).test(line)) closes.push(i);
|
||||
});
|
||||
|
||||
const syntax = `Expected exactly one "# region: pig-demo/${marker}" and one "# endregion: pig-demo/${marker}".`;
|
||||
if (opens.length !== 1 || closes.length !== 1) {
|
||||
report.fail(
|
||||
rel(onDisk),
|
||||
RULE_MARKER,
|
||||
`marker "${marker}" (referenced from ${where}) resolves to ${opens.length} region marker(s) and ` +
|
||||
`${closes.length} endregion marker(s). ${syntax} ` +
|
||||
(opens.length > 1 || closes.length > 1
|
||||
? 'More than one pair is ambiguous: the receipt would quote whichever region the reader found first.'
|
||||
: 'A missing marker renders the receipt panel empty, which reads as the code not existing.'),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [open] = opens;
|
||||
const [close] = closes;
|
||||
if (close <= open) {
|
||||
report.fail(
|
||||
rel(onDisk),
|
||||
RULE_MARKER,
|
||||
`marker "${marker}": the endregion is on line ${close + 1}, at or before the region on line ${open + 1}.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const region = lines.slice(open + 1, close);
|
||||
report.check(
|
||||
region.some((line) => line.trim() !== ''),
|
||||
rel(onDisk),
|
||||
RULE_MARKER,
|
||||
`marker "${marker}" delimits an empty region (lines ${open + 2}-${close}). The markers are exclusive, ` +
|
||||
'so a pair on adjacent lines quotes nothing.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
report.finish();
|
||||
Reference in New Issue
Block a user