Prerender, social cards, and a yellow that reads as yellow
Two silent bugs in the prerender pass, and the second was caused by the fix for
the first.
`waitForSelector('#root > *')` defaults to waiting for VISIBILITY, and the app's
first child is the skip link, which is hidden until focused. So it burned the
full 30s timeout on every one of 17 routes — twelve minutes of a script that
printed nothing, because its output was buffered behind a pipe — while the page
had rendered the whole time. Switching to `state: 'attached'` then fired too
early instead: useSeo writes the head from an effect, so the title was still
index.html's for a tick, and every route would have baked the homepage's head.
That is the exact bug this script exists to prevent. It now waits for `main`,
then for readyState, then settles.
The board's yellow was --warning, 32 95% 31% — darkened until white text cleared
4.5:1, and at that lightness it renders BROWN. On a board where people arrive
knowing this square should be yellow, a brown square reads as a bug in the
scorer, which on a page arguing "the grader is correct" is the worst thing it
could look like. The fill is now a real yellow and the glyph went dark: more
expected AND higher contrast, 10.02:1 against 5.03:1.
Also measured something the Honesty page had honestly declined to claim. It said
our word list is easier than the original's because our dictionary rule keeps
plurals the original's editor removed by hand. Running the same greedy solver
over both pools, 250 sampled words each: original 2,315 needs 3.552 guesses
(opener RAISE, worst 5), ours 4,603 needs 3.700 (opener TARES, worst 6). The
doubled pool outweighs the plurals. Ours is harder, and the page now says so
with the table.
177 gate checks pass. Entry chunk 106.9 kB gzipped against 160 kB.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
@@ -0,0 +1,220 @@
|
||||
/**
|
||||
* The lazy half of a demo: everything that costs bundle size.
|
||||
*
|
||||
* A demo exists because this file and `meta.ts` exist. There is no registry to
|
||||
* edit, no route to add and no import to insert anywhere else — if you find
|
||||
* yourself editing a shared file to make a new demo appear, that is a bug in
|
||||
* `src/lib/demo-kit/registry.ts`, not a missing step here.
|
||||
*
|
||||
* Two rules the checker enforces, both worth understanding rather than
|
||||
* working around:
|
||||
*
|
||||
* · Import ONLY from '@/lib/demo-kit'. The barrel is the contract. Reaching
|
||||
* into '@/lib/demo-kit/player' or '@/components/demo/…' makes the shell
|
||||
* impossible to change without opening every demo (rule 9).
|
||||
* · Every step sets a non-empty `announce`. Reduced motion clamps the
|
||||
* animation to nothing, so for a screen-reader user the announcement IS
|
||||
* the result, not a courtesy (rule 13).
|
||||
*/
|
||||
|
||||
import { defineDemo, type DemoEpisode, type DemoStep, type RewardValues } from '@/lib/demo-kit';
|
||||
|
||||
import meta from './meta';
|
||||
import Board, { empty__Pascal__, type QueueItem, type __Pascal__State } from './surface';
|
||||
// The Python is quoted verbatim beside the number it produced. `?raw` is how
|
||||
// the real file gets into the bundle without being reimplemented in TypeScript.
|
||||
import rewardSource from './reward.py?raw';
|
||||
|
||||
/* ------------------------------------------------------------------ adapt */
|
||||
|
||||
/** Reads the queue off a recorded turn without trusting its shape. */
|
||||
function itemsFrom(info: Record<string, unknown> | undefined): QueueItem[] {
|
||||
const raw = info?.['items'];
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return raw.flatMap((entry): QueueItem[] => {
|
||||
if (entry === null || typeof entry !== 'object') return [];
|
||||
const row = entry as Record<string, unknown>;
|
||||
const id = typeof row['id'] === 'string' ? row['id'] : null;
|
||||
if (id === null) return [];
|
||||
return [
|
||||
{
|
||||
id,
|
||||
label: typeof row['label'] === 'string' ? row['label'] : id,
|
||||
risk: typeof row['risk'] === 'number' ? row['risk'] : 0,
|
||||
needsPerson: row['needs_person'] === true || row['needsPerson'] === true,
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
function escalatedFrom(info: Record<string, unknown> | undefined): string[] {
|
||||
const raw = info?.['escalated'];
|
||||
return Array.isArray(raw) ? raw.filter((id): id is string => typeof id === 'string') : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* A recorded episode becomes the boards the player renders. Pure, and total:
|
||||
* a turn the environment recorded badly must degrade to a readable step, never
|
||||
* throw — the shell has an error boundary, but a page that renders the error
|
||||
* card instead of the run has still lost the argument.
|
||||
*/
|
||||
export function adapt(episode: DemoEpisode): DemoStep<__Pascal__State>[] {
|
||||
let items: QueueItem[] = [];
|
||||
|
||||
return episode.turns.map((turn, index) => {
|
||||
const turnItems = itemsFrom(turn.info);
|
||||
// The queue is only sent once, on the first turn of most recordings, so
|
||||
// carry the last one we saw rather than blanking the board mid-replay.
|
||||
if (turnItems.length > 0) items = turnItems;
|
||||
|
||||
const escalated = escalatedFrom(turn.info);
|
||||
const last = escalated[escalated.length - 1];
|
||||
const lastItem = last === undefined ? undefined : items.find((item) => item.id === last);
|
||||
const isLastTurn = index === episode.turns.length - 1;
|
||||
|
||||
const state: __Pascal__State = {
|
||||
seed: episode.seed,
|
||||
items,
|
||||
escalated,
|
||||
outcome: isLastTurn && episode.outcome !== 'aborted' ? episode.outcome : 'pending',
|
||||
};
|
||||
|
||||
return {
|
||||
index,
|
||||
state,
|
||||
reply: turn.reply,
|
||||
reasoning: turn.reasoning,
|
||||
call: turn.call,
|
||||
announce:
|
||||
lastItem === undefined
|
||||
? `Turn ${index + 1}: nothing escalated, ${items.length} left in the queue.`
|
||||
: `Turn ${index + 1}: escalated ${lastItem.label}. ${escalated.length} of ${items.length} escalated.`,
|
||||
...(lastItem === undefined ? {} : { caption: `Escalated ${lastItem.label}` }),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------- verify */
|
||||
|
||||
/**
|
||||
* Re-derives the score in the browser, so the page can PROVE the recorded
|
||||
* numbers rather than assert them. Returning null means 'unverifiable' — a
|
||||
* truncated trace — and the badge renders that as neutral, never as zero.
|
||||
*/
|
||||
function recompute(episode: DemoEpisode): RewardValues | null {
|
||||
if (episode.truncated) return null;
|
||||
const steps = adapt(episode);
|
||||
const final = steps[steps.length - 1];
|
||||
if (!final || final.state.items.length === 0) return null;
|
||||
|
||||
const needed = final.state.items.filter((item) => item.needsPerson);
|
||||
const escalated = new Set(final.state.escalated);
|
||||
const noise = [...escalated].filter((id) => !needed.some((item) => item.id === id));
|
||||
|
||||
return {
|
||||
caught: needed.length === 0 ? 1 : needed.filter((item) => escalated.has(item.id)).length / needed.length,
|
||||
restraint: escalated.size === 0 ? 1 : 1 - noise.length / escalated.size,
|
||||
well_formed: 1,
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ demo */
|
||||
|
||||
export default defineDemo<__Pascal__State>({
|
||||
meta,
|
||||
|
||||
narrative: {
|
||||
thesis:
|
||||
'The queue is not the problem. Deciding which three of four hundred items are worth a person is the ' +
|
||||
'problem, and that decision is exactly the kind of judgement a reward can be written down for.',
|
||||
anxiety: 'What stops it escalating everything so it never misses one?',
|
||||
beats: [
|
||||
{ id: 'hero', title: 'One queue, one decision', claim: 'Every item is either worth a person or it is not.', surface: 'hero' },
|
||||
{ id: 'anatomy', title: 'What the environment is', claim: 'A task, a fixed action set, a grader, and a score that moves.', surface: 'anatomy' },
|
||||
{ id: 'play', title: 'Watch a recorded run', claim: 'These are recorded turns, not a scripted animation.', surface: 'split-play' },
|
||||
{ id: 'reward', title: 'Move the weights yourself', claim: 'Pay only for catches and the queue gets escalated whole.', surface: 'reward-editor' },
|
||||
{ id: 'receipt', title: 'The code that scored it', claim: 'The number on this page came out of the function below it.', surface: 'receipt' },
|
||||
{ id: 'limits', title: 'What this does not show', claim: 'One queue, one grader, and no cost of being wrong.', surface: 'limits' },
|
||||
],
|
||||
limits: [
|
||||
{
|
||||
text: 'The grader knows which items needed a person because the dataset says so. A real queue has no such column, and building one is most of the work.',
|
||||
},
|
||||
{
|
||||
text: 'Escalating wrongly and missing something both cost the same here. In production they do not, and the weights should say so.',
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
/**
|
||||
* The four boxes. `check-demos` rule 6 fails an empty one for a `spec` demo
|
||||
* as loudly as for a `live` one: a specification with a blank grader is a
|
||||
* coming-soon card wearing a contract.
|
||||
*/
|
||||
anatomy: {
|
||||
task: 'Work one queue of items and escalate only the ones that genuinely need a person.',
|
||||
actions: 'Escalate an item by id, or end the turn. Any other output is refused and counts as a malformed action.',
|
||||
grader: 'Compares the escalated set to the labelled set. It counts; it does not judge.',
|
||||
score: 'Three fifths for the ones caught, a quarter for not escalating anything else, and the rest is a well-formed-output gate.',
|
||||
},
|
||||
|
||||
reward: {
|
||||
components: [
|
||||
{
|
||||
key: 'caught',
|
||||
label: 'Caught what mattered',
|
||||
description: 'The share of the items that genuinely needed a person which the agent escalated.',
|
||||
weight: 0.6,
|
||||
role: 'objective',
|
||||
},
|
||||
{
|
||||
key: 'restraint',
|
||||
label: 'Left the rest alone',
|
||||
description:
|
||||
'The share of everything it escalated that was worth escalating. This is the term that makes ' +
|
||||
'escalating the whole queue a losing strategy rather than a safe one.',
|
||||
weight: 0.25,
|
||||
role: 'counterweight',
|
||||
},
|
||||
{
|
||||
key: 'well_formed',
|
||||
label: 'Answered in the required form',
|
||||
description: 'One point unless the run emitted an action the environment could not parse.',
|
||||
weight: 0.15,
|
||||
role: 'gate',
|
||||
},
|
||||
],
|
||||
metrics: [
|
||||
{
|
||||
key: 'escalations_that_landed',
|
||||
label: 'Escalations that landed',
|
||||
description: 'A count, not a rate. Rendered beside the reward and never summed into it.',
|
||||
},
|
||||
],
|
||||
source: {
|
||||
path: 'src/demos/_template/reward.py',
|
||||
code: rewardSource,
|
||||
marker: 'score',
|
||||
},
|
||||
},
|
||||
|
||||
provenance: {
|
||||
envPackage: '__package__',
|
||||
tasksetId: '__slug__',
|
||||
verifiersVersion: '0.3.2.dev12',
|
||||
command: 'uv run python envs/probe.py --env __package__',
|
||||
credits: [
|
||||
{
|
||||
label: 'verifiers — the environment API this mirrors',
|
||||
href: 'https://github.com/PrimeIntellect-ai/verifiers',
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
adapt,
|
||||
Surface: Board,
|
||||
verify: recompute,
|
||||
});
|
||||
|
||||
/** Re-exported so a scaffolded demo's tests can build a board without the shell. */
|
||||
export { empty__Pascal__ };
|
||||
@@ -2,8 +2,9 @@
|
||||
* ── THE TEMPLATE ───────────────────────────────────────────────────────────
|
||||
*
|
||||
* `node scripts/new-demo.mjs <slug>` copies this directory to
|
||||
* `src/demos/<slug>` and substitutes the `__token__` names. Everything here is
|
||||
* a WORKED EXAMPLE, not filler: it is written the way a real demo is written
|
||||
* `src/demos/<slug>` and substitutes the placeholder names (the table is in
|
||||
* that script's header). Everything here is a WORKED EXAMPLE, not filler: it
|
||||
* is written the way a real demo is written
|
||||
* so that a scaffold passes `pnpm check` on the first run and you edit prose
|
||||
* rather than discover the contract one failing rule at a time.
|
||||
*
|
||||
@@ -14,6 +15,9 @@
|
||||
*
|
||||
* `meta.ts` is loaded EAGERLY for every demo on every page, so it stays plain
|
||||
* serialisable data: no React, no lucide component, no imports beyond the kit.
|
||||
*
|
||||
* Delete this block once you have scaffolded from it — it describes the
|
||||
* template, not your demo.
|
||||
*/
|
||||
|
||||
import { defineMeta } from '@/lib/demo-kit';
|
||||
|
||||
Reference in New Issue
Block a user