wordle-five: the engine, the reward, the solver and the probe that checks them
The Python is the source of truth; src/demos/wordle/engine.ts will be a port of it, and CI gates the two against a SHA-256 over all 21.2M (guess, answer) pattern pairs rather than a hand-picked vector file — a vector file only ever catches the cases somebody thought of. The reward is three weighted components, and the third one is the reason this demo is worth building. `solved` and `economy` pull toward winning. `consistency` pulls against them, because a player maximising information deliberately guesses words that cannot win — a word that splits the remaining candidates evenly teaches more than a word that might happen to be right. That is good play, and it costs consistency. The probe ladder proves the tension is real rather than asserted: inaction 0.0000 crude 0.0111 plausible 0.1224 candidate_only 0.8925 exhaustive 0.9031 oracle 0.9458 The two good policies are 0.05 apart and neither dominates — the entropy oracle takes 1.00 economy and 0.73 consistency, the candidate-only player takes 0.75 and 1.00. Which one wins is a decision about what you want, which is the whole argument the site exists to make. probe.py fails CI if either starts dominating. Two traps found by building it. `consistency` is scored over turns SPENT, not guesses accepted: counting only legal guesses hands a free 1.0 to a policy that plays one word and then jams the parser five times — one guess, no contradictions, perfect score. And `economy`'s denominator is the depth the SHIPPED solver reaches, not a depth-optimal search: entropy-greedy is not depth-optimal, so grading it against an exact optimum would make the oracle rung fail its own assertion on some seeds. The word lists are built from Wordnik (MIT) intersected with SCOWL, never from the original game's 2,315 answers. 4,603 answers makes this materially harder than the original, so the published SALET/3.4212 results are cited as belonging to that list and our own reference player's TARES/3.72 is measured here. verifiers is an optional extra. The engine, reward, solver and probe all run — and gate — without an RL stack resolvable. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Identity functions with types attached.
|
||||
*
|
||||
* They exist for two reasons that a plain object literal does not give you:
|
||||
* inference (a demo writes `defineDemo<WordleState>({...})` once and every
|
||||
* callback inside is typed), and a stable grep target — `defineDemo(` finds
|
||||
* every demo in the repo, which is what `scripts/check-demos.mjs` and any
|
||||
* future codemod key off. Do not "simplify" these away.
|
||||
*/
|
||||
|
||||
import type { DemoMeta, DemoModule } from './types';
|
||||
|
||||
/** Wrap the object exported from a demo's `meta.ts`. */
|
||||
export function defineMeta(meta: DemoMeta): DemoMeta {
|
||||
return meta;
|
||||
}
|
||||
|
||||
/** Wrap the object exported from a demo's `demo.tsx`. */
|
||||
export function defineDemo<TState>(demo: DemoModule<TState>): DemoModule<TState> {
|
||||
return demo;
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Loading and scoring recorded runs.
|
||||
*
|
||||
* The one rule this module exists to enforce: **`null` is "not scored", and it
|
||||
* is never 0.0.** A run that failed to grade, a component the environment did
|
||||
* not emit, a truncated trace — all of those are absences. Rendering an absence
|
||||
* as a zero turns a missing measurement into a claim about the model, and this
|
||||
* whole site is an argument that the numbers are real.
|
||||
*/
|
||||
|
||||
import type { DemoEpisode, RewardComponent, RewardValues, RunRef } from './types';
|
||||
|
||||
/**
|
||||
* True when a value must not be summed, averaged or drawn as a bar.
|
||||
*
|
||||
* The signature narrows the FALSE branch to `number`, which is the point — the
|
||||
* caller gets a real number without a second check. It is a small lie for
|
||||
* exactly one input: `NaN` is a `number` but returns `true` here, because a
|
||||
* corrupted fixture must be treated as unscored rather than poison every
|
||||
* downstream sum. Callers only ever reach the narrowed branch when the value is
|
||||
* finite, so the lie is unobservable.
|
||||
*/
|
||||
export function isNotScored(value: number | null | undefined): value is null | undefined {
|
||||
return value === null || value === undefined || !Number.isFinite(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Total reward: sum of `score x weight` over the components that were scored.
|
||||
*
|
||||
* Unscored components are SKIPPED, not zeroed, and the weights are deliberately
|
||||
* NOT renormalised over the survivors — renormalising would quietly invent a
|
||||
* different reward function than the one the environment shipped. If nothing
|
||||
* was scored at all, the answer is `null`, not `0`.
|
||||
*/
|
||||
export function rewardTotal(
|
||||
values: RewardValues,
|
||||
components: readonly RewardComponent[],
|
||||
): number | null {
|
||||
let total = 0;
|
||||
let scored = 0;
|
||||
|
||||
for (const component of components) {
|
||||
const raw = values[component.key];
|
||||
if (isNotScored(raw)) continue;
|
||||
total += raw * component.weight;
|
||||
scored += 1;
|
||||
}
|
||||
|
||||
return scored === 0 ? null : total;
|
||||
}
|
||||
|
||||
/** How many of `components` the episode actually carries a number for. */
|
||||
export function scoredCount(
|
||||
values: RewardValues,
|
||||
components: readonly RewardComponent[],
|
||||
): number {
|
||||
return components.filter((c) => !isNotScored(values[c.key])).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-flight and settled fetches, keyed by path.
|
||||
*
|
||||
* The promise is cached, not the value, so two panels mounting in the same tick
|
||||
* share one request. A rejected promise is evicted, so a failed load can be
|
||||
* retried by simply calling again — a cached rejection would make one flaky
|
||||
* network moment permanent for the life of the tab.
|
||||
*/
|
||||
const episodeCache = new Map<string, Promise<DemoEpisode>>();
|
||||
|
||||
/** Fetch and cache one recorded run. */
|
||||
export function loadEpisode(runRef: RunRef): Promise<DemoEpisode> {
|
||||
const cached = episodeCache.get(runRef.path);
|
||||
if (cached) return cached;
|
||||
|
||||
const pending = fetch(runRef.path, { headers: { accept: 'application/json' } })
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Could not load run "${runRef.id}" (${response.status} from ${runRef.path})`);
|
||||
}
|
||||
return assertEpisode(await response.json(), runRef);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
episodeCache.delete(runRef.path);
|
||||
throw error;
|
||||
});
|
||||
|
||||
episodeCache.set(runRef.path, pending);
|
||||
return pending;
|
||||
}
|
||||
|
||||
/** Drop a cached run. Only useful in tests and the dev-time fixture watcher. */
|
||||
export function clearEpisodeCache(path?: string): void {
|
||||
if (path === undefined) episodeCache.clear();
|
||||
else episodeCache.delete(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural check on a fixture.
|
||||
*
|
||||
* Loud and early beats a board that renders half a run. Everything checked here
|
||||
* is something the surfaces read without a guard.
|
||||
*/
|
||||
function assertEpisode(raw: unknown, runRef: RunRef): DemoEpisode {
|
||||
if (raw === null || typeof raw !== 'object') {
|
||||
throw new Error(`Run "${runRef.id}" is not a JSON object.`);
|
||||
}
|
||||
const episode = raw as Partial<DemoEpisode>;
|
||||
|
||||
if (!Array.isArray(episode.turns)) {
|
||||
throw new Error(`Run "${runRef.id}" has no \`turns\` array.`);
|
||||
}
|
||||
if (episode.rewards === null || typeof episode.rewards !== 'object') {
|
||||
throw new Error(`Run "${runRef.id}" has no \`rewards\` object.`);
|
||||
}
|
||||
if (typeof episode.outcome !== 'string') {
|
||||
throw new Error(`Run "${runRef.id}" has no \`outcome\`.`);
|
||||
}
|
||||
|
||||
return episode as DemoEpisode;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run manifest, `public/traces/manifest.json`.
|
||||
*
|
||||
* `RunRef` says runs are "listed in public/traces/manifest.json", but nothing in
|
||||
* the contract hands the shell a `RunRef[]` — `DemoModule` has no `runs` field.
|
||||
* So the manifest is the only source, and this is its reader. Three shapes are
|
||||
* accepted because the generator and the shell are written in different places
|
||||
* and a mismatch here would be a blank page rather than a type error:
|
||||
*
|
||||
* { "demos": { "wordle-five": [RunRef, ...] } }
|
||||
* { "runs": [ { ...RunRef, "demo": "wordle-five" }, ... ] }
|
||||
* [ { ...RunRef, "demo": "wordle-five" }, ... ]
|
||||
*/
|
||||
export const MANIFEST_PATH = '/traces/manifest.json';
|
||||
|
||||
export type RunManifest = Record<string, RunRef[]>;
|
||||
|
||||
let manifestPromise: Promise<RunManifest> | null = null;
|
||||
|
||||
export function loadManifest(path: string = MANIFEST_PATH): Promise<RunManifest> {
|
||||
if (manifestPromise) return manifestPromise;
|
||||
|
||||
manifestPromise = fetch(path, { headers: { accept: 'application/json' } })
|
||||
.then(async (response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Could not load the run manifest (${response.status} from ${path}).`);
|
||||
}
|
||||
return normaliseManifest(await response.json());
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
manifestPromise = null;
|
||||
throw error;
|
||||
});
|
||||
|
||||
return manifestPromise;
|
||||
}
|
||||
|
||||
/** The runs recorded for one demo, in manifest order. Empty when there are none. */
|
||||
export async function listRuns(slug: string): Promise<RunRef[]> {
|
||||
const manifest = await loadManifest();
|
||||
return manifest[slug] ?? [];
|
||||
}
|
||||
|
||||
function normaliseManifest(raw: unknown): RunManifest {
|
||||
const out: RunManifest = {};
|
||||
|
||||
const push = (slug: string, run: RunRef): void => {
|
||||
const bucket = out[slug];
|
||||
if (bucket) bucket.push(run);
|
||||
else out[slug] = [run];
|
||||
};
|
||||
|
||||
const flat = (entries: unknown[]): void => {
|
||||
for (const entry of entries) {
|
||||
if (entry === null || typeof entry !== 'object') continue;
|
||||
const record = entry as RunRef & { demo?: string; slug?: string };
|
||||
const slug = record.demo ?? record.slug;
|
||||
if (typeof slug !== 'string') continue;
|
||||
push(slug, record);
|
||||
}
|
||||
};
|
||||
|
||||
if (Array.isArray(raw)) {
|
||||
flat(raw);
|
||||
return out;
|
||||
}
|
||||
if (raw === null || typeof raw !== 'object') return out;
|
||||
|
||||
const object = raw as { demos?: unknown; runs?: unknown };
|
||||
if (Array.isArray(object.runs)) flat(object.runs);
|
||||
|
||||
const demos = object.demos;
|
||||
if (demos !== null && typeof demos === 'object') {
|
||||
for (const [slug, runs] of Object.entries(demos as Record<string, unknown>)) {
|
||||
if (Array.isArray(runs)) {
|
||||
for (const run of runs) {
|
||||
if (run !== null && typeof run === 'object') push(slug, run as RunRef);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* The demo registry: discovery by existence.
|
||||
*
|
||||
* There is no list of demos anywhere in this repo. A demo exists because
|
||||
* `src/demos/<slug>/meta.ts` and `src/demos/<slug>/demo.tsx` exist. Adding one
|
||||
* is `mkdir` plus two files; the header, the gallery, the vertical pages and
|
||||
* the router all pick it up with no edit to shared code. That property is the
|
||||
* whole reason this file is a glob and not an array, so resist the urge to
|
||||
* "just add an import" for the one awkward demo.
|
||||
*
|
||||
* Two globs, deliberately different:
|
||||
* - `meta.ts` is EAGER. Every page needs every meta (the header lists them),
|
||||
* it is plain serialisable data, and the contract forbids React or icon
|
||||
* components in it precisely so this eager glob stays cheap.
|
||||
* - `demo.tsx` is LAZY. It is the expensive half — components, adapters,
|
||||
* the reward source imported with `?raw` — and only one of them is ever
|
||||
* needed at a time.
|
||||
*
|
||||
* A demo whose meta is malformed is QUARANTINED: dropped from the registry with
|
||||
* a console error. One bad demo must never be able to white-page the site.
|
||||
*/
|
||||
|
||||
import type { DemoMeta, DemoModule, Vertical } from './types';
|
||||
|
||||
/**
|
||||
* The shell is generic over each demo's board type and never inspects it, but
|
||||
* the registry has to hand back demos of *different* board types from one
|
||||
* function. `DemoModule<unknown>` does not work: `Surface` takes `{ state: T }`
|
||||
* in a contravariant position, so `DemoModule<WordleState>` is not assignable
|
||||
* to `DemoModule<unknown>` and the shell could not pass a state back in either.
|
||||
* `any` is the one thing that is assignable in both directions here. Each demo
|
||||
* is still fully checked against `DemoModule<TState>` at its own `defineDemo()`
|
||||
* call site, which is where the type actually protects anyone.
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export type AnyDemoModule = DemoModule<any>;
|
||||
|
||||
/** Every vertical in the contract, in the order the site presents them. */
|
||||
export const VERTICAL_ORDER: readonly Vertical[] = [
|
||||
'reference',
|
||||
'support',
|
||||
'healthcare',
|
||||
'insurance',
|
||||
'financial-crime',
|
||||
'energy',
|
||||
'logistics',
|
||||
'code',
|
||||
'retail',
|
||||
'telecom',
|
||||
'data',
|
||||
'legal',
|
||||
];
|
||||
|
||||
/** Exec-facing names. The union member is a slug; this is what a person reads. */
|
||||
export const VERTICAL_LABELS: Readonly<Record<Vertical, string>> = {
|
||||
reference: 'Reference',
|
||||
support: 'Customer support',
|
||||
healthcare: 'Healthcare',
|
||||
insurance: 'Insurance',
|
||||
'financial-crime': 'Financial crime',
|
||||
energy: 'Energy',
|
||||
logistics: 'Logistics',
|
||||
code: 'Software',
|
||||
retail: 'Retail',
|
||||
telecom: 'Telecom',
|
||||
data: 'Data',
|
||||
legal: 'Legal',
|
||||
};
|
||||
|
||||
const VERTICAL_SET = new Set<string>(VERTICAL_ORDER);
|
||||
|
||||
export interface VerticalGroup {
|
||||
vertical: Vertical;
|
||||
label: string;
|
||||
demos: DemoMeta[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Demos may export their meta as `meta` or as the default. Both are accepted
|
||||
* because the alternative is a build that compiles and a site that is empty.
|
||||
*/
|
||||
interface MetaModuleShape {
|
||||
readonly meta?: unknown;
|
||||
readonly default?: unknown;
|
||||
}
|
||||
|
||||
interface DemoModuleShape {
|
||||
readonly demo?: unknown;
|
||||
readonly default?: unknown;
|
||||
}
|
||||
|
||||
const metaModules = import.meta.glob<MetaModuleShape>('../../demos/*/meta.ts', {
|
||||
eager: true,
|
||||
});
|
||||
|
||||
const demoLoaders = import.meta.glob<DemoModuleShape>('../../demos/*/demo.tsx');
|
||||
|
||||
/** `../../demos/wordle-five/meta.ts` -> `wordle-five` */
|
||||
function slugFromPath(path: string): string | null {
|
||||
const match = /\/demos\/([^/]+)\/[^/]+$/.exec(path);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === 'string' && value.trim().length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the reason this meta is unusable, or `null` if it is fine.
|
||||
*
|
||||
* Everything checked here is read without a guard by some surface. `slug` is
|
||||
* checked against the directory name rather than merely being present, because
|
||||
* a slug that disagrees with its directory produces a card that links to a
|
||||
* route that 404s — the single most confusing failure this registry can have.
|
||||
*/
|
||||
function validationError(candidate: unknown, dirName: string): string | null {
|
||||
if (candidate === null || typeof candidate !== 'object') {
|
||||
return 'meta.ts must export a `meta` object (or a default export).';
|
||||
}
|
||||
const meta = candidate as Partial<DemoMeta>;
|
||||
|
||||
if (!isNonEmptyString(meta.slug)) return '`slug` is missing or empty.';
|
||||
if (meta.slug !== dirName) {
|
||||
return `\`slug\` is "${meta.slug}" but the directory is "${dirName}". They must match.`;
|
||||
}
|
||||
if (!isNonEmptyString(meta.title)) return '`title` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.tagline)) return '`tagline` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.icon)) return '`icon` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.persona)) return '`persona` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.rewardLine)) return '`rewardLine` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.ogImage)) return '`ogImage` is missing or empty.';
|
||||
if (!isNonEmptyString(meta.vertical) || !VERTICAL_SET.has(meta.vertical)) {
|
||||
return `\`vertical\` is "${String(meta.vertical)}", which is not one of: ${VERTICAL_ORDER.join(', ')}.`;
|
||||
}
|
||||
if (meta.status !== 'live' && meta.status !== 'spec') {
|
||||
return `\`status\` is "${String(meta.status)}"; expected "live" or "spec".`;
|
||||
}
|
||||
if (typeof meta.order !== 'number' || !Number.isFinite(meta.order)) {
|
||||
return '`order` must be a finite number.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function byOrderThenSlug(a: DemoMeta, b: DemoMeta): number {
|
||||
return a.order - b.order || a.slug.localeCompare(b.slug);
|
||||
}
|
||||
|
||||
/** Built once at module load. Quarantine decisions are logged exactly once. */
|
||||
const demosBySlug: ReadonlyMap<string, DemoMeta> = (() => {
|
||||
const accepted = new Map<string, DemoMeta>();
|
||||
|
||||
for (const [path, module] of Object.entries(metaModules)) {
|
||||
const dirName = slugFromPath(path);
|
||||
if (dirName === null) {
|
||||
console.error(`[demo-kit] Ignoring "${path}": could not read a slug from the path.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const candidate = module.meta ?? module.default;
|
||||
const problem = validationError(candidate, dirName);
|
||||
if (problem !== null) {
|
||||
console.error(`[demo-kit] Quarantined demo "${dirName}": ${problem}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const meta = candidate as DemoMeta;
|
||||
if (!Object.hasOwn(demoLoaders, `../../demos/${dirName}/demo.tsx`)) {
|
||||
console.error(
|
||||
`[demo-kit] Quarantined demo "${dirName}": meta.ts exists but demo.tsx does not.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
accepted.set(meta.slug, meta);
|
||||
}
|
||||
|
||||
return accepted;
|
||||
})();
|
||||
|
||||
const orderedDemos: readonly DemoMeta[] = [...demosBySlug.values()].sort(byOrderThenSlug);
|
||||
|
||||
/** Every demo that survived validation, sorted by `order` then slug. */
|
||||
export function listDemos(): DemoMeta[] {
|
||||
return [...orderedDemos];
|
||||
}
|
||||
|
||||
/** One demo's meta, or `undefined` for an unknown or quarantined slug. */
|
||||
export function getDemo(slug: string): DemoMeta | undefined {
|
||||
return demosBySlug.get(slug);
|
||||
}
|
||||
|
||||
export function hasDemo(slug: string): boolean {
|
||||
return demosBySlug.has(slug);
|
||||
}
|
||||
|
||||
/**
|
||||
* Demos grouped by vertical, in `VERTICAL_ORDER`. Verticals with no demos are
|
||||
* omitted — an empty "Telecom" heading reads as a broken page, not a roadmap.
|
||||
*/
|
||||
export function listVerticals(): VerticalGroup[] {
|
||||
const groups = new Map<Vertical, DemoMeta[]>();
|
||||
for (const demo of orderedDemos) {
|
||||
const bucket = groups.get(demo.vertical);
|
||||
if (bucket) bucket.push(demo);
|
||||
else groups.set(demo.vertical, [demo]);
|
||||
}
|
||||
|
||||
return VERTICAL_ORDER.flatMap((vertical) => {
|
||||
const demos = groups.get(vertical);
|
||||
if (!demos || demos.length === 0) return [];
|
||||
return [{ vertical, label: VERTICAL_LABELS[vertical], demos }];
|
||||
});
|
||||
}
|
||||
|
||||
export function getVertical(slug: string): VerticalGroup | undefined {
|
||||
return listVerticals().find((group) => group.vertical === slug);
|
||||
}
|
||||
|
||||
/**
|
||||
* Promise cache, not value cache: two callers in the same tick (the route
|
||||
* loader and the page itself) share one dynamic import instead of racing.
|
||||
* A rejection is evicted so a failed chunk fetch can be retried.
|
||||
*/
|
||||
const moduleCache = new Map<string, Promise<AnyDemoModule>>();
|
||||
|
||||
/** Load a demo's heavy half. Rejects for an unknown or quarantined slug. */
|
||||
export function loadDemoModule(slug: string): Promise<AnyDemoModule> {
|
||||
const cached = moduleCache.get(slug);
|
||||
if (cached) return cached;
|
||||
|
||||
const meta = demosBySlug.get(slug);
|
||||
const loader = demoLoaders[`../../demos/${slug}/demo.tsx`];
|
||||
if (!meta || !loader) {
|
||||
return Promise.reject(new Error(`No demo named "${slug}".`));
|
||||
}
|
||||
|
||||
const pending = loader()
|
||||
.then((module) => {
|
||||
const candidate = module.demo ?? module.default;
|
||||
if (candidate === null || typeof candidate !== 'object') {
|
||||
throw new Error(`Demo "${slug}" does not export a demo object from demo.tsx.`);
|
||||
}
|
||||
const demo = candidate as AnyDemoModule;
|
||||
if (demo.meta?.slug !== slug) {
|
||||
throw new Error(
|
||||
`Demo "${slug}" exports a module whose meta.slug is "${String(demo.meta?.slug)}".`,
|
||||
);
|
||||
}
|
||||
return demo;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
moduleCache.delete(slug);
|
||||
throw error;
|
||||
});
|
||||
|
||||
moduleCache.set(slug, pending);
|
||||
return pending;
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Taking a reward apart, and putting it back together with different weights.
|
||||
*
|
||||
* The reward editor is the point of the whole site: change what "good" means
|
||||
* and watch the ranking move. This module is the arithmetic behind that, and it
|
||||
* has one job beyond adding numbers up — never to manufacture one. An unscored
|
||||
* component stays `null` all the way to the bar chart.
|
||||
*/
|
||||
|
||||
import { isNotScored, rewardTotal } from './episode';
|
||||
import type { RewardComponent, RewardValues } from './types';
|
||||
|
||||
/** Weights closer than this are the same weight. Guards float drift in sliders. */
|
||||
export const WEIGHT_EPSILON = 1e-9;
|
||||
|
||||
/** A user's edits to the shipped weights, keyed by component. */
|
||||
export type WeightOverrides = Readonly<Record<string, number>>;
|
||||
|
||||
export interface RewardRow {
|
||||
key: string;
|
||||
/** Plain English, straight off the component. */
|
||||
label: string;
|
||||
description: string;
|
||||
/** The raw score the environment emitted. `null` means not scored. */
|
||||
score: number | null;
|
||||
/** The weight in force — shipped, or edited, depending what you passed in. */
|
||||
weight: number;
|
||||
/** `score x weight`, the component's actual contribution. `null` when unscored. */
|
||||
value: number | null;
|
||||
role: RewardComponent['role'];
|
||||
}
|
||||
|
||||
export interface RewardBreakdown {
|
||||
rows: RewardRow[];
|
||||
/** Sum of the scored contributions, or `null` when nothing was scored. */
|
||||
total: number | null;
|
||||
/** How many components carry a real number. */
|
||||
scored: number;
|
||||
/** Components the environment did not grade. Rendered as "not scored". */
|
||||
unscored: string[];
|
||||
}
|
||||
|
||||
/** Per-component rows plus the total, ready to render. */
|
||||
export function decompose(
|
||||
values: RewardValues,
|
||||
components: readonly RewardComponent[],
|
||||
): RewardBreakdown {
|
||||
const rows: RewardRow[] = components.map((component) => {
|
||||
const raw = values[component.key];
|
||||
const scored = !isNotScored(raw);
|
||||
return {
|
||||
key: component.key,
|
||||
label: component.label,
|
||||
description: component.description,
|
||||
score: scored ? raw : null,
|
||||
weight: component.weight,
|
||||
value: scored ? raw * component.weight : null,
|
||||
role: component.role,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
rows,
|
||||
total: rewardTotal(values, components),
|
||||
scored: rows.filter((row) => row.score !== null).length,
|
||||
unscored: rows.filter((row) => row.score === null).map((row) => row.key),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the visitor's weight edits and renormalise so the weights sum to 1.0.
|
||||
*
|
||||
* Renormalising is what makes the editor honest. Without it, dragging one
|
||||
* slider up raises the total for every arm at once and the ranking looks like
|
||||
* it moved when only the scale did. With it, the visitor is trading weight
|
||||
* between components — which is the actual decision a reward designer makes.
|
||||
*
|
||||
* Negative weights are clamped to zero: a negative weight survives
|
||||
* normalisation as a sign flip somewhere else in the vector, and the resulting
|
||||
* chart is arithmetically correct and completely unreadable. If you want a
|
||||
* component to subtract, that belongs in the environment's grader, not here.
|
||||
*
|
||||
* If every weight is edited to zero the result is all zeros — there is no
|
||||
* honest way to normalise a zero vector, and inventing an equal split would be
|
||||
* putting words in the visitor's mouth. Call `weightsAreDegenerate()` on the
|
||||
* result and render "no weight assigned" rather than a 0.00 total.
|
||||
*/
|
||||
export function reweight(
|
||||
components: readonly RewardComponent[],
|
||||
overrides: WeightOverrides,
|
||||
): RewardComponent[] {
|
||||
const clamped = components.map((component) => {
|
||||
const override = overrides[component.key];
|
||||
const weight = override === undefined || !Number.isFinite(override) ? component.weight : override;
|
||||
return { component, weight: Math.max(0, weight) };
|
||||
});
|
||||
|
||||
const sum = clamped.reduce((acc, entry) => acc + entry.weight, 0);
|
||||
if (sum <= WEIGHT_EPSILON) {
|
||||
return clamped.map(({ component }) => ({ ...component, weight: 0 }));
|
||||
}
|
||||
|
||||
return clamped.map(({ component, weight }) => ({ ...component, weight: weight / sum }));
|
||||
}
|
||||
|
||||
/** True when `reweight` could not normalise, i.e. everything was zeroed. */
|
||||
export function weightsAreDegenerate(components: readonly RewardComponent[]): boolean {
|
||||
return components.reduce((acc, c) => acc + c.weight, 0) <= WEIGHT_EPSILON;
|
||||
}
|
||||
|
||||
/**
|
||||
* Has the visitor actually changed anything?
|
||||
*
|
||||
* Pass `components` whenever you have them. Without them this can only ask
|
||||
* "are there any override keys", which reports an edit for a slider that was
|
||||
* dragged and put back — and then the page shows a "modified reward" badge over
|
||||
* the shipped numbers, which is a lie in the other direction.
|
||||
*/
|
||||
export function isEdited(
|
||||
overrides: WeightOverrides,
|
||||
components?: readonly RewardComponent[],
|
||||
): boolean {
|
||||
const keys = Object.keys(overrides);
|
||||
if (keys.length === 0) return false;
|
||||
if (!components) return true;
|
||||
|
||||
return components.some((component) => {
|
||||
const override = overrides[component.key];
|
||||
if (override === undefined || !Number.isFinite(override)) return false;
|
||||
return Math.abs(override - component.weight) > WEIGHT_EPSILON;
|
||||
});
|
||||
}
|
||||
|
||||
/** Drop overrides that match the shipped weight, so a reset yields a clean URL. */
|
||||
export function pruneOverrides(
|
||||
overrides: WeightOverrides,
|
||||
components: readonly RewardComponent[],
|
||||
): WeightOverrides {
|
||||
const out: Record<string, number> = {};
|
||||
for (const component of components) {
|
||||
const override = overrides[component.key];
|
||||
if (override === undefined || !Number.isFinite(override)) continue;
|
||||
if (Math.abs(override - component.weight) > WEIGHT_EPSILON) out[component.key] = override;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Re-deriving a recorded reward in the visitor's own browser.
|
||||
*
|
||||
* The page claims the numbers on it are real. This is the only part of the site
|
||||
* that can actually demonstrate that rather than assert it: it runs the demo's
|
||||
* own `verify()` over the recorded trace and compares the answer to the numbers
|
||||
* shipped in the fixture.
|
||||
*
|
||||
* The failure modes are asymmetric and that asymmetry is the whole design.
|
||||
* `mismatch` is a serious accusation — it says the published fixture disagrees
|
||||
* with the code that supposedly produced it. It must only ever be reached by
|
||||
* comparing two real numbers. Everything else — no verifier, a truncated trace,
|
||||
* an ungraded run, a verifier that threw — is `unverifiable`, which is an
|
||||
* honest "we can't check this here" and is NEVER a zero and NEVER a mismatch.
|
||||
*/
|
||||
|
||||
import { isNotScored, rewardTotal } from './episode';
|
||||
import type { AnyDemoModule } from './registry';
|
||||
import type { DemoEpisode, RewardValues } from './types';
|
||||
|
||||
/**
|
||||
* Floating-point tolerance. The recorded numbers came out of Python and the
|
||||
* recomputed ones out of JavaScript; both are IEEE 754 doubles doing the same
|
||||
* arithmetic in a different order, so they agree to roughly this much and no
|
||||
* further. Anything above it is a real disagreement, not a rounding artefact.
|
||||
*/
|
||||
export const VERIFY_TOLERANCE = 1e-7;
|
||||
|
||||
export type VerifyStatus = 'match' | 'mismatch' | 'unverifiable';
|
||||
|
||||
export interface ComponentComparison {
|
||||
key: string;
|
||||
label: string;
|
||||
recorded: number | null;
|
||||
recomputed: number | null;
|
||||
/** `recomputed - recorded`, or `null` when either side is unscored. */
|
||||
delta: number | null;
|
||||
}
|
||||
|
||||
export interface VerifyResult {
|
||||
status: VerifyStatus;
|
||||
/** Weighted total from re-running the grader here. */
|
||||
recomputed: number | null;
|
||||
/** Weighted total as shipped in the fixture. */
|
||||
recorded: number | null;
|
||||
/** `recomputed - recorded`. `null` when either side is unavailable. */
|
||||
delta: number | null;
|
||||
/**
|
||||
* The first component that disagrees, worst delta first. Named so the UI can
|
||||
* say WHICH term is wrong instead of just flashing red at a total.
|
||||
*/
|
||||
culprit?: ComponentComparison;
|
||||
/** Every component that could be compared, plus the ones that could not. */
|
||||
components: ComponentComparison[];
|
||||
/** Why this is unverifiable, in a sentence fit to render. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
function unverifiable(reason: string, recorded: number | null = null): VerifyResult {
|
||||
return { status: 'unverifiable', recomputed: null, recorded, delta: null, components: [], reason };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a demo's browser-side grader against its recorded fixture.
|
||||
*
|
||||
* Pure and synchronous: the demo's `verify()` is required to be pure over the
|
||||
* episode, so this can run during render without a loading state.
|
||||
*/
|
||||
export function verifyEpisode(module: AnyDemoModule, episode: DemoEpisode): VerifyResult {
|
||||
const components = module.reward.components;
|
||||
const recordedTotal = rewardTotal(episode.rewards, components);
|
||||
|
||||
if (typeof module.verify !== 'function') {
|
||||
return unverifiable('This demo does not ship a browser-side grader, so the recorded numbers cannot be re-derived here. The environment source is in the repository.', recordedTotal);
|
||||
}
|
||||
|
||||
if (episode.truncated === true) {
|
||||
return unverifiable('The recorded trace is truncated, so the grader has nothing complete to score. A truncated run is unverifiable, not a zero.', recordedTotal);
|
||||
}
|
||||
|
||||
let recomputedValues: RewardValues | null;
|
||||
try {
|
||||
recomputedValues = module.verify(episode);
|
||||
} catch (error: unknown) {
|
||||
// A grader that throws is a bug in the grader, not evidence about the run.
|
||||
// Reporting it as a mismatch would accuse the fixture of being wrong.
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
return unverifiable(`The browser-side grader could not run: ${detail}`, recordedTotal);
|
||||
}
|
||||
|
||||
if (recomputedValues === null) {
|
||||
return unverifiable('The demo reported this run as unverifiable — the trace does not carry everything the grader needs.', recordedTotal);
|
||||
}
|
||||
|
||||
const comparisons: ComponentComparison[] = components.map((component) => {
|
||||
const recordedRaw = episode.rewards[component.key];
|
||||
const recomputedRaw = recomputedValues[component.key];
|
||||
const recorded = isNotScored(recordedRaw) ? null : recordedRaw;
|
||||
const recomputed = isNotScored(recomputedRaw) ? null : recomputedRaw;
|
||||
return {
|
||||
key: component.key,
|
||||
label: component.label,
|
||||
recorded,
|
||||
recomputed,
|
||||
delta: recorded === null || recomputed === null ? null : recomputed - recorded,
|
||||
};
|
||||
});
|
||||
|
||||
const comparable = comparisons.filter((c) => c.delta !== null);
|
||||
if (comparable.length === 0) {
|
||||
return unverifiable('This run carries no graded components to compare against, so there is nothing to verify. Not scored is not zero.', recordedTotal);
|
||||
}
|
||||
|
||||
const recomputedTotal = rewardTotal(recomputedValues, components);
|
||||
|
||||
// Worst first, so the culprit is the component that actually moved the total.
|
||||
const disagreeing = comparable
|
||||
.filter((c) => Math.abs(c.delta ?? 0) > VERIFY_TOLERANCE)
|
||||
.sort((a, b) => Math.abs(b.delta ?? 0) - Math.abs(a.delta ?? 0));
|
||||
|
||||
const totalDelta =
|
||||
recomputedTotal === null || recordedTotal === null ? null : recomputedTotal - recordedTotal;
|
||||
|
||||
const totalsAgree = totalDelta !== null && Math.abs(totalDelta) <= VERIFY_TOLERANCE;
|
||||
const matched = disagreeing.length === 0 && totalsAgree;
|
||||
|
||||
const result: VerifyResult = {
|
||||
status: matched ? 'match' : 'mismatch',
|
||||
recomputed: recomputedTotal,
|
||||
recorded: recordedTotal,
|
||||
delta: totalDelta,
|
||||
components: comparisons,
|
||||
};
|
||||
|
||||
const culprit = disagreeing[0];
|
||||
if (culprit) result.culprit = culprit;
|
||||
return result;
|
||||
}
|
||||
|
||||
/** One line an exec can read, given a result. Keeps the wording in one place. */
|
||||
export function verifySummary(result: VerifyResult): string {
|
||||
switch (result.status) {
|
||||
case 'match':
|
||||
return 'Re-computed in your browser from the recorded trace. It matches the published number.';
|
||||
case 'mismatch':
|
||||
return result.culprit
|
||||
? `Re-computed in your browser and it disagrees on "${result.culprit.label}".`
|
||||
: 'Re-computed in your browser and it disagrees with the published number.';
|
||||
case 'unverifiable':
|
||||
return result.reason ?? 'This run cannot be re-computed in the browser.';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { clsx, type ClassValue } from 'clsx';
|
||||
import { twMerge } from 'tailwind-merge';
|
||||
|
||||
/**
|
||||
* The shadcn `cn`. `twMerge` is v2 here — importing from a v3 path
|
||||
* (`tailwind-merge/v3` or the `createTailwindMerge` split entry) resolves at
|
||||
* type level and then fails at bundle time, so keep this import bare.
|
||||
*/
|
||||
export function cn(...inputs: ClassValue[]): string {
|
||||
return twMerge(clsx(inputs));
|
||||
}
|
||||
Reference in New Issue
Block a user