Capture harness, fixture verification, CI, and the public README
The site does no live inference. Rollouts are captured once against spark-1 and replayed at their recorded wall-clock — a public demo with no auth cannot hold an API key, and a recorded run can be scrubbed, permalinked, blind-compared and verified in ways a live one cannot. What stops it being a video is that the browser re-derives every number from the recorded moves. verify_fixtures.py is the Python half of that: it replays every committed fixture through the engine and reproduces its own rewards. All 16 land at delta 0.0. A fixture that cannot be regenerated is a claim with no receipt. First real measurement, thinking off, 8 seeds: solved 0/8. The model repeats guesses it has already played, invents words (trape, slith, postt, boomy), and contradicts its own feedback — consistency 0.09 to 0.17. That is the published failure taxonomy showing up in our own data on the first run, and it is why `consistency` is a reward component rather than a footnote. A capture failure is recorded as a turn with a null reply, never dropped. A capture that silently discarded failed turns would be reporting a better model than the one that ran. CI gates both halves and four things that fail silently in production: the word lists must rebuild byte-identically, the prerendered routes must carry their own baked og tags (crawlers do not run JS, so without them every shared link previews as the homepage), no blob: URL may reach the bundle (the site's CSP has no worker-src, so it falls back to default-src 'self' and a blob worker is blocked with no error), and the conformance digest must match across languages. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
@@ -0,0 +1,340 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import * as Slider from '@radix-ui/react-slider';
|
||||
import { ArrowDown, ArrowUp, Minus, RotateCcw } from 'lucide-react';
|
||||
import type { RewardSpec, RewardValues } from '@/lib/demo-kit/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { formatNumber, formatOrDash } from './format';
|
||||
import { scoreReward, shippedWeights } from './reward-math';
|
||||
import { EditedChip } from './StatStrip';
|
||||
|
||||
/** One recorded arm — a run, or a group of runs already reduced to one score. */
|
||||
export interface RewardArm {
|
||||
id: string;
|
||||
label: string;
|
||||
/** The environment's per-component scores. Re-weighted, never re-run. */
|
||||
values: RewardValues;
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export interface RewardPreset {
|
||||
id: string;
|
||||
label: string;
|
||||
description?: string;
|
||||
weights: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface RewardEditorProps {
|
||||
spec: RewardSpec;
|
||||
arms: RewardArm[];
|
||||
/** Two is the right number. More and the visitor reads instead of playing. */
|
||||
presets?: RewardPreset[];
|
||||
onWeightsChange?: (weights: Record<string, number>, edited: boolean) => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const STEP = 0.05;
|
||||
|
||||
/**
|
||||
* Presets built from the spec's own labels, for a demo that does not supply
|
||||
* its own. Both are stated as the choice a buyer would actually argue for in a
|
||||
* meeting, not as "preset A" and "preset B".
|
||||
*/
|
||||
function derivePresets(spec: RewardSpec): RewardPreset[] {
|
||||
const shipped = shippedWeights(spec);
|
||||
const counterweights = spec.components.filter((c) => c.role === 'counterweight');
|
||||
const objectives = spec.components.filter((c) => c.role === 'objective');
|
||||
const presets: RewardPreset[] = [
|
||||
{
|
||||
id: 'shipped',
|
||||
label: 'What we ship',
|
||||
description: 'The weights in the environment as committed.',
|
||||
weights: shipped,
|
||||
},
|
||||
];
|
||||
|
||||
const firstObjective = objectives[0];
|
||||
if (counterweights.length > 0 && firstObjective) {
|
||||
const onlyObjective = { ...shipped };
|
||||
for (const component of counterweights) onlyObjective[component.key] = 0;
|
||||
presets.push({
|
||||
id: 'objective-only',
|
||||
label: `${firstObjective.label} at any cost`,
|
||||
description: `Drops ${counterweights
|
||||
.map((c) => c.label.toLowerCase())
|
||||
.join(' and ')} to zero.`,
|
||||
weights: onlyObjective,
|
||||
});
|
||||
|
||||
const doubled = { ...shipped };
|
||||
for (const component of counterweights) doubled[component.key] = component.weight * 2;
|
||||
presets.push({
|
||||
id: 'counterweight-heavy',
|
||||
label: `Double ${counterweights[0]?.label.toLowerCase() ?? 'the counterweight'}`,
|
||||
description: 'What a risk-averse buyer would ask for.',
|
||||
weights: doubled,
|
||||
});
|
||||
}
|
||||
return presets;
|
||||
}
|
||||
|
||||
function weightsEqual(a: Record<string, number>, b: Record<string, number>): boolean {
|
||||
const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
|
||||
for (const key of keys) {
|
||||
if (Math.abs((a[key] ?? 0) - (b[key] ?? 0)) > 1e-9) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Change what "good" means and watch the ranking move.
|
||||
*
|
||||
* The honesty problem this component has to solve: re-weighting recorded scores
|
||||
* is NOT training. It shows you the ranking a different reward would have
|
||||
* produced over these exact attempts; it cannot show you the different attempts
|
||||
* a model trained on that reward would have made. That distinction is the
|
||||
* permanent caption at the bottom, and it is not collapsible.
|
||||
*/
|
||||
export function RewardEditor({
|
||||
spec,
|
||||
arms,
|
||||
presets,
|
||||
onWeightsChange,
|
||||
className,
|
||||
}: RewardEditorProps) {
|
||||
const shipped = useMemo(() => shippedWeights(spec), [spec]);
|
||||
const [weights, setWeights] = useState<Record<string, number>>(shipped);
|
||||
const effectivePresets = useMemo(() => presets ?? derivePresets(spec), [presets, spec]);
|
||||
|
||||
const bounds = useMemo(() => {
|
||||
const values = spec.components.map((c) => c.weight);
|
||||
const max = Math.max(2, ...values.map((v) => Math.ceil(Math.abs(v) * 2)));
|
||||
const min = Math.min(0, ...values.map((v) => Math.floor(v)));
|
||||
return { min, max };
|
||||
}, [spec]);
|
||||
|
||||
const edited = !weightsEqual(weights, shipped);
|
||||
|
||||
const apply = (next: Record<string, number>) => {
|
||||
setWeights(next);
|
||||
onWeightsChange?.(next, !weightsEqual(next, shipped));
|
||||
};
|
||||
|
||||
const ranked = useMemo(() => {
|
||||
const shippedTotals = new Map(
|
||||
arms.map((arm) => [arm.id, scoreReward(spec, arm.values).total]),
|
||||
);
|
||||
const rows = arms.map((arm) => ({
|
||||
arm,
|
||||
total: scoreReward(spec, arm.values, weights).total,
|
||||
shippedTotal: shippedTotals.get(arm.id) ?? null,
|
||||
}));
|
||||
// Nulls sort last: an unscored arm is not a zero-scoring arm.
|
||||
const byTotal = (a: { total: number | null }, b: { total: number | null }) => {
|
||||
if (a.total === null && b.total === null) return 0;
|
||||
if (a.total === null) return 1;
|
||||
if (b.total === null) return -1;
|
||||
return b.total - a.total;
|
||||
};
|
||||
const shippedOrder = [...rows]
|
||||
.sort((a, b) => byTotal({ total: a.shippedTotal }, { total: b.shippedTotal }))
|
||||
.map((row) => row.arm.id);
|
||||
return [...rows].sort(byTotal).map((row, index) => ({
|
||||
...row,
|
||||
rank: index + 1,
|
||||
shippedRank: shippedOrder.indexOf(row.arm.id) + 1,
|
||||
}));
|
||||
}, [arms, spec, weights]);
|
||||
|
||||
const span = useMemo(() => {
|
||||
const totals = ranked.map((row) => row.total).filter((t): t is number => t !== null);
|
||||
if (totals.length === 0) return { lo: 0, hi: 1 };
|
||||
const lo = Math.min(0, ...totals);
|
||||
const hi = Math.max(...totals);
|
||||
return { lo, hi: hi === lo ? lo + 1 : hi };
|
||||
}, [ranked]);
|
||||
|
||||
const leader = ranked[0];
|
||||
|
||||
return (
|
||||
<div className={cn('grid gap-4 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)]', className)}>
|
||||
<section aria-label="Reward weights" className="card p-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h3 className="text-sm font-semibold">Change what good means</h3>
|
||||
{edited ? <EditedChip /> : null}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => apply(shipped)}
|
||||
disabled={!edited}
|
||||
className="tap ml-auto inline-flex items-center gap-1.5 rounded-lg border border-border px-2.5 text-xs font-medium transition-colors duration-2 ease-enter hover:bg-surface-2 disabled:opacity-40"
|
||||
>
|
||||
<RotateCcw className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
Reset to shipped
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="mt-2 flex flex-wrap gap-2">
|
||||
{effectivePresets.map((preset) => {
|
||||
const active = weightsEqual(weights, preset.weights);
|
||||
return (
|
||||
<button
|
||||
key={preset.id}
|
||||
type="button"
|
||||
onClick={() => apply({ ...preset.weights })}
|
||||
aria-pressed={active}
|
||||
title={preset.description ?? preset.label}
|
||||
className={cn(
|
||||
'tap rounded-lg border px-3 text-left text-xs font-medium transition-colors duration-2 ease-enter',
|
||||
active
|
||||
? 'border-brand bg-accent-subtle text-accent-fg'
|
||||
: 'border-border hover:bg-surface-2',
|
||||
)}
|
||||
>
|
||||
{preset.label}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div className="mt-4 space-y-4">
|
||||
{spec.components.map((component) => {
|
||||
const value = weights[component.key] ?? component.weight;
|
||||
const changed = Math.abs(value - component.weight) > 1e-9;
|
||||
return (
|
||||
<div key={component.key}>
|
||||
<div className="flex items-baseline justify-between gap-2">
|
||||
{/* A <label htmlFor> would point at Radix's root <span>,
|
||||
which is not a labelable element — the association would
|
||||
silently do nothing. The thumb takes its name from this
|
||||
text via aria-labelledby instead. */}
|
||||
<span
|
||||
id={`weight-label-${component.key}`}
|
||||
className="text-sm font-medium text-fg"
|
||||
>
|
||||
{component.label}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
'nums font-mono text-sm',
|
||||
changed ? 'font-semibold text-accent-fg' : 'text-muted',
|
||||
)}
|
||||
>
|
||||
{formatNumber(value, 2)}
|
||||
</span>
|
||||
</div>
|
||||
<p className="mb-1.5 text-xs leading-snug text-muted">{component.description}</p>
|
||||
<Slider.Root
|
||||
className="relative flex h-6 w-full touch-none select-none items-center"
|
||||
min={bounds.min}
|
||||
max={bounds.max}
|
||||
step={STEP}
|
||||
value={[value]}
|
||||
onValueChange={(next) =>
|
||||
apply({ ...weights, [component.key]: next[0] ?? component.weight })
|
||||
}
|
||||
>
|
||||
<Slider.Track className="relative h-1.5 w-full grow rounded-full bg-surface-2">
|
||||
<Slider.Range className="absolute h-full rounded-full bg-brand" />
|
||||
</Slider.Track>
|
||||
{/* 44px of hit area around a 16px dot: the visible thumb is
|
||||
small enough to read the track under it, and still catches
|
||||
a thumb on a phone. */}
|
||||
<Slider.Thumb
|
||||
aria-labelledby={`weight-label-${component.key}`}
|
||||
className="block h-6 w-6 rounded-full border-4 border-brand bg-surface shadow-sm"
|
||||
/>
|
||||
</Slider.Root>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section aria-label="Ranking under this reward" className="card flex flex-col p-3">
|
||||
<h3 className="text-sm font-semibold">
|
||||
Ranking under this reward {edited ? <EditedChip className="ml-1 align-middle" /> : null}
|
||||
</h3>
|
||||
|
||||
<ol className="mt-3 space-y-2">
|
||||
{ranked.map((row) => {
|
||||
const moved = row.rank - row.shippedRank;
|
||||
const width =
|
||||
row.total === null
|
||||
? 0
|
||||
: Math.max(2, ((row.total - span.lo) / (span.hi - span.lo)) * 100);
|
||||
return (
|
||||
<li
|
||||
key={row.arm.id}
|
||||
className={cn(
|
||||
'rounded-lg border p-2.5 transition-colors duration-3 ease-enter',
|
||||
row.rank === 1 ? 'border-brand bg-accent-subtle/50' : 'border-border',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className="nums text-sm font-semibold text-muted">{row.rank}</span>
|
||||
<span className="min-w-0 flex-1 truncate text-sm font-medium">
|
||||
{row.arm.label}
|
||||
</span>
|
||||
<RankMove moved={moved} />
|
||||
<span className="nums font-mono text-sm font-semibold">
|
||||
{formatOrDash(row.total)}
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
aria-hidden="true"
|
||||
className="mt-1.5 h-1.5 w-full overflow-hidden rounded-full bg-surface-2"
|
||||
>
|
||||
<div
|
||||
className="h-full rounded-full bg-brand transition-[width] duration-3 ease-enter"
|
||||
style={{ width: `${width}%` }}
|
||||
/>
|
||||
</div>
|
||||
{row.arm.note ? (
|
||||
<p className="mt-1 text-xs text-muted">{row.arm.note}</p>
|
||||
) : null}
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ol>
|
||||
|
||||
{/* User-initiated, so it is safe to announce here without fighting the
|
||||
shell's step-change region: the two never fire from one action. */}
|
||||
<p role="status" className="sr-only">
|
||||
{leader
|
||||
? `Leading under this reward: ${leader.arm.label}, ${formatOrDash(leader.total)}.`
|
||||
: 'No arms to rank.'}
|
||||
</p>
|
||||
|
||||
<p className="mt-auto pt-3 text-xs leading-relaxed text-muted">
|
||||
We re-scored the same recorded attempts under your reward. Training on it would change
|
||||
the behaviour, not just the ranking.
|
||||
</p>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RankMove({ moved }: { moved: number }) {
|
||||
if (moved === 0) {
|
||||
return (
|
||||
<span className="inline-flex items-center text-muted" title="Same rank as shipped">
|
||||
<Minus className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
<span className="sr-only">unchanged</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
const up = moved < 0;
|
||||
return (
|
||||
<span
|
||||
className={cn('nums inline-flex items-center text-xs', up ? 'text-positive' : 'text-danger')}
|
||||
title={`${Math.abs(moved)} place${Math.abs(moved) === 1 ? '' : 's'} ${up ? 'up' : 'down'} from the shipped reward`}
|
||||
>
|
||||
{up ? (
|
||||
<ArrowUp className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
) : (
|
||||
<ArrowDown className="h-3.5 w-3.5" aria-hidden="true" />
|
||||
)}
|
||||
{Math.abs(moved)}
|
||||
<span className="sr-only">{up ? ' places up' : ' places down'}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user