Pin seed->word across both languages with a shared hash
engine.ts used mulberry32 and engine.py used random.Random(seed). Same seed, different word — so every ?seed= permalink on the site would have shown a different puzzle than the recorded run it claimed to be replaying, and nobody would have noticed until someone checked one by hand. Both now derive the index from FNV-1a 32-bit over the decimal seed. A hash rather than a PRNG because there is no honest one-line JavaScript equivalent of Mersenne Twister, and this way there is nothing to keep in step: both sides compute the same integer from the same string. Math.imul on the JS side is load-bearing — a plain multiply overflows into a double and diverges after the first few bytes. Twelve seeds are pinned as a vector in both test suites. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
@@ -0,0 +1,82 @@
|
||||
import type { ModelCall } from '@/lib/demo-kit/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { DASH, formatInt, formatMs, humaniseToken } from './format';
|
||||
|
||||
export interface ModelCallPanelProps {
|
||||
call: ModelCall | null;
|
||||
className?: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
interface Row {
|
||||
label: string;
|
||||
value: string;
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The model call, straight off the trace.
|
||||
*
|
||||
* Nothing here is computed, averaged or estimated. Every field is a value the
|
||||
* recorder wrote down, and a field the recorder did not write down renders as
|
||||
* an em dash — never as zero, and never quietly omitted. A provider that does
|
||||
* not report reasoning tokens is a fact about the trace, and hiding the row
|
||||
* would turn "we do not know" into "there were none".
|
||||
*/
|
||||
export function ModelCallPanel({ call, className, title = 'Model call' }: ModelCallPanelProps) {
|
||||
const rows: Row[] = [
|
||||
{
|
||||
label: 'finish_reason',
|
||||
value: call?.finishReason ? humaniseToken(call.finishReason) : DASH,
|
||||
hint: 'Why the model stopped generating',
|
||||
},
|
||||
{
|
||||
label: 'Prompt tokens',
|
||||
value: formatInt(call?.promptTokens ?? null),
|
||||
hint: 'Everything sent in: system, board, history',
|
||||
},
|
||||
{
|
||||
label: 'Completion tokens',
|
||||
value: formatInt(call?.completionTokens ?? null),
|
||||
hint: 'The visible reply',
|
||||
},
|
||||
{
|
||||
label: 'Reasoning tokens',
|
||||
value: formatInt(call?.reasoningTokens ?? null),
|
||||
hint: 'Billed thinking, when the provider reports it',
|
||||
},
|
||||
{
|
||||
label: 'Latency',
|
||||
value: formatMs(call?.durationMs ?? null),
|
||||
hint: 'Real elapsed time when the run was recorded',
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<section aria-label={title} className={cn('card overflow-hidden', className)}>
|
||||
<header className="border-b border-border px-3 py-2">
|
||||
<h3 className="text-sm font-semibold">{title}</h3>
|
||||
</header>
|
||||
{call === null ? (
|
||||
<p className="px-3 py-3 text-sm text-muted">
|
||||
This step did not involve a model call — it is a state change the environment made on
|
||||
its own.
|
||||
</p>
|
||||
) : (
|
||||
<dl className="divide-y divide-border">
|
||||
{rows.map((row) => (
|
||||
<div key={row.label} className="flex items-baseline gap-3 px-3 py-2">
|
||||
<dt className="min-w-0 flex-1">
|
||||
<span className="block text-sm font-medium">{row.label}</span>
|
||||
{row.hint ? (
|
||||
<span className="block text-xs leading-snug text-muted">{row.hint}</span>
|
||||
) : null}
|
||||
</dt>
|
||||
<dd className="nums shrink-0 font-mono text-sm text-fg">{row.value}</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { useState } from 'react';
|
||||
import { Drawer } from 'vaul';
|
||||
import { Brain, ChevronUp } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { ReasoningPanel } from './ReasoningPanel';
|
||||
import type { ReasoningPanelProps } from './ReasoningPanel';
|
||||
|
||||
const SNAP_POINTS = [0.4, 0.9];
|
||||
|
||||
export interface ReasoningDrawerProps extends ReasoningPanelProps {
|
||||
/** Controlled from the shell when it wants the drawer open on a step change. */
|
||||
open?: boolean;
|
||||
onOpenChange?: (open: boolean) => void;
|
||||
triggerClassName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The reasoning panel, for a phone.
|
||||
*
|
||||
* Below `lg` there is no room for a column beside the board, and putting the
|
||||
* reasoning under the board means the visitor watches the run with the thinking
|
||||
* off-screen. A drawer at 40% shows the first few lines without covering the
|
||||
* board; dragging to 90% is the "let me actually read this" gesture.
|
||||
*
|
||||
* Rendering is caller-gated rather than CSS-gated: mounting a vaul drawer on
|
||||
* desktop and hiding it with `lg:hidden` still locks body scroll when it opens,
|
||||
* so the shell mounts this only under `lg`.
|
||||
*/
|
||||
export function ReasoningDrawer({
|
||||
open,
|
||||
onOpenChange,
|
||||
triggerClassName,
|
||||
...panel
|
||||
}: ReasoningDrawerProps) {
|
||||
const [snap, setSnap] = useState<number | string | null>(SNAP_POINTS[0] ?? 0.4);
|
||||
const hasReasoning = (panel.reasoning ?? '').length > 0;
|
||||
|
||||
return (
|
||||
<Drawer.Root
|
||||
snapPoints={SNAP_POINTS}
|
||||
activeSnapPoint={snap}
|
||||
setActiveSnapPoint={setSnap}
|
||||
{...(open === undefined ? {} : { open })}
|
||||
{...(onOpenChange ? { onOpenChange } : {})}
|
||||
>
|
||||
<Drawer.Trigger
|
||||
className={cn(
|
||||
'tap flex w-full items-center gap-2 rounded-lg border border-border bg-surface px-3 py-2 text-sm font-medium transition-colors duration-2 ease-enter hover:bg-surface-2',
|
||||
triggerClassName,
|
||||
)}
|
||||
>
|
||||
<Brain className="h-4 w-4 text-muted" aria-hidden="true" />
|
||||
<span>{hasReasoning ? 'Read the reasoning' : 'No reasoning on this step'}</span>
|
||||
<ChevronUp className="ml-auto h-4 w-4 text-muted" aria-hidden="true" />
|
||||
</Drawer.Trigger>
|
||||
<Drawer.Portal>
|
||||
<Drawer.Overlay className="fixed inset-0 z-40 bg-fg/40" />
|
||||
<Drawer.Content
|
||||
className="fixed inset-x-0 bottom-0 z-50 mx-auto flex h-full max-h-[97%] max-w-canvas flex-col rounded-t-xl border border-border bg-surface outline-none"
|
||||
style={{ paddingBottom: 'var(--safe-bottom)' }}
|
||||
>
|
||||
<div
|
||||
aria-hidden="true"
|
||||
className="mx-auto mt-2 h-1.5 w-12 shrink-0 rounded-full bg-border"
|
||||
/>
|
||||
<div className="px-4 pb-2 pt-3">
|
||||
<Drawer.Title className="text-sm font-semibold">
|
||||
{panel.title ?? 'Reasoning'}
|
||||
</Drawer.Title>
|
||||
<Drawer.Description className="text-xs text-muted">
|
||||
Recorded verbatim from step {panel.stepIndex + 1} of this run.
|
||||
</Drawer.Description>
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-hidden px-4 pb-4">
|
||||
{/* The panel keeps its own reserved height inside the sheet so the
|
||||
sheet does not resize as the text streams under the drag. */}
|
||||
<ReasoningPanel {...panel} className="h-full border-0" reservedLines={14} />
|
||||
</div>
|
||||
</Drawer.Content>
|
||||
</Drawer.Portal>
|
||||
</Drawer.Root>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import * as ScrollArea from '@radix-ui/react-scroll-area';
|
||||
import { Brain } from 'lucide-react';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { usePrefersReducedMotion } from './format';
|
||||
import type { PlaybackSpeed } from './TracePlayer';
|
||||
|
||||
/**
|
||||
* Characters per second, clamped. A 40-character reasoning trace recorded over
|
||||
* 30 seconds would otherwise crawl at 1.3 chars/s and read as a hung page,
|
||||
* and a 6,000-character one recorded in 800 ms would flash past unread.
|
||||
*/
|
||||
const MIN_CPS = 24;
|
||||
const MAX_CPS = 900;
|
||||
const FALLBACK_CPS = 90;
|
||||
|
||||
export interface ReasoningPanelProps {
|
||||
reasoning: string | null;
|
||||
/** The recorded latency of the call this reasoning came from. */
|
||||
durationMs: number | null;
|
||||
playing: boolean;
|
||||
speed: PlaybackSpeed;
|
||||
/** Changing this restarts the stream. Pass the step index. */
|
||||
stepIndex: number;
|
||||
/**
|
||||
* Lines of height held open whether or not there is text. Reserving the box
|
||||
* is not a nicety: this panel sits beside the board, and letting it grow as
|
||||
* the text arrives shoves the board down the page mid-run.
|
||||
*/
|
||||
reservedLines?: number;
|
||||
title?: string;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
function charsPerSecond(length: number, durationMs: number | null): number {
|
||||
if (!durationMs || durationMs <= 0 || length === 0) return FALLBACK_CPS;
|
||||
return Math.min(Math.max((length / durationMs) * 1000, MIN_CPS), MAX_CPS);
|
||||
}
|
||||
|
||||
export function ReasoningPanel({
|
||||
reasoning,
|
||||
durationMs,
|
||||
playing,
|
||||
speed,
|
||||
stepIndex,
|
||||
reservedLines = 10,
|
||||
title = 'Reasoning',
|
||||
className,
|
||||
}: ReasoningPanelProps) {
|
||||
const reducedMotion = usePrefersReducedMotion();
|
||||
const full = reasoning ?? '';
|
||||
const [visible, setVisible] = useState(full.length);
|
||||
const viewportRef = useRef<HTMLDivElement | null>(null);
|
||||
|
||||
// Whether this step's text should stream at all. Scrubbing to a step while
|
||||
// paused shows it whole — someone reading at their own pace is not asking to
|
||||
// be typed at.
|
||||
const shouldStream = playing && speed !== 'instant' && !reducedMotion && full.length > 0;
|
||||
|
||||
useEffect(() => {
|
||||
if (!shouldStream) {
|
||||
setVisible(full.length);
|
||||
return;
|
||||
}
|
||||
setVisible(0);
|
||||
const cps = charsPerSecond(full.length, durationMs) * (typeof speed === 'number' ? speed : 1);
|
||||
const started = performance.now();
|
||||
let frame = 0;
|
||||
let last = -1;
|
||||
const tick = (now: number) => {
|
||||
const next = Math.min(Math.floor(((now - started) / 1000) * cps), full.length);
|
||||
// Only re-render when a character actually lands; at 120 Hz the naive
|
||||
// version re-renders the whole panel twice per revealed character.
|
||||
if (next !== last) {
|
||||
last = next;
|
||||
setVisible(next);
|
||||
const viewport = viewportRef.current;
|
||||
if (viewport) viewport.scrollTop = viewport.scrollHeight;
|
||||
}
|
||||
if (next < full.length) frame = requestAnimationFrame(tick);
|
||||
};
|
||||
frame = requestAnimationFrame(tick);
|
||||
return () => cancelAnimationFrame(frame);
|
||||
}, [shouldStream, full, durationMs, speed, stepIndex]);
|
||||
|
||||
const streaming = visible < full.length;
|
||||
|
||||
return (
|
||||
<section
|
||||
aria-label={title}
|
||||
className={cn('card flex flex-col overflow-hidden', className)}
|
||||
>
|
||||
<header className="flex items-center gap-2 border-b border-border px-3 py-2">
|
||||
<Brain className="h-4 w-4 text-muted" aria-hidden="true" />
|
||||
<h3 className="text-sm font-semibold">{title}</h3>
|
||||
{streaming ? (
|
||||
<span className="nums ml-auto text-xs text-muted">
|
||||
{visible}/{full.length}
|
||||
</span>
|
||||
) : null}
|
||||
</header>
|
||||
|
||||
<ScrollArea.Root
|
||||
type="auto"
|
||||
className="min-h-0 flex-1"
|
||||
// Height, not min-height: the panel is the same size on every step,
|
||||
// including the ones with no reasoning at all.
|
||||
style={{ height: `${reservedLines * 1.45}rem` }}
|
||||
>
|
||||
<ScrollArea.Viewport
|
||||
ref={viewportRef}
|
||||
className="h-full w-full px-3 py-2.5"
|
||||
// The shell owns the page's single polite live region. A streaming
|
||||
// region here would read every partial word over the top of it.
|
||||
aria-live="off"
|
||||
>
|
||||
{full.length === 0 ? (
|
||||
<p className="text-sm leading-relaxed text-muted">
|
||||
This step has no recorded reasoning. The run was captured with thinking disabled, so
|
||||
there is nothing to show here — which is different from the model having thought
|
||||
nothing.
|
||||
</p>
|
||||
) : (
|
||||
<p className="whitespace-pre-wrap font-mono text-[13px] leading-relaxed text-fg">
|
||||
{full.slice(0, visible)}
|
||||
{streaming ? (
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className="ml-px inline-block h-[1em] w-[0.5ch] translate-y-[0.15em] animate-pulse bg-brand align-baseline"
|
||||
/>
|
||||
) : null}
|
||||
</p>
|
||||
)}
|
||||
</ScrollArea.Viewport>
|
||||
<ScrollArea.Scrollbar
|
||||
orientation="vertical"
|
||||
className="flex w-2 touch-none select-none p-0.5"
|
||||
>
|
||||
<ScrollArea.Thumb className="flex-1 rounded-full bg-border" />
|
||||
</ScrollArea.Scrollbar>
|
||||
</ScrollArea.Root>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
import { Scale, Target, Weight } from 'lucide-react';
|
||||
import type { LucideIcon } from 'lucide-react';
|
||||
import type { RewardComponent, RewardSpec, RewardValues } from '@/lib/demo-kit/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { DASH, formatNumber, formatOrDash } from './format';
|
||||
import { scoreReward } from './reward-math';
|
||||
import { EditedChip } from './StatStrip';
|
||||
|
||||
const ROLE_META: Record<RewardComponent['role'], { label: string; Icon: LucideIcon }> = {
|
||||
objective: { label: 'Objective', Icon: Target },
|
||||
counterweight: { label: 'Counterweight', Icon: Weight },
|
||||
gate: { label: 'Gate', Icon: Scale },
|
||||
};
|
||||
|
||||
export interface RewardBreakdownProps {
|
||||
spec: RewardSpec;
|
||||
values: RewardValues;
|
||||
/** Unweighted diagnostics. Rendered, never summed — the contract is explicit. */
|
||||
metrics?: Record<string, number | null>;
|
||||
/** Overridden weights from the editor. Absent means the shipped weights. */
|
||||
weights?: Record<string, number>;
|
||||
/** Set when `weights` came from the visitor rather than the environment. */
|
||||
edited?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* `score x weight = value`, per component, plus the total.
|
||||
*
|
||||
* The counterweight row is called out because it is the row that makes the
|
||||
* reward an opinion rather than a scoreboard. Everyone understands "pay for
|
||||
* solving it". The interesting engineering is the term that takes points away,
|
||||
* and an exec who leaves this page understanding only that has got the point.
|
||||
*/
|
||||
export function RewardBreakdown({
|
||||
spec,
|
||||
values,
|
||||
metrics,
|
||||
weights,
|
||||
edited = false,
|
||||
className,
|
||||
}: RewardBreakdownProps) {
|
||||
const { rows, total } = scoreReward(spec, values, weights);
|
||||
const counterweights = rows.filter((row) => row.component.role === 'counterweight');
|
||||
|
||||
return (
|
||||
<div className={cn('card overflow-hidden', className)}>
|
||||
<table className="w-full border-collapse text-sm">
|
||||
<caption className="sr-only">
|
||||
Reward components, their weights and their contribution to the total score
|
||||
</caption>
|
||||
<thead>
|
||||
<tr className="border-b border-border text-left text-xs uppercase tracking-wide text-muted">
|
||||
<th scope="col" className="px-3 py-2 font-medium">
|
||||
Component
|
||||
</th>
|
||||
<th scope="col" className="px-2 py-2 text-right font-medium">
|
||||
Score
|
||||
</th>
|
||||
<th scope="col" className="px-2 py-2 text-right font-medium">
|
||||
Weight
|
||||
</th>
|
||||
<th scope="col" className="px-3 py-2 text-right font-medium">
|
||||
Value
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-border">
|
||||
{rows.map((row) => {
|
||||
const role = ROLE_META[row.component.role];
|
||||
const isCounterweight = row.component.role === 'counterweight';
|
||||
return (
|
||||
<tr
|
||||
key={row.component.key}
|
||||
className={cn(isCounterweight && 'bg-accent-subtle/40')}
|
||||
>
|
||||
<th scope="row" className="max-w-0 px-3 py-2.5 text-left font-normal">
|
||||
<span className="flex flex-wrap items-center gap-1.5">
|
||||
<span className="font-medium text-fg">{row.component.label}</span>
|
||||
<span
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1 rounded-md px-1.5 py-0.5 text-[10px] font-semibold uppercase tracking-wide',
|
||||
isCounterweight
|
||||
? 'bg-brand/15 text-accent-fg'
|
||||
: 'bg-surface-2 text-muted',
|
||||
)}
|
||||
>
|
||||
<role.Icon className="h-3 w-3" aria-hidden="true" />
|
||||
{role.label}
|
||||
</span>
|
||||
</span>
|
||||
<span className="mt-0.5 block text-xs leading-snug text-muted">
|
||||
{row.component.description}
|
||||
</span>
|
||||
<span className="mt-0.5 block font-mono text-[11px] text-muted">
|
||||
{row.component.key}
|
||||
</span>
|
||||
</th>
|
||||
<td className="nums px-2 py-2.5 text-right align-top font-mono">
|
||||
{row.score === null ? (
|
||||
<span className="text-muted" title="The environment did not score this run">
|
||||
not scored
|
||||
</span>
|
||||
) : (
|
||||
formatNumber(row.score)
|
||||
)}
|
||||
</td>
|
||||
<td className="nums px-2 py-2.5 text-right align-top font-mono">
|
||||
<span className={cn(edited && 'text-accent-fg')}>
|
||||
{formatNumber(row.weight, 2)}
|
||||
</span>
|
||||
</td>
|
||||
<td className="nums px-3 py-2.5 text-right align-top font-mono font-semibold">
|
||||
{row.value === null ? DASH : formatNumber(row.value)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
<tfoot>
|
||||
<tr className="border-t-2 border-border bg-surface-2">
|
||||
<th scope="row" className="px-3 py-2.5 text-left">
|
||||
<span className="flex items-center gap-2 font-semibold">
|
||||
Total reward
|
||||
{edited ? <EditedChip /> : null}
|
||||
</span>
|
||||
</th>
|
||||
<td colSpan={2} />
|
||||
<td className="nums px-3 py-2.5 text-right font-mono text-base font-semibold">
|
||||
{total === null ? (
|
||||
<span className="text-muted">not scored</span>
|
||||
) : (
|
||||
formatOrDash(total)
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
</tfoot>
|
||||
</table>
|
||||
|
||||
{counterweights.length > 0 ? (
|
||||
<p className="border-t border-border bg-accent-subtle/40 px-3 py-2.5 text-xs leading-relaxed text-fg">
|
||||
<span className="font-semibold">
|
||||
{counterweights.map((row) => row.component.label).join(' and ')}
|
||||
</span>{' '}
|
||||
{counterweights.length > 1 ? 'are counterweights' : 'is the counterweight'}: without a
|
||||
term pulling the other way, the cheapest way to maximise the objective is a behaviour
|
||||
you would never ship, and the model will find it.
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{spec.metrics && spec.metrics.length > 0 ? (
|
||||
<div className="border-t border-border px-3 py-2.5">
|
||||
<h4 className="text-xs font-semibold uppercase tracking-wide text-muted">
|
||||
Diagnostics — reported, never summed
|
||||
</h4>
|
||||
<dl className="mt-1.5 grid gap-x-4 gap-y-1 sm:grid-cols-2">
|
||||
{spec.metrics.map((metric) => (
|
||||
<div key={metric.key} className="flex items-baseline justify-between gap-2">
|
||||
<dt className="text-xs text-muted" title={metric.description}>
|
||||
{metric.label}
|
||||
</dt>
|
||||
<dd className="nums font-mono text-xs">
|
||||
{formatOrDash(metrics?.[metric.key] ?? null)}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import type { ComponentType, KeyboardEvent } from 'react';
|
||||
import type { DemoStep } from '@/lib/demo-kit/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { DASH, formatInt, formatMs, humaniseToken, usePrefersReducedMotion } from './format';
|
||||
|
||||
export interface StepTimelineProps<T> {
|
||||
steps: DemoStep<T>[];
|
||||
current: number;
|
||||
onSelect: (index: number) => void;
|
||||
/** The demo's own board, drawn small. Omit and the chips are text only. */
|
||||
Surface?: ComponentType<{ state: T; compact?: boolean }>;
|
||||
/** Bound to Space, per the transport convention on the rest of the page. */
|
||||
onTogglePlay?: () => void;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The scrubber: one chip per recorded step, each carrying its own board
|
||||
* preview and its own real numbers.
|
||||
*
|
||||
* Why numbers on a chip at all — the chips are the only place the cost of the
|
||||
* run is visible without opening a panel, and "this took 6 calls and 4,200
|
||||
* tokens" is the sentence a buyer repeats to their CFO.
|
||||
*
|
||||
* Keyboard: this is a radiogroup, so arrows move AND select, which is the
|
||||
* standard pattern. Space is bound to play/pause rather than to select — a
|
||||
* deliberate break from the radio pattern, because the timeline sits under a
|
||||
* transport bar where Space means play everywhere else, and being internally
|
||||
* consistent beats being technically canonical. It is advertised on the group
|
||||
* with `aria-keyshortcuts`.
|
||||
*/
|
||||
export function StepTimeline<T>({
|
||||
steps,
|
||||
current,
|
||||
onSelect,
|
||||
Surface,
|
||||
onTogglePlay,
|
||||
className,
|
||||
}: StepTimelineProps<T>) {
|
||||
const reducedMotion = usePrefersReducedMotion();
|
||||
const chipRefs = useRef<(HTMLDivElement | null)[]>([]);
|
||||
const scrollerRef = useRef<HTMLDivElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const chip = chipRefs.current[current];
|
||||
const scroller = scrollerRef.current;
|
||||
if (!chip || !scroller) return;
|
||||
// `scrollIntoView` on the element would also scroll the PAGE to the
|
||||
// timeline on every step, which is intolerable while the run plays. Scroll
|
||||
// the strip only.
|
||||
const chipBox = chip.getBoundingClientRect();
|
||||
const viewBox = scroller.getBoundingClientRect();
|
||||
const delta = chipBox.left - viewBox.left - (viewBox.width - chipBox.width) / 2;
|
||||
scroller.scrollBy({ left: delta, behavior: reducedMotion ? 'auto' : 'smooth' });
|
||||
}, [current, reducedMotion]);
|
||||
|
||||
const move = (next: number, event: KeyboardEvent) => {
|
||||
event.preventDefault();
|
||||
const clamped = Math.min(Math.max(next, 0), steps.length - 1);
|
||||
onSelect(clamped);
|
||||
chipRefs.current[clamped]?.focus();
|
||||
};
|
||||
|
||||
const handleKeyDown = (event: KeyboardEvent<HTMLDivElement>) => {
|
||||
switch (event.key) {
|
||||
case 'ArrowRight':
|
||||
case 'ArrowDown':
|
||||
move(current + 1, event);
|
||||
break;
|
||||
case 'ArrowLeft':
|
||||
case 'ArrowUp':
|
||||
move(current - 1, event);
|
||||
break;
|
||||
case 'Home':
|
||||
move(0, event);
|
||||
break;
|
||||
case 'End':
|
||||
move(steps.length - 1, event);
|
||||
break;
|
||||
case ' ':
|
||||
case 'Spacebar':
|
||||
if (onTogglePlay) {
|
||||
event.preventDefault();
|
||||
onTogglePlay();
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={scrollerRef}
|
||||
role="radiogroup"
|
||||
aria-label="Steps in the recorded run"
|
||||
aria-keyshortcuts="ArrowLeft ArrowRight Home End Space"
|
||||
onKeyDown={handleKeyDown}
|
||||
className={cn('flex snap-x gap-2 overflow-x-auto pb-2', className)}
|
||||
>
|
||||
{steps.map((step, index) => {
|
||||
const selected = index === current;
|
||||
const call = step.call;
|
||||
return (
|
||||
// A div rather than a button: the chip embeds the demo's own Surface,
|
||||
// and a Surface is a grid of divs. Nesting flow content inside a
|
||||
// <button> is invalid HTML and browsers reflow it unpredictably. The
|
||||
// radio role plus the group's key handling gives the same semantics.
|
||||
<div
|
||||
key={step.index}
|
||||
role="radio"
|
||||
aria-checked={selected}
|
||||
// Roving tabindex: one stop for the whole strip, arrows inside it.
|
||||
tabIndex={selected ? 0 : -1}
|
||||
ref={(node) => {
|
||||
chipRefs.current[index] = node;
|
||||
}}
|
||||
onClick={() => onSelect(index)}
|
||||
className={cn(
|
||||
'tap w-[9.5rem] shrink-0 snap-center rounded-lg border p-2 text-left transition-colors duration-2 ease-enter',
|
||||
selected
|
||||
? 'border-brand bg-accent-subtle/60'
|
||||
: 'border-border bg-surface hover:bg-surface-2',
|
||||
)}
|
||||
>
|
||||
<span className="flex items-baseline justify-between gap-1">
|
||||
<span className="nums text-xs font-semibold">Step {index + 1}</span>
|
||||
{call?.finishReason ? (
|
||||
<span
|
||||
className="max-w-[4.5rem] truncate text-[10px] uppercase tracking-wide text-muted"
|
||||
title={call.finishReason}
|
||||
>
|
||||
{humaniseToken(call.finishReason)}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
|
||||
{Surface ? (
|
||||
<span className="mt-1.5 block overflow-hidden rounded-md bg-surface-2 p-1.5">
|
||||
<Surface state={step.state} compact />
|
||||
</span>
|
||||
) : null}
|
||||
|
||||
{step.caption ? (
|
||||
<span className="mt-1.5 block truncate text-xs text-muted" title={step.caption}>
|
||||
{step.caption}
|
||||
</span>
|
||||
) : null}
|
||||
|
||||
<span className="nums mt-1 flex flex-wrap gap-x-2 text-[10px] text-muted">
|
||||
<span title="Completion tokens">
|
||||
{call?.completionTokens === null || call?.completionTokens === undefined
|
||||
? DASH
|
||||
: `${formatInt(call.completionTokens)} tok`}
|
||||
</span>
|
||||
<span title="Recorded latency">{formatMs(call?.durationMs ?? null)}</span>
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { CheckCircle2, ChevronDown, HelpCircle, XCircle } from 'lucide-react';
|
||||
import { verifyEpisode } from '@/lib/demo-kit';
|
||||
import type { DemoEpisode, DemoModule, RewardValues } from '@/lib/demo-kit/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { DASH, formatDelta, formatNumber, formatOrDash } from './format';
|
||||
import { rewardDelta, scoreReward } from './reward-math';
|
||||
|
||||
/**
|
||||
* Float tolerance for "the browser agrees with Python".
|
||||
*
|
||||
* 1e-9 would be theatre: the two runtimes accumulate a sum in a different
|
||||
* order, and IEEE-754 does not promise associativity. 1e-6 is well below any
|
||||
* difference a reward change would produce and well above the noise.
|
||||
*/
|
||||
const EPSILON = 1e-6;
|
||||
|
||||
type Verdict =
|
||||
| { kind: 'match'; recomputed: RewardValues; recordedTotal: number | null; recomputedTotal: number | null; maxDelta: number; rows: VerifyRow[] }
|
||||
| { kind: 'mismatch'; recomputed: RewardValues; recordedTotal: number | null; recomputedTotal: number | null; maxDelta: number; rows: VerifyRow[]; guilty: string[] }
|
||||
| { kind: 'unverifiable'; reason: string };
|
||||
|
||||
interface VerifyRow {
|
||||
key: string;
|
||||
label: string;
|
||||
recorded: number | null;
|
||||
recomputed: number | null;
|
||||
delta: number;
|
||||
}
|
||||
|
||||
export interface VerifyBadgeProps<T> {
|
||||
demo: DemoModule<T>;
|
||||
episode: DemoEpisode;
|
||||
className?: string;
|
||||
/** Open the receipt on load. The sceptic we are writing for opens it anyway. */
|
||||
defaultOpen?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* The receipt.
|
||||
*
|
||||
* This object exists for one person: the engineer sitting next to the CEO who
|
||||
* assumes the numbers on a vendor's demo page are hard-coded. It re-runs every
|
||||
* recorded move through the TypeScript engine in the visitor's own browser,
|
||||
* rescores it, and prints the comparison — including the delta, to seven
|
||||
* decimals, because a comparison without a delta is an assertion.
|
||||
*
|
||||
* It must therefore be allowed to FAIL loudly. A verifier that silently
|
||||
* degrades to "verified" when it cannot check anything is worse than no
|
||||
* verifier: it teaches the sceptic that the badge is decoration.
|
||||
*/
|
||||
export function VerifyBadge<T>({ demo, episode, className, defaultOpen = false }: VerifyBadgeProps<T>) {
|
||||
const [open, setOpen] = useState(defaultOpen);
|
||||
|
||||
const verdict = useMemo<Verdict>(() => {
|
||||
if (!demo.verify) {
|
||||
return {
|
||||
kind: 'unverifiable',
|
||||
reason:
|
||||
'This demo does not ship a browser-side engine, so the recorded scores cannot be re-derived here. The Python that produced them is in the repository and the eval command is below.',
|
||||
};
|
||||
}
|
||||
|
||||
let recomputed: RewardValues | null;
|
||||
try {
|
||||
recomputed = verifyEpisode(demo, episode);
|
||||
} catch (error) {
|
||||
// A verifier that throws is a bug on our side, not a failed run. Say so
|
||||
// rather than showing a red mismatch that blames the recorded numbers.
|
||||
return {
|
||||
kind: 'unverifiable',
|
||||
reason: `The in-browser verifier threw while re-running this episode: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (recomputed === null) {
|
||||
return {
|
||||
kind: 'unverifiable',
|
||||
reason: episode.truncated
|
||||
? 'This run was truncated before the environment reached a terminal state, so there is nothing complete to re-score. The recorded partial numbers are shown as they were captured.'
|
||||
: 'The environment could not re-derive this episode from the recorded transcript. Nothing here is being asserted as verified.',
|
||||
};
|
||||
}
|
||||
|
||||
const deltas = rewardDelta(episode.rewards, recomputed);
|
||||
const labels = new Map(demo.reward.components.map((c) => [c.key, c.label]));
|
||||
const rows: VerifyRow[] = deltas
|
||||
.map(({ key, delta }) => ({
|
||||
key,
|
||||
label: labels.get(key) ?? key,
|
||||
recorded: episode.rewards[key] ?? null,
|
||||
recomputed: recomputed[key] ?? null,
|
||||
delta,
|
||||
}))
|
||||
.sort((a, b) => b.delta - a.delta || a.key.localeCompare(b.key));
|
||||
|
||||
const recordedTotal = scoreReward(demo.reward, episode.rewards).total;
|
||||
const recomputedTotal = scoreReward(demo.reward, recomputed).total;
|
||||
const totalDelta =
|
||||
recordedTotal === null || recomputedTotal === null
|
||||
? recordedTotal === recomputedTotal
|
||||
? 0
|
||||
: Number.POSITIVE_INFINITY
|
||||
: Math.abs(recordedTotal - recomputedTotal);
|
||||
const maxDelta = rows.reduce((worst, row) => Math.max(worst, row.delta), totalDelta);
|
||||
const guilty = rows.filter((row) => row.delta > EPSILON).map((row) => row.label);
|
||||
|
||||
if (guilty.length === 0 && maxDelta <= EPSILON) {
|
||||
return { kind: 'match', recomputed, recordedTotal, recomputedTotal, maxDelta, rows };
|
||||
}
|
||||
return { kind: 'mismatch', recomputed, recordedTotal, recomputedTotal, maxDelta, rows, guilty };
|
||||
}, [demo, episode]);
|
||||
|
||||
if (verdict.kind === 'unverifiable') {
|
||||
return (
|
||||
<section
|
||||
aria-label="Verification"
|
||||
className={cn('card border-border bg-surface-2 p-3', className)}
|
||||
>
|
||||
<p className="flex items-start gap-2 text-sm">
|
||||
<HelpCircle className="mt-0.5 h-4 w-4 shrink-0 text-muted" aria-hidden="true" />
|
||||
<span>
|
||||
<span className="font-semibold">Unverifiable in your browser.</span>{' '}
|
||||
<span className="text-muted">{verdict.reason}</span>
|
||||
</span>
|
||||
</p>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
const matched = verdict.kind === 'match';
|
||||
|
||||
return (
|
||||
<section
|
||||
aria-label="Verification"
|
||||
className={cn(
|
||||
'card overflow-hidden',
|
||||
matched ? 'border-positive/40' : 'border-danger',
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<div className={cn('p-3', matched ? 'bg-positive/10' : 'bg-danger/10')}>
|
||||
<p className="flex items-start gap-2 text-sm leading-relaxed">
|
||||
{matched ? (
|
||||
<CheckCircle2 className="mt-0.5 h-4 w-4 shrink-0 text-positive" aria-hidden="true" />
|
||||
) : (
|
||||
<XCircle className="mt-0.5 h-4 w-4 shrink-0 text-danger" aria-hidden="true" />
|
||||
)}
|
||||
<span>
|
||||
{matched ? (
|
||||
<>
|
||||
<span className="font-semibold text-positive">Verified in your browser</span>{' '}
|
||||
<span className="text-fg">
|
||||
— re-ran every move through the TypeScript engine and rescored.
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<span className="font-semibold text-danger">
|
||||
Mismatch on {verdict.guilty.join(', ')}
|
||||
</span>{' '}
|
||||
<span className="text-fg">
|
||||
— the browser re-run disagrees with the recorded score. Trust the source, not
|
||||
this page.
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
</p>
|
||||
<p className="nums mt-1.5 pl-6 font-mono text-xs text-muted">
|
||||
Recomputed {formatOrDash(verdict.recomputedTotal)} · recorded{' '}
|
||||
{formatOrDash(verdict.recordedTotal)} · Δ {formatDelta(verdict.maxDelta)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen((was) => !was)}
|
||||
aria-expanded={open}
|
||||
className="tap flex w-full items-center gap-1.5 border-t border-border px-3 text-left text-xs font-medium text-muted transition-colors duration-2 ease-enter hover:bg-surface-2 hover:text-fg"
|
||||
>
|
||||
<ChevronDown
|
||||
className={cn('h-4 w-4 transition-transform duration-2 ease-enter', open && 'rotate-180')}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
{open ? 'Hide the component-by-component receipt' : 'Show the component-by-component receipt'}
|
||||
</button>
|
||||
|
||||
{open ? (
|
||||
<div className="overflow-x-auto border-t border-border">
|
||||
<table className="nums w-full border-collapse font-mono text-xs">
|
||||
<thead>
|
||||
<tr className="text-left text-muted">
|
||||
<th scope="col" className="px-3 py-1.5 font-medium">
|
||||
Component
|
||||
</th>
|
||||
<th scope="col" className="px-2 py-1.5 text-right font-medium">
|
||||
Recorded
|
||||
</th>
|
||||
<th scope="col" className="px-2 py-1.5 text-right font-medium">
|
||||
Recomputed
|
||||
</th>
|
||||
<th scope="col" className="px-3 py-1.5 text-right font-medium">
|
||||
Δ
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-border">
|
||||
{verdict.rows.map((row) => {
|
||||
const bad = row.delta > EPSILON;
|
||||
return (
|
||||
<tr key={row.key} className={cn(bad && 'bg-danger/10')}>
|
||||
<th scope="row" className="px-3 py-1.5 text-left font-normal">
|
||||
{row.label}
|
||||
</th>
|
||||
<td className="px-2 py-1.5 text-right">
|
||||
{row.recorded === null ? DASH : formatNumber(row.recorded, 6)}
|
||||
</td>
|
||||
<td className="px-2 py-1.5 text-right">
|
||||
{row.recomputed === null ? DASH : formatNumber(row.recomputed, 6)}
|
||||
</td>
|
||||
<td
|
||||
className={cn('px-3 py-1.5 text-right', bad ? 'text-danger' : 'text-muted')}
|
||||
>
|
||||
{Number.isFinite(row.delta) ? formatDelta(row.delta) : 'not comparable'}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
<p className="px-3 py-2 text-[11px] leading-relaxed text-muted">
|
||||
Tolerance {EPSILON.toExponential()}. The browser engine and the Python environment
|
||||
can sum the same terms in a different order, and IEEE-754 addition is not
|
||||
associative, so the comparison is made within a tolerance rather than demanding
|
||||
bit-identical floats.
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { RewardComponent, RewardSpec, RewardValues } from '@/lib/demo-kit/types';
|
||||
|
||||
export interface ScoredRow {
|
||||
component: RewardComponent;
|
||||
/** The environment's raw per-component score. `null` means NOT SCORED. */
|
||||
score: number | null;
|
||||
/** The weight in force — shipped, or the visitor's edit. */
|
||||
weight: number;
|
||||
/** `score x weight`, or null when the component was not scored. */
|
||||
value: number | null;
|
||||
}
|
||||
|
||||
export interface ScoredReward {
|
||||
rows: ScoredRow[];
|
||||
/**
|
||||
* The weighted sum over components that were actually scored. `null` when
|
||||
* none of them were: a total of 0 would claim the run scored nothing, which
|
||||
* is a different and much stronger statement than "we could not score it".
|
||||
*/
|
||||
total: number | null;
|
||||
}
|
||||
|
||||
/** The weights the environment ships, as a plain map the editor can copy. */
|
||||
export function shippedWeights(spec: RewardSpec): Record<string, number> {
|
||||
const out: Record<string, number> = {};
|
||||
for (const component of spec.components) out[component.key] = component.weight;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function scoreReward(
|
||||
spec: RewardSpec,
|
||||
values: RewardValues,
|
||||
weights?: Record<string, number>,
|
||||
): ScoredReward {
|
||||
let total = 0;
|
||||
let anyScored = false;
|
||||
const rows = spec.components.map((component) => {
|
||||
const raw = values[component.key];
|
||||
const score = raw === undefined ? null : raw;
|
||||
const weight = weights?.[component.key] ?? component.weight;
|
||||
const value = score === null ? null : score * weight;
|
||||
if (value !== null) {
|
||||
total += value;
|
||||
anyScored = true;
|
||||
}
|
||||
return { component, score, weight, value };
|
||||
});
|
||||
return { rows, total: anyScored ? total : null };
|
||||
}
|
||||
|
||||
/** True when two reward maps agree to within float noise on every key. */
|
||||
export function rewardDelta(a: RewardValues, b: RewardValues): { key: string; delta: number }[] {
|
||||
const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
|
||||
const out: { key: string; delta: number }[] = [];
|
||||
for (const key of keys) {
|
||||
const left = a[key];
|
||||
const right = b[key];
|
||||
// Both absent or both explicitly not-scored is agreement, not a zero
|
||||
// delta on a number nobody produced.
|
||||
if ((left === null || left === undefined) && (right === null || right === undefined)) {
|
||||
out.push({ key, delta: 0 });
|
||||
continue;
|
||||
}
|
||||
if (left === null || left === undefined || right === null || right === undefined) {
|
||||
// One side scored and the other did not. That is a real disagreement and
|
||||
// it has no numeric magnitude, so flag it as infinite rather than as 0.
|
||||
out.push({ key, delta: Number.POSITIVE_INFINITY });
|
||||
continue;
|
||||
}
|
||||
out.push({ key, delta: Math.abs(left - right) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
Reference in New Issue
Block a user