diff --git a/src/components/demo/DemoShell.tsx b/src/components/demo/DemoShell.tsx index af46c80..b1097b5 100644 --- a/src/components/demo/DemoShell.tsx +++ b/src/components/demo/DemoShell.tsx @@ -7,7 +7,7 @@ import { listRuns, loadEpisode, rewardTotal } from '@/lib/demo-kit/episode'; import { usePlayer } from '@/lib/demo-kit/player'; import { loadDemoModule } from '@/lib/demo-kit/registry'; import type { AnyDemoModule } from '@/lib/demo-kit/registry'; -import type { DemoEpisode, DemoStep, DemoTabId, RunRef } from '@/lib/demo-kit/types'; +import type { DemoEpisode, DemoStep, DemoTabId, RewardValues, RunRef } from '@/lib/demo-kit/types'; import { useRunParam, useSpeedParam, useStepParam, useTabParam } from '@/lib/url-state'; import * as st from '@/content/styles'; import { cn } from '@/lib/utils'; @@ -172,7 +172,7 @@ function DemoBody({ bundle }: { bundle: DemoBundle }) { const [speedParam, setSpeedParam] = useSpeedParam(); const run = useMemo( - () => runs.find((candidate) => candidate.id === runParam) ?? runs[0], + () => runs.find((candidate) => candidate.id === runParam) ?? defaultRun(runs, episodes), [runs, runParam], ); const episode = run ? episodes[run.id] : undefined; @@ -215,20 +215,35 @@ function DemoBody({ bundle }: { bundle: DemoBundle }) { // React-only, not a URL param. See DEFAULT_DETAIL_PANEL. const [detailPanel, setDetailPanel] = useState(DEFAULT_DETAIL_PANEL); - const arms = useMemo( - () => - runs.map((candidate) => { - const armEpisode = episodes[candidate.id]; - const arm: RewardArm = { - id: candidate.id, - label: candidate.label, - values: armEpisode?.rewards ?? {}, - }; - if (candidate.intervention) arm.note = candidate.intervention; - return arm; - }), - [runs, episodes], - ); + // One arm per AGENT, not per run. Four agents over eight seeds is thirty-two + // runs, and a ranking of thirty-two rows carrying four distinct labels buries + // the one thing the Reward tab exists to show: move a slider, the order + // flips. Each component is averaged over the agent's scored runs; a component + // no run scored stays null rather than becoming a zero, because a zero is a + // claim about the agent and a null is an admission we do not know. + const arms = useMemo(() => { + const byLabel = new Map(); + for (const candidate of runs) { + const group = byLabel.get(candidate.label) ?? { runs: [] }; + group.runs.push(candidate); + if (candidate.intervention && !group.note) group.note = candidate.intervention; + byLabel.set(candidate.label, group); + } + return [...byLabel.entries()].map(([label, group]) => { + const values: RewardValues = {}; + for (const component of demo.reward.components) { + const scored = group.runs + .map((r) => episodes[r.id]?.rewards[component.key]) + .filter((v): v is number => typeof v === 'number' && Number.isFinite(v)); + values[component.key] = + scored.length === 0 ? null : scored.reduce((a, b) => a + b, 0) / scored.length; + } + const arm: RewardArm = { id: label, label, values }; + if (group.note) arm.note = `${group.note} Mean over ${group.runs.length} recorded runs.`; + else arm.note = `Mean over ${group.runs.length} recorded runs.`; + return arm; + }); + }, [runs, episodes, demo.reward.components]); const blindPair = useMemo(() => { for (let i = 0; i < runs.length; i += 1) { @@ -723,3 +738,28 @@ function ShellSkeleton() { ); } + + +/** + * The run a visitor sees before choosing one. + * + * `runs[0]` is the manifest's first entry — the weakest agent on seed 0, which + * for wordle is a failed game with thinking off. So the Watch tab opened on a + * loss with an empty reasoning panel and the Reward tab on a row of zeros: + * the model's least interesting attempt, chosen by accident of sort order. + * + * Prefer, in order: a run with recorded reasoning (there is something to + * stream), then a solved one (the board reaches a conclusion), then the + * earliest seed so the choice is stable across deploys. The visitor can still + * pick any run; this only decides what the page leads with. + */ +function defaultRun(runs: RunRef[], episodes: Record): RunRef | undefined { + const score = (run: RunRef): number => { + const ep = episodes[run.id]; + if (!ep) return -1; + const hasReasoning = ep.turns.some((t) => t.reasoning && t.reasoning.length > 0); + const solved = ep.outcome === 'solved'; + return (hasReasoning ? 2 : 0) + (solved ? 1 : 0); + }; + return [...runs].sort((a, b) => score(b) - score(a) || a.seed - b.seed)[0]; +}