Rank agents, not runs; open on a run worth watching

The Reward tab's ranking was thirty-two rows carrying four distinct labels —
"Never wastes a guess 1.000" seven times in a row. The flip is the one thing
that tab exists to show and it was invisible under that scroll. Arms are now
one per agent, each component averaged over that agent's scored runs, with a
null staying null rather than becoming a zero. Four rows. "Found the word at
any cost" moves Best-known play up one place and the shipped weights move it
back, visibly, in the browser.

The default run was runs[0]: the weakest agent on seed 0, a failed game with
thinking off. So Watch opened on a loss with an empty reasoning panel and
Reward opened on a row of zeros — the model's least interesting attempt, chosen
by sort order. It now prefers a run with recorded reasoning, then a solved one,
then the lowest seed so the choice is stable across deploys.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB

src/components/demo/DemoShell.tsx

@@ -7,7 +7,7 @@ import { listRuns, loadEpisode, rewardTotal } from '@/lib/demo-kit/episode';
 import { usePlayer } from '@/lib/demo-kit/player';
 import { loadDemoModule } from '@/lib/demo-kit/registry';
 import type { AnyDemoModule } from '@/lib/demo-kit/registry';
-import type { DemoEpisode, DemoStep, DemoTabId, RunRef } from '@/lib/demo-kit/types';
+import type { DemoEpisode, DemoStep, DemoTabId, RewardValues, RunRef } from '@/lib/demo-kit/types';
 import { useRunParam, useSpeedParam, useStepParam, useTabParam } from '@/lib/url-state';
 import * as st from '@/content/styles';
 import { cn } from '@/lib/utils';
@@ -172,7 +172,7 @@ function DemoBody({ bundle }: { bundle: DemoBundle }) {
   const [speedParam, setSpeedParam] = useSpeedParam();
 
   const run = useMemo(
-    () => runs.find((candidate) => candidate.id === runParam) ?? runs[0],
+    () => runs.find((candidate) => candidate.id === runParam) ?? defaultRun(runs, episodes),
     [runs, runParam],
   );
   const episode = run ? episodes[run.id] : undefined;
@@ -215,20 +215,35 @@ function DemoBody({ bundle }: { bundle: DemoBundle }) {
   // React-only, not a URL param. See DEFAULT_DETAIL_PANEL.
   const [detailPanel, setDetailPanel] = useState(DEFAULT_DETAIL_PANEL);
 
-  const arms = useMemo<RewardArm[]>(
+  // One arm per AGENT, not per run. Four agents over eight seeds is thirty-two
-    () =>
+  // runs, and a ranking of thirty-two rows carrying four distinct labels buries
-      runs.map((candidate) => {
+  // the one thing the Reward tab exists to show: move a slider, the order
-        const armEpisode = episodes[candidate.id];
+  // flips. Each component is averaged over the agent's scored runs; a component
-        const arm: RewardArm = {
+  // no run scored stays null rather than becoming a zero, because a zero is a
-          id: candidate.id,
+  // claim about the agent and a null is an admission we do not know.
-          label: candidate.label,
+  const arms = useMemo<RewardArm[]>(() => {
-          values: armEpisode?.rewards ?? {},
+    const byLabel = new Map<string, { runs: RunRef[]; note?: string }>();
-        };
+    for (const candidate of runs) {
-        if (candidate.intervention) arm.note = candidate.intervention;
+      const group = byLabel.get(candidate.label) ?? { runs: [] };
+      group.runs.push(candidate);
+      if (candidate.intervention && !group.note) group.note = candidate.intervention;
+      byLabel.set(candidate.label, group);
+    }
+    return [...byLabel.entries()].map(([label, group]) => {
+      const values: RewardValues = {};
+      for (const component of demo.reward.components) {
+        const scored = group.runs
+          .map((r) => episodes[r.id]?.rewards[component.key])
+          .filter((v): v is number => typeof v === 'number' && Number.isFinite(v));
+        values[component.key] =
+          scored.length === 0 ? null : scored.reduce((a, b) => a + b, 0) / scored.length;
+      }
+      const arm: RewardArm = { id: label, label, values };
+      if (group.note) arm.note = `${group.note} Mean over ${group.runs.length} recorded runs.`;
+      else arm.note = `Mean over ${group.runs.length} recorded runs.`;
       return arm;
-      }),
+    });
-    [runs, episodes],
+  }, [runs, episodes, demo.reward.components]);
-  );
 
   const blindPair = useMemo(() => {
     for (let i = 0; i < runs.length; i += 1) {
@@ -723,3 +738,28 @@ function ShellSkeleton() {
     </div>
   );
 }
+
+
+/**
+ * The run a visitor sees before choosing one.
+ *
+ * `runs[0]` is the manifest's first entry — the weakest agent on seed 0, which
+ * for wordle is a failed game with thinking off. So the Watch tab opened on a
+ * loss with an empty reasoning panel and the Reward tab on a row of zeros:
+ * the model's least interesting attempt, chosen by accident of sort order.
+ *
+ * Prefer, in order: a run with recorded reasoning (there is something to
+ * stream), then a solved one (the board reaches a conclusion), then the
+ * earliest seed so the choice is stable across deploys. The visitor can still
+ * pick any run; this only decides what the page leads with.
+ */
+function defaultRun(runs: RunRef[], episodes: Record<string, DemoEpisode>): RunRef | undefined {
+  const score = (run: RunRef): number => {
+    const ep = episodes[run.id];
+    if (!ep) return -1;
+    const hasReasoning = ep.turns.some((t) => t.reasoning && t.reasoning.length > 0);
+    const solved = ep.outcome === 'solved';
+    return (hasReasoning ? 2 : 0) + (solved ? 1 : 0);
+  };
+  return [...runs].sort((a, b) => score(b) - score(a) || a.seed - b.seed)[0];
+}