a21596b3e4
Eighteen agents: three design directions judged on whether a COO on a phone actually learns what an environment is, on craft, and on landing without a rewrite; one spec; a foundation of measured tokens; six build lanes; three browser verifiers; a final gate pass. The materials are Prime Intellect's, measured from their site: near-black grounds, one green, sharp radii, mono small-caps labels, Geist and Geist Mono self-hosted because production CSP is font-src 'self'. Two of their own greys fail contrast on their own ground (#737373 is 4.02:1, #6E6E6E is 3.73:1 on #0F0F0F), so --muted is lifted and the CSS comment carries the number — or someone will 'correct' it back. Every text-on-ground pair in both themes is tabulated in src/index.css with its measured ratio. The two rules that resolved every conflict: data is mono, sentences are sans; the language wins on materials, the lesson wins on legibility. Light mode is a finished paper theme, not an inversion. What did not change: the derived-tabs contract, the honesty markers, the isolation lint, every gate. 419 contract checks, entry chunk at 74% of budget, zero horizontal overflow on any route at 390/1024/1440 in either theme. Also flips Alert Triage to status 'live' — the pipeline built it but never promoted it, so it was badged SPEC on its own playable page and the home page counted one environment. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
873 lines
34 KiB
TypeScript
873 lines
34 KiB
TypeScript
import { useEffect, useMemo, useState } from 'react';
|
|
import type { ComponentProps, ComponentType, ReactNode } from 'react';
|
|
import { useParams } from 'react-router-dom';
|
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
|
import { Skeleton } from '@/components/ui/skeleton';
|
|
import { TermScope } from '@/components/site/Term';
|
|
import { listRuns, loadEpisode, rewardTotal } from '@/lib/demo-kit/episode';
|
|
import { usePlayer } from '@/lib/demo-kit/player';
|
|
import { loadDemoModule } from '@/lib/demo-kit/registry';
|
|
import type { AnyDemoModule } from '@/lib/demo-kit/registry';
|
|
import type { DemoEpisode, DemoStep, DemoTabId, RewardValues, RunRef } from '@/lib/demo-kit/types';
|
|
import { useRunParam, useSpeedParam, useStepParam, useTabParam } from '@/lib/url-state';
|
|
import * as st from '@/content/styles';
|
|
import { cn } from '@/lib/utils';
|
|
import { BlindCompare } from './BlindCompare';
|
|
import { Chip, Label, StatusPill } from './chrome';
|
|
import { CodeReceipt } from './CodeReceipt';
|
|
import { DemoErrorBoundary } from './DemoErrorBoundary';
|
|
import { DemoTabBar, TabClaim, resolveTab, visibleTabs } from './DemoTabs';
|
|
import { EnvAnatomy } from './EnvAnatomy';
|
|
import { LimitsCallout } from './LimitsCallout';
|
|
import { MetricMover } from './MetricMover';
|
|
import { ModelCallPanel } from './ModelCallPanel';
|
|
import { PlayYourself } from './PlayYourself';
|
|
import { ProvenanceCard } from './ProvenanceCard';
|
|
import { ReasoningDrawer } from './ReasoningDrawer';
|
|
import { ReasoningPanel } from './ReasoningPanel';
|
|
import { RewardBreakdown } from './RewardBreakdown';
|
|
import { RewardEditor } from './RewardEditor';
|
|
import type { RewardArm } from './RewardEditor';
|
|
import { SegmentedControl } from './SegmentedControl';
|
|
import { SlotRegion } from './SlotRegion';
|
|
import { StepTimeline } from './StepTimeline';
|
|
import { StatStrip } from './StatStrip';
|
|
import type { Stat } from './StatStrip';
|
|
import { RecordedBadge, TracePlayer } from './TracePlayer';
|
|
import { VerifyBadge } from './VerifyBadge';
|
|
import { formatDate, formatOrDash, useIsDesktop } from './format';
|
|
|
|
const REPO_BLOB = 'https://git.karti.ai/PIG/PIG-Demo/src/branch/main/';
|
|
|
|
/**
|
|
* The panel the step-detail strip inside the Watch tab opens on.
|
|
*
|
|
* This control is deliberately NOT in the URL. `?tab=` belongs to the page's
|
|
* four top-level tabs, and one param cannot address two nested controls without
|
|
* one of them silently winning; a permalink to `?tab=call` would land the reader
|
|
* on a page with no such top-level tab.
|
|
*/
|
|
const DEFAULT_DETAIL_PANEL = 'reasoning';
|
|
|
|
/** Reserved slug for the shell's own hand-written demo. Dev builds only. */
|
|
const MOCK_SLUG = '__mock';
|
|
|
|
/**
|
|
* A top-level tab panel. Each one is its own `TermScope`, so "first occurrence
|
|
* per panel gets the underline" is literally per panel rather than per page.
|
|
*/
|
|
function Panel({ children, ...props }: ComponentProps<typeof TabsContent>) {
|
|
return (
|
|
<TabsContent {...props}>
|
|
<TermScope>{children}</TermScope>
|
|
</TabsContent>
|
|
);
|
|
}
|
|
|
|
export interface DemoBundle {
|
|
demo: AnyDemoModule;
|
|
runs: RunRef[];
|
|
episodes: Record<string, DemoEpisode>;
|
|
}
|
|
|
|
type LoadState =
|
|
| { status: 'loading' }
|
|
| { status: 'ready'; bundle: DemoBundle; dropped: number }
|
|
| { status: 'error'; message: string };
|
|
|
|
/**
|
|
* A demo's module plus every recorded run it has.
|
|
*
|
|
* `loadDemoModule` and `loadEpisode` both cache their promises, so the route
|
|
* loader having already fetched the module makes this resolve without a second
|
|
* request. Runs are loaded with `allSettled` on purpose: one unreadable trace
|
|
* drops that arm rather than blanking the page — and the page SAYS it dropped
|
|
* one, because a missing arm that goes unmentioned is a missing arm nobody can
|
|
* ask about.
|
|
*/
|
|
async function loadBundle(slug: string): Promise<{ bundle: DemoBundle; dropped: number }> {
|
|
if (slug === MOCK_SLUG) {
|
|
// Dynamic, so the mock lands in its own chunk and production never fetches
|
|
// it. A static import would ship several hundred lines of fake trace to
|
|
// every visitor of every real demo.
|
|
const mock = await import('./mock');
|
|
return {
|
|
bundle: { demo: mock.mockDemo, runs: mock.mockRuns, episodes: mock.mockEpisodes },
|
|
dropped: 0,
|
|
};
|
|
}
|
|
|
|
const demo = await loadDemoModule(slug);
|
|
const runs = await listRuns(slug).catch(() => [] as RunRef[]);
|
|
const settled = await Promise.allSettled(runs.map((run) => loadEpisode(run)));
|
|
|
|
const episodes: Record<string, DemoEpisode> = {};
|
|
let dropped = 0;
|
|
settled.forEach((outcome, index) => {
|
|
const run = runs[index];
|
|
if (!run) return;
|
|
if (outcome.status === 'fulfilled') episodes[run.id] = outcome.value;
|
|
else {
|
|
dropped += 1;
|
|
console.error(`[pig-demo] dropped run "${run.id}":`, outcome.reason);
|
|
}
|
|
});
|
|
|
|
return {
|
|
bundle: { demo, runs: runs.filter((run) => episodes[run.id] !== undefined), episodes },
|
|
dropped,
|
|
};
|
|
}
|
|
|
|
export interface DemoShellProps {
|
|
/** Overrides the route param. Useful for previews and tests. */
|
|
slug?: string;
|
|
/** Skips loading entirely when the caller already has the bundle. */
|
|
bundle?: DemoBundle;
|
|
}
|
|
|
|
/**
|
|
* The route component every demo is rendered through.
|
|
*
|
|
* It owns four things and no more: loading, which tabs exist, the URL state,
|
|
* and the page's single polite live region. Everything visual is delegated to
|
|
* the surfaces in this directory, and the demo module is never reached into —
|
|
* the shell only ever calls `adapt` and renders `Surface`.
|
|
*/
|
|
export function DemoShell({ slug: slugProp, bundle }: DemoShellProps) {
|
|
const params = useParams();
|
|
const slug = slugProp ?? params['slug'] ?? '';
|
|
const [state, setState] = useState<LoadState>(
|
|
bundle ? { status: 'ready', bundle, dropped: 0 } : { status: 'loading' },
|
|
);
|
|
|
|
useEffect(() => {
|
|
if (bundle) {
|
|
setState({ status: 'ready', bundle, dropped: 0 });
|
|
return;
|
|
}
|
|
let live = true;
|
|
setState({ status: 'loading' });
|
|
loadBundle(slug)
|
|
.then((loaded) => {
|
|
if (live) setState({ status: 'ready', bundle: loaded.bundle, dropped: loaded.dropped });
|
|
})
|
|
.catch((error: unknown) => {
|
|
if (!live) return;
|
|
setState({
|
|
status: 'error',
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
});
|
|
return () => {
|
|
live = false;
|
|
};
|
|
}, [slug, bundle]);
|
|
|
|
if (state.status === 'loading') return <ShellSkeleton />;
|
|
if (state.status === 'error') {
|
|
return (
|
|
<main className={cn(st.shell, 'py-16')}>
|
|
<div role="alert" className="max-w-xl rounded-xl border border-border bg-surface p-5 sm:p-6">
|
|
<h1 className="text-h2 text-fg">That environment is not here</h1>
|
|
<p className="mt-3 max-w-measure text-prose text-fg-2">{state.message}</p>
|
|
<a href="/gallery" className={cn(st.btnSecondary, 'mt-6')}>
|
|
All environments
|
|
</a>
|
|
</div>
|
|
</main>
|
|
);
|
|
}
|
|
|
|
return (
|
|
// A second boundary inside the route's own: this one is keyed to the demo
|
|
// so a crash names it, and resetting re-renders the surfaces rather than
|
|
// re-navigating.
|
|
<DemoErrorBoundary demoTitle={state.bundle.demo.meta.title}>
|
|
<DemoBody bundle={state.bundle} dropped={state.dropped} />
|
|
</DemoErrorBoundary>
|
|
);
|
|
}
|
|
|
|
function DemoBody({ bundle, dropped }: { bundle: DemoBundle; dropped: number }) {
|
|
const { demo, runs, episodes } = bundle;
|
|
const isDesktop = useIsDesktop();
|
|
|
|
const [runParam, setRunParam] = useRunParam();
|
|
const [stepParam, setStepParam] = useStepParam();
|
|
const [speedParam, setSpeedParam] = useSpeedParam();
|
|
|
|
const run = useMemo(
|
|
() => runs.find((candidate) => candidate.id === runParam) ?? defaultRun(runs, episodes),
|
|
[runs, runParam],
|
|
);
|
|
const episode = run ? episodes[run.id] : undefined;
|
|
|
|
const steps = useMemo<DemoStep<unknown>[]>(
|
|
() => (episode ? demo.adapt(episode) : []),
|
|
[demo, episode],
|
|
);
|
|
|
|
const player = usePlayer(steps, {
|
|
initialIndex: stepParam,
|
|
initialSpeed: speedParam,
|
|
onIndexChange: setStepParam,
|
|
});
|
|
|
|
// The URL is the other writer of this state — Back, a pasted permalink, the
|
|
// run switcher. The player is the source of truth while it is running, so it
|
|
// only follows the URL when the two have actually diverged.
|
|
const { seek } = player;
|
|
useEffect(() => {
|
|
if (stepParam !== player.index) seek(stepParam);
|
|
// Intentionally keyed on the URL only: including `player.index` here would
|
|
// re-run the effect on the player's own advance and fight it.
|
|
}, [stepParam, seek]);
|
|
|
|
// Derived, never declared. A demo that ships no interactive mode has no Play
|
|
// tab and opens on Watch; one whose traces failed to load has no Watch tab
|
|
// and opens on Reward.
|
|
const hasRecording = Boolean(run && episode && steps.length > 0);
|
|
const tabs = useMemo(
|
|
() => visibleTabs({ play: Boolean(demo.interactive), watch: hasRecording }),
|
|
[demo.interactive, hasRecording],
|
|
);
|
|
// `visibleTabs` always keeps `reward` and `evidence`, so index 0 exists; the
|
|
// fallback is here only so the type does not need an assertion.
|
|
const defaultTab: DemoTabId = tabs[0] ?? 'evidence';
|
|
const [tabParam, setTabParam] = useTabParam(defaultTab);
|
|
const activeTab = resolveTab(tabParam, tabs, defaultTab);
|
|
|
|
// React-only, not a URL param. See DEFAULT_DETAIL_PANEL.
|
|
const [detailPanel, setDetailPanel] = useState(DEFAULT_DETAIL_PANEL);
|
|
|
|
// One arm per AGENT, not per run. Four agents over eight seeds is thirty-two
|
|
// runs, and a ranking of thirty-two rows carrying four distinct labels buries
|
|
// the one thing the Reward tab exists to show: move a slider, the order
|
|
// flips. Each component is averaged over the agent's scored runs; a component
|
|
// no run scored stays null rather than becoming a zero, because a zero is a
|
|
// claim about the agent and a null is an admission we do not know.
|
|
const arms = useMemo<RewardArm[]>(() => {
|
|
const byLabel = new Map<string, { runs: RunRef[]; note?: string }>();
|
|
for (const candidate of runs) {
|
|
const group = byLabel.get(candidate.label) ?? { runs: [] };
|
|
group.runs.push(candidate);
|
|
if (candidate.intervention && !group.note) group.note = candidate.intervention;
|
|
byLabel.set(candidate.label, group);
|
|
}
|
|
return [...byLabel.entries()].map(([label, group]) => {
|
|
const values: RewardValues = {};
|
|
for (const component of demo.reward.components) {
|
|
const scored = group.runs
|
|
.map((r) => episodes[r.id]?.rewards[component.key])
|
|
.filter((v): v is number => typeof v === 'number' && Number.isFinite(v));
|
|
values[component.key] =
|
|
scored.length === 0 ? null : scored.reduce((a, b) => a + b, 0) / scored.length;
|
|
}
|
|
const arm: RewardArm = { id: label, label, values };
|
|
if (group.note) arm.note = `${group.note} Mean over ${group.runs.length} recorded runs.`;
|
|
else arm.note = `Mean over ${group.runs.length} recorded runs.`;
|
|
return arm;
|
|
});
|
|
}, [runs, episodes, demo.reward.components]);
|
|
|
|
const blindPair = useMemo(() => {
|
|
for (let i = 0; i < runs.length; i += 1) {
|
|
for (let j = i + 1; j < runs.length; j += 1) {
|
|
const left = runs[i];
|
|
const right = runs[j];
|
|
if (!left || !right || left.seed !== right.seed) continue;
|
|
const leftEpisode = episodes[left.id];
|
|
const rightEpisode = episodes[right.id];
|
|
if (!leftEpisode || !rightEpisode) continue;
|
|
return { left, right, leftEpisode, rightEpisode };
|
|
}
|
|
}
|
|
// Two runs on different seeds are two different puzzles; showing them side
|
|
// by side would be a comparison of luck.
|
|
return null;
|
|
}, [runs, episodes]);
|
|
|
|
const Surface = demo.Surface as ComponentType<{ state: unknown; compact?: boolean }>;
|
|
const current = steps[player.index];
|
|
const claims = demo.narrative.claims;
|
|
const extras = demo.tabs ?? [];
|
|
|
|
// The seed is the shared coordinate between the visitor's board and the
|
|
// agent's. With no recording to match, the demo's own first board will do.
|
|
const playSeed = run?.seed ?? 0;
|
|
|
|
// The newest capture, for the Evidence readout. `runs` is manifest order,
|
|
// which is not date order.
|
|
const newestCapture = useMemo(
|
|
() =>
|
|
runs.reduce<string | null>(
|
|
(newest, candidate) =>
|
|
newest === null || candidate.capturedAt > newest ? candidate.capturedAt : newest,
|
|
null,
|
|
),
|
|
[runs],
|
|
);
|
|
|
|
const headerStats: Stat[] = episode
|
|
? [
|
|
{
|
|
label: 'Outcome',
|
|
value: episode.outcome,
|
|
tone: episode.outcome === 'solved' ? 'positive' : 'warning',
|
|
title: episode.truncated
|
|
? 'Truncated before a terminal state'
|
|
: 'How the recorded run ended',
|
|
},
|
|
{
|
|
label: 'Reward',
|
|
value: formatOrDash(rewardTotal(episode.rewards, demo.reward.components)),
|
|
tone: 'brand',
|
|
title: 'Reward — total under the shipped weights',
|
|
},
|
|
{ label: 'Steps', value: steps.length, title: 'Steps — model calls in this run' },
|
|
{
|
|
label: 'Seed',
|
|
value: episode.seed,
|
|
title: 'Seed — the same seed reproduces this board',
|
|
},
|
|
]
|
|
: [];
|
|
|
|
const detailPanels: { id: string; label: string; content: ReactNode }[] = [
|
|
{
|
|
id: 'reasoning',
|
|
label: 'Reasoning',
|
|
content: isDesktop ? (
|
|
<ReasoningPanel
|
|
reasoning={current?.reasoning ?? null}
|
|
durationMs={current?.call?.durationMs ?? null}
|
|
playing={player.isPlaying}
|
|
speed={player.speed}
|
|
stepIndex={player.index}
|
|
/>
|
|
) : (
|
|
// Under `lg` there is no column for this, and putting it below the
|
|
// board means watching the run with the thinking off-screen. The sheet
|
|
// is mounted only here, so vaul never locks body scroll on desktop.
|
|
<ReasoningDrawer
|
|
reasoning={current?.reasoning ?? null}
|
|
durationMs={current?.call?.durationMs ?? null}
|
|
playing={player.isPlaying}
|
|
speed={player.speed}
|
|
stepIndex={player.index}
|
|
/>
|
|
),
|
|
},
|
|
{
|
|
id: 'call',
|
|
label: 'Model call',
|
|
content: <ModelCallPanel call={current?.call ?? null} reply={current?.reply ?? null} />,
|
|
},
|
|
// A demo's own extra panels ride alongside the step detail, where they sit
|
|
// next to the step they are almost always about. With no recording there is
|
|
// no step detail, so the Evidence tab picks them up instead.
|
|
...(hasRecording
|
|
? extras.map((tab) => ({ id: tab.id, label: tab.label, content: <tab.Component /> }))
|
|
: []),
|
|
];
|
|
const activeDetail = detailPanels.some((panel) => panel.id === detailPanel)
|
|
? detailPanel
|
|
: DEFAULT_DETAIL_PANEL;
|
|
|
|
return (
|
|
<main className={cn(st.shell, 'pb-16')}>
|
|
{/*
|
|
The page's ONE live region. Every step change lands here and nowhere
|
|
else: with reduced motion the board animation is gone, so this sentence
|
|
is the only thing that tells a screen-reader user what just happened.
|
|
*/}
|
|
<div aria-live="polite" aria-atomic="true" className="sr-only">
|
|
{current?.announce ?? ''}
|
|
</div>
|
|
|
|
<header className="pt-6">
|
|
<div className="flex flex-wrap items-center gap-2">
|
|
<Label className="text-accent-fg">For {demo.meta.persona}</Label>
|
|
<StatusPill status={demo.meta.status} />
|
|
</div>
|
|
<h1 className="mt-1.5 text-h1 text-fg lg:text-h1-lg">{demo.meta.title}</h1>
|
|
<p className="mt-2 max-w-measure-wide text-pretty text-lede text-fg-2 lg:text-lede-lg">
|
|
{demo.meta.tagline}
|
|
</p>
|
|
{/* The buyer's question, kept quiet on purpose: it is the thing they
|
|
walked in with, not the thing this page is asserting. */}
|
|
<div className="mt-4 border-l-2 border-border-strong pl-3">
|
|
<Label>The question you walked in with</Label>
|
|
<p className="mt-0.5 max-w-[60ch] text-pretty text-prose text-fg-2">
|
|
{demo.narrative.anxiety}
|
|
</p>
|
|
</div>
|
|
{/*
|
|
The stats describe the RECORDED RUN, so they only belong on the tabs
|
|
whose subject is that run. On Play they sat above the visitor's own
|
|
empty board reading "Outcome: failed", which parses as *your* game
|
|
having already failed before you have touched a key.
|
|
*/}
|
|
{run && headerStats.length > 0 && activeTab !== 'play' ? (
|
|
<div className="mt-4 flex flex-wrap items-center gap-x-4 gap-y-2">
|
|
<StatStrip stats={headerStats} />
|
|
<RecordedBadge
|
|
model={run.model}
|
|
capturedAt={run.capturedAt}
|
|
{...(run.intervention ? { intervention: run.intervention } : {})}
|
|
{...(player.timingIsReal || activeTab !== 'watch'
|
|
? {}
|
|
: { timingNote: 'pacing approximate' })}
|
|
/>
|
|
</div>
|
|
) : null}
|
|
<SlotRegion id="hero-aside" />
|
|
</header>
|
|
|
|
<Tabs value={activeTab} onValueChange={setTabParam} className="mt-5 sm:mt-6">
|
|
<DemoTabBar tabs={tabs} />
|
|
|
|
{tabs.includes('play') ? (
|
|
// `forceMount` keeps the visitor's half-finished board alive while
|
|
// they read the other tabs, so a game in progress survives a trip to
|
|
// Reward and back. Radix leaves the hiding to the author under
|
|
// `forceMount`, which is what the `data-[state=inactive]` class does —
|
|
// it is load-bearing, not belt-and-braces.
|
|
<Panel
|
|
value="play"
|
|
forceMount
|
|
className="mt-5 space-y-6 data-[state=inactive]:hidden sm:mt-6"
|
|
>
|
|
<TabClaim>{claims.play}</TabClaim>
|
|
<PlayYourself
|
|
demo={demo}
|
|
seed={playSeed}
|
|
// The bridge to Watch, rendered only once the visitor has acted
|
|
// and only when there is a Watch tab to bridge to. No slug, no
|
|
// narrative literal: the same sentence on every environment.
|
|
{...(tabs.includes('watch')
|
|
? {
|
|
afterMove: (
|
|
<p className="max-w-measure text-prose text-fg-2">
|
|
The score on this board comes from the grader, not from us.{' '}
|
|
<button
|
|
type="button"
|
|
onClick={() => setTabParam('watch')}
|
|
className="font-medium text-accent-fg underline decoration-border-strong underline-offset-4 hover:decoration-accent-fg"
|
|
>
|
|
Watch a model take the same board →
|
|
</button>
|
|
</p>
|
|
),
|
|
}
|
|
: {})}
|
|
/>
|
|
<SlotRegion id="below-board" />
|
|
<div className="space-y-3">
|
|
<Label as="h2">The machine you are inside</Label>
|
|
<EnvAnatomy anatomy={demo.anatomy} rewardLine={demo.meta.rewardLine} compact />
|
|
</div>
|
|
</Panel>
|
|
) : null}
|
|
|
|
{tabs.includes('watch') && run && episode ? (
|
|
<Panel value="watch" className="mt-5 space-y-4 sm:mt-6">
|
|
<TabClaim>{claims.watch}</TabClaim>
|
|
|
|
{runs.length > 1 ? (
|
|
<RunSwitcher
|
|
runs={runs}
|
|
activeId={run.id}
|
|
onSelect={(id) => {
|
|
player.pause();
|
|
// The run param setter also zeroes `step`: step 6 of a
|
|
// nine-turn rollout is not step 6 of a three-turn one.
|
|
setRunParam(id);
|
|
}}
|
|
/>
|
|
) : null}
|
|
|
|
<TracePlayer
|
|
playing={player.isPlaying}
|
|
onPlayingChange={(next) => (next ? player.play() : player.pause())}
|
|
speed={player.speed}
|
|
onSpeedChange={(next) => {
|
|
player.setSpeed(next);
|
|
setSpeedParam(next);
|
|
// `instant` is a destination, not a rate. The player only
|
|
// consumes it while running, so choosing it from a paused
|
|
// transport has to start the run — otherwise the button
|
|
// visibly does nothing, which reads as broken.
|
|
if (next === 'instant') player.play();
|
|
}}
|
|
onRestart={player.restart}
|
|
step={player.index}
|
|
stepCount={steps.length}
|
|
onStepChange={(next) => {
|
|
player.pause();
|
|
player.seek(next);
|
|
}}
|
|
progress={player.progress}
|
|
timingIsReal={player.timingIsReal}
|
|
model={run.model}
|
|
capturedAt={run.capturedAt}
|
|
{...(run.intervention ? { intervention: run.intervention } : {})}
|
|
/>
|
|
|
|
<div className="grid grid-cols-1 gap-3 lg:grid-cols-[auto_minmax(0,1fr)] lg:items-start">
|
|
<div className="w-fit max-w-full rounded-xl border border-border bg-surface p-3 sm:p-4">
|
|
{current ? <Surface state={current.state} /> : null}
|
|
</div>
|
|
|
|
<div className="min-w-0">
|
|
<Tabs value={activeDetail} onValueChange={setDetailPanel}>
|
|
{/* The nested list keeps the PILL segment style, so the
|
|
primary underline bar and this never look alike. */}
|
|
<TabsList aria-label="Details for this step" className="max-w-full overflow-x-auto">
|
|
{detailPanels.map((panel) => (
|
|
<TabsTrigger key={panel.id} value={panel.id}>
|
|
{panel.label}
|
|
</TabsTrigger>
|
|
))}
|
|
</TabsList>
|
|
{detailPanels.map((panel) => (
|
|
<TabsContent key={panel.id} value={panel.id} className="mt-3">
|
|
{panel.content}
|
|
</TabsContent>
|
|
))}
|
|
</Tabs>
|
|
</div>
|
|
</div>
|
|
|
|
<StepTimeline
|
|
steps={steps}
|
|
current={player.index}
|
|
onSelect={(next) => {
|
|
player.pause();
|
|
player.seek(next);
|
|
}}
|
|
Surface={Surface}
|
|
onTogglePlay={player.toggle}
|
|
/>
|
|
{player.timingIsReal ? null : (
|
|
<p className="max-w-measure text-small text-muted">
|
|
Some steps in this run carried no recorded latency, so their dwell on the timeline
|
|
is the player's fallback rather than a measurement.
|
|
</p>
|
|
)}
|
|
{dropped > 0 ? (
|
|
<p className="font-mono text-caption text-muted">
|
|
{dropped} recorded {dropped === 1 ? 'run' : 'runs'} could not be loaded
|
|
</p>
|
|
) : null}
|
|
<SlotRegion id="below-timeline" />
|
|
|
|
{blindPair ? (
|
|
<BlindCompare
|
|
seed={blindPair.left.seed}
|
|
Surface={Surface}
|
|
a={{
|
|
runId: blindPair.left.id,
|
|
label: blindPair.left.label,
|
|
model: blindPair.left.model,
|
|
...(blindPair.left.intervention
|
|
? { intervention: blindPair.left.intervention }
|
|
: {}),
|
|
steps: demo.adapt(blindPair.leftEpisode),
|
|
total: rewardTotal(blindPair.leftEpisode.rewards, demo.reward.components),
|
|
}}
|
|
b={{
|
|
runId: blindPair.right.id,
|
|
label: blindPair.right.label,
|
|
model: blindPair.right.model,
|
|
...(blindPair.right.intervention
|
|
? { intervention: blindPair.right.intervention }
|
|
: {}),
|
|
steps: demo.adapt(blindPair.rightEpisode),
|
|
total: rewardTotal(blindPair.rightEpisode.rewards, demo.reward.components),
|
|
}}
|
|
/>
|
|
) : null}
|
|
</Panel>
|
|
) : null}
|
|
|
|
<Panel value="reward" className="mt-5 space-y-4 sm:mt-6">
|
|
<TabClaim>{claims.reward}</TabClaim>
|
|
|
|
{episode ? (
|
|
<RewardBreakdown
|
|
spec={demo.reward}
|
|
values={episode.rewards}
|
|
{...(episode.metrics ? { metrics: episode.metrics } : {})}
|
|
/>
|
|
) : (
|
|
<>
|
|
<p className="max-w-measure text-prose text-fg-2">
|
|
No recorded run has been scored for this environment yet, so every term below
|
|
reads as not scored rather than as zero. The weights are the ones the environment
|
|
ships.
|
|
</p>
|
|
<RewardBreakdown spec={demo.reward} values={{}} />
|
|
</>
|
|
)}
|
|
|
|
{/* The editor carries the ranking it re-orders in its own right-hand
|
|
column, so the two are never on screen apart. Absent, not greyed,
|
|
when there is nothing to rank. */}
|
|
{arms.length > 1 ? <RewardEditor spec={demo.reward} arms={arms} /> : null}
|
|
|
|
{run && episode ? (
|
|
<HeadlineMetric
|
|
label={`Total reward — ${run.label}`}
|
|
arms={arms}
|
|
components={demo.reward.components}
|
|
currentRewards={episode.rewards}
|
|
/>
|
|
) : null}
|
|
|
|
<SlotRegion id="beside-reward" />
|
|
</Panel>
|
|
|
|
<Panel value="evidence" className="mt-5 space-y-6 sm:mt-6">
|
|
<TabClaim>{claims.evidence}</TabClaim>
|
|
|
|
{/*
|
|
`narrative.thesis` is a required field of the contract and the only
|
|
paragraph on a demo that argues for the environment as a whole
|
|
rather than for one tab. It has to be SOMEWHERE, and this is the
|
|
tab a visitor opens to read rather than to do — Play stays a board
|
|
above the fold, which is the one thing a paragraph here would cost.
|
|
*/}
|
|
<div className="space-y-3">
|
|
<p className="max-w-measure text-pretty text-prose text-fg-2">{demo.narrative.thesis}</p>
|
|
{/* The readout that left the landing. Every value is data. */}
|
|
<p className="flex flex-wrap gap-x-2 gap-y-1 font-mono text-caption text-muted">
|
|
<span>
|
|
<span className="uppercase tracking-label">verifiers</span>{' '}
|
|
<span className="text-fg">{demo.provenance.verifiersVersion}</span>
|
|
</span>
|
|
{newestCapture ? (
|
|
<>
|
|
<span aria-hidden="true">·</span>
|
|
<span>
|
|
<span className="uppercase tracking-label">captured</span>{' '}
|
|
<span className="text-fg">{formatDate(newestCapture)}</span>
|
|
</span>
|
|
</>
|
|
) : null}
|
|
{run ? (
|
|
<>
|
|
<span aria-hidden="true">·</span>
|
|
<span title="The same seed reproduces this board">
|
|
<span className="uppercase tracking-label">seed</span>{' '}
|
|
<span className="text-fg">{run.seed}</span>
|
|
</span>
|
|
</>
|
|
) : null}
|
|
</p>
|
|
</div>
|
|
|
|
<EnvAnatomy anatomy={demo.anatomy} rewardLine={demo.meta.rewardLine} />
|
|
|
|
{/*
|
|
The receipt sits on Evidence, not Reward: the Evidence claim both
|
|
demos make is "here is the grader, and here is proof your browser
|
|
ran it", and that proof belongs next to the printed grader rather
|
|
than under a slider. Reward keeps the breakdown and the editor.
|
|
*/}
|
|
{episode ? <VerifyBadge demo={demo} episode={episode} /> : null}
|
|
|
|
<div className="grid grid-cols-1 gap-4 lg:grid-cols-2 lg:items-start">
|
|
<ProvenanceCard provenance={demo.provenance} {...(run ? { run } : {})} />
|
|
<CodeReceipt
|
|
code={demo.reward.source.code}
|
|
path={demo.reward.source.path}
|
|
{...(demo.reward.source.marker ? { marker: demo.reward.source.marker } : {})}
|
|
href={`${REPO_BLOB}${demo.reward.source.path}`}
|
|
/>
|
|
<SlotRegion id="after-receipts" className="lg:col-span-2" />
|
|
</div>
|
|
|
|
{!hasRecording && extras.length > 0 ? (
|
|
<div className="space-y-4">
|
|
{extras.map((tab) => (
|
|
<tab.Component key={tab.id} />
|
|
))}
|
|
</div>
|
|
) : null}
|
|
|
|
<SlotRegion id="before-limits" />
|
|
<LimitsCallout limits={demo.narrative.limits} />
|
|
</Panel>
|
|
</Tabs>
|
|
</main>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The headline metric, with every recorded arm on the same line.
|
|
*
|
|
* Arms that were never scored are dropped rather than plotted at zero — the
|
|
* difference between "scored badly" and "not scored" is the site's whole
|
|
* argument, and a chart is the easiest place in the world to lose it.
|
|
*/
|
|
function HeadlineMetric({
|
|
label,
|
|
arms,
|
|
components,
|
|
currentRewards,
|
|
}: {
|
|
label: string;
|
|
arms: RewardArm[];
|
|
components: AnyDemoModule['reward']['components'];
|
|
currentRewards: DemoEpisode['rewards'];
|
|
}) {
|
|
const points = arms
|
|
.map((arm) => ({ x: arm.label, y: rewardTotal(arm.values, components) }))
|
|
.filter((point): point is { x: string; y: number } => point.y !== null);
|
|
const baselineArm = arms[0];
|
|
const baselineValue = baselineArm ? rewardTotal(baselineArm.values, components) : null;
|
|
|
|
return (
|
|
<MetricMover
|
|
label={label}
|
|
value={rewardTotal(currentRewards, components) ?? 0}
|
|
{...(baselineArm && baselineValue !== null && arms.length > 1
|
|
? { baseline: { value: baselineValue, label: baselineArm.label } }
|
|
: {})}
|
|
series={points}
|
|
caption="Every point is a recorded run scored by the same grader. Nothing here is a projection."
|
|
/>
|
|
);
|
|
}
|
|
|
|
function RunSwitcher({
|
|
runs,
|
|
activeId,
|
|
onSelect,
|
|
}: {
|
|
runs: RunRef[];
|
|
activeId: string;
|
|
onSelect: (id: string) => void;
|
|
}) {
|
|
// Two axes, not one list. With four arms over eight seeds a flat control is
|
|
// thirty buttons carrying four distinct labels, which reads as a bug. Arms
|
|
// are grouped by `label` because that is what an arm IS in the manifest —
|
|
// the shell has no other notion of one, and inventing a field for it would
|
|
// put demo-specific structure into the contract.
|
|
const arms = useMemo(() => {
|
|
const byLabel = new Map<string, RunRef[]>();
|
|
for (const run of runs) {
|
|
const list = byLabel.get(run.label);
|
|
if (list) list.push(run);
|
|
else byLabel.set(run.label, [run]);
|
|
}
|
|
return [...byLabel.entries()].map(([label, group]) => ({ label, runs: group }));
|
|
}, [runs]);
|
|
|
|
const active = runs.find((r) => r.id === activeId) ?? runs[0];
|
|
if (!active) return null;
|
|
const activeArm = arms.find((a) => a.label === active.label) ?? arms[0];
|
|
if (!activeArm) return null;
|
|
|
|
const pickArm = (label: string) => {
|
|
const arm = arms.find((a) => a.label === label);
|
|
if (!arm) return;
|
|
// Hold the seed across an arm change where the arm has it. Comparing two
|
|
// agents means comparing them on the SAME hidden word; silently jumping to
|
|
// a different seed would make the comparison meaningless while looking fine.
|
|
const sameSeed = arm.runs.find((r) => r.seed === active.seed);
|
|
onSelect((sameSeed ?? arm.runs[0])!.id);
|
|
};
|
|
|
|
return (
|
|
<div className="flex flex-wrap items-center gap-x-4 gap-y-2">
|
|
<div className="flex flex-wrap items-center gap-2">
|
|
<Label as="span">Agent</Label>
|
|
<SegmentedControl
|
|
label="Agent"
|
|
options={arms.map((arm) => {
|
|
const intervention = arm.runs.find((r) => r.intervention)?.intervention;
|
|
return {
|
|
value: arm.label,
|
|
label: intervention ? (
|
|
<span className="inline-flex items-center gap-1.5">
|
|
{arm.label}
|
|
{/* PROMPTED: this arm's runs were intervened on. The chip
|
|
travels with the arm so a switch never hides it. */}
|
|
<Chip variant="default">prompted</Chip>
|
|
</span>
|
|
) : (
|
|
arm.label
|
|
),
|
|
...(intervention ? { title: intervention } : {}),
|
|
};
|
|
})}
|
|
value={activeArm.label}
|
|
onChange={pickArm}
|
|
/>
|
|
</div>
|
|
{activeArm.runs.length > 1 ? (
|
|
<div className="flex flex-wrap items-center gap-2">
|
|
<Label as="span">Seed</Label>
|
|
<SegmentedControl
|
|
label="Seed"
|
|
options={activeArm.runs.map((run) => ({
|
|
value: run.id,
|
|
label: `#${run.seed}`,
|
|
title: `Seed ${run.seed} — the same seed reproduces this board`,
|
|
}))}
|
|
value={active.id}
|
|
onChange={onSelect}
|
|
optionClassName="px-2.5 font-mono text-caption"
|
|
/>
|
|
</div>
|
|
) : null}
|
|
</div>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The loading state. Shaped like the page it becomes, and with no spinner: a
|
|
* spinner here would imply a live model call, which is the one thing the whole
|
|
* page is at pains to say is not happening.
|
|
*/
|
|
function ShellSkeleton() {
|
|
return (
|
|
<div className={cn(st.shell, 'py-10')} aria-busy="true">
|
|
<p className="sr-only">Loading the recorded run.</p>
|
|
<Skeleton className="h-3 w-40" />
|
|
<Skeleton className="mt-3 h-8 w-64" />
|
|
<Skeleton className="mt-3 h-5 w-full max-w-md" />
|
|
<Skeleton className="mt-6 h-11 w-full max-w-md" />
|
|
<Skeleton className="mt-6 h-80" />
|
|
</div>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The run a visitor sees before choosing one.
|
|
*
|
|
* `runs[0]` is the manifest's first entry — the weakest agent on seed 0, which
|
|
* for the first demo shipped was a failed game with thinking off. So the Watch
|
|
* tab opened on a loss with an empty reasoning panel and the Reward tab on a
|
|
* row of zeros: the model's least interesting attempt, chosen by accident of
|
|
* sort order.
|
|
*
|
|
* Prefer, in order: a run with recorded reasoning (there is something to
|
|
* stream), then a solved one (the board reaches a conclusion), then the
|
|
* earliest seed so the choice is stable across deploys. The visitor can still
|
|
* pick any run; this only decides what the page leads with.
|
|
*/
|
|
function defaultRun(runs: RunRef[], episodes: Record<string, DemoEpisode>): RunRef | undefined {
|
|
const score = (run: RunRef): number => {
|
|
const ep = episodes[run.id];
|
|
if (!ep) return -1;
|
|
const hasReasoning = ep.turns.some((t) => t.reasoning && t.reasoning.length > 0);
|
|
const solved = ep.outcome === 'solved';
|
|
return (hasReasoning ? 2 : 0) + (solved ? 1 : 0);
|
|
};
|
|
return [...runs].sort((a, b) => score(b) - score(a) || a.seed - b.seed)[0];
|
|
}
|