karti-ai
5bbf913664
The environment page was a linear scroll of eight narrative beats. That is an essay, and it is the wrong shape for somebody who has just chosen an environment and wants to use it. It is now four tabs — Play, Watch, Reward, Evidence — opening on Play, with the board above the fold at 390x844 and the anatomy strip directly beneath it. The landing page leads with the picker instead of burying it under the thesis. The contract changed rather than layering tabs over beats. `Narrative.beats` is gone; `claims: Record<DemoTabId, string>` replaces it, one required sentence per tab. Writing the claim is how an author discovers whether a tab has anything to say — a tab whose claim is hard to write is usually a tab with nothing in it. Doing this now costs one migration; doing it after eleven more environments costs twelve. Tabs are derived, never declared: Play iff the demo ships an `interactive` mode, Watch iff it has recorded runs. A demo that could name its own tabs would mean environment seven inventing a fifth one and the site ceasing to be one product. One thing the browser caught that no gate would have. The header stat strip describes the RECORDED RUN, and on Play it sat above the visitor's own empty board reading "Outcome: failed" — which parses as your game having already failed before you touch a key. It now renders only on the tabs whose subject is that run, which also moved the board 54px up the page. The picker is honest about the shape of the lineup by construction: one built environment gets its own block and the demo's real board as its thumbnail, twelve written specifications render dimmed with a Spec badge, and every count on the page is derived from the data rather than typed. 206 contract checks pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
726 lines
27 KiB
TypeScript
726 lines
27 KiB
TypeScript
import { useEffect, useMemo, useState } from 'react';
|
|
import type { ComponentType, ReactNode } from 'react';
|
|
import { useParams } from 'react-router-dom';
|
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
|
import { Skeleton } from '@/components/ui/skeleton';
|
|
import { listRuns, loadEpisode, rewardTotal } from '@/lib/demo-kit/episode';
|
|
import { usePlayer } from '@/lib/demo-kit/player';
|
|
import { loadDemoModule } from '@/lib/demo-kit/registry';
|
|
import type { AnyDemoModule } from '@/lib/demo-kit/registry';
|
|
import type { DemoEpisode, DemoStep, DemoTabId, RunRef } from '@/lib/demo-kit/types';
|
|
import { useRunParam, useSpeedParam, useStepParam, useTabParam } from '@/lib/url-state';
|
|
import * as st from '@/content/styles';
|
|
import { cn } from '@/lib/utils';
|
|
import { BlindCompare } from './BlindCompare';
|
|
import { CodeReceipt } from './CodeReceipt';
|
|
import { DemoErrorBoundary } from './DemoErrorBoundary';
|
|
import { DemoTabBar, TabClaim, resolveTab, visibleTabs } from './DemoTabs';
|
|
import { EnvAnatomy } from './EnvAnatomy';
|
|
import { LimitsCallout } from './LimitsCallout';
|
|
import { MetricMover } from './MetricMover';
|
|
import { ModelCallPanel } from './ModelCallPanel';
|
|
import { PlayYourself } from './PlayYourself';
|
|
import { ProvenanceCard } from './ProvenanceCard';
|
|
import { ReasoningDrawer } from './ReasoningDrawer';
|
|
import { ReasoningPanel } from './ReasoningPanel';
|
|
import { RewardBreakdown } from './RewardBreakdown';
|
|
import { RewardEditor } from './RewardEditor';
|
|
import type { RewardArm } from './RewardEditor';
|
|
import { SegmentedControl } from './SegmentedControl';
|
|
import { SlotRegion } from './SlotRegion';
|
|
import { StepTimeline } from './StepTimeline';
|
|
import { StatStrip } from './StatStrip';
|
|
import type { Stat } from './StatStrip';
|
|
import { RecordedBadge, TracePlayer } from './TracePlayer';
|
|
import { VerifyBadge } from './VerifyBadge';
|
|
import { formatOrDash, useIsDesktop } from './format';
|
|
|
|
const REPO_BLOB = 'https://git.karti.ai/PIG/PIG-Demo/src/branch/main/';
|
|
|
|
/**
|
|
* The panel the step-detail strip inside the Watch tab opens on.
|
|
*
|
|
* This control is deliberately NOT in the URL. `?tab=` now belongs to the page's
|
|
* four top-level tabs, and one param cannot address two nested controls without
|
|
* one of them silently winning; a permalink to `?tab=call` would land the reader
|
|
* on a page with no such top-level tab.
|
|
*/
|
|
const DEFAULT_DETAIL_PANEL = 'reasoning';
|
|
|
|
/** Reserved slug for the shell's own hand-written demo. Dev builds only. */
|
|
const MOCK_SLUG = '__mock';
|
|
|
|
export interface DemoBundle {
|
|
demo: AnyDemoModule;
|
|
runs: RunRef[];
|
|
episodes: Record<string, DemoEpisode>;
|
|
}
|
|
|
|
type LoadState =
|
|
| { status: 'loading' }
|
|
| { status: 'ready'; bundle: DemoBundle }
|
|
| { status: 'error'; message: string };
|
|
|
|
/**
|
|
* A demo's module plus every recorded run it has.
|
|
*
|
|
* `loadDemoModule` and `loadEpisode` both cache their promises, so the route
|
|
* loader having already fetched the module makes this resolve without a second
|
|
* request. Runs are loaded with `allSettled` on purpose: one unreadable trace
|
|
* drops that arm rather than blanking the page.
|
|
*/
|
|
async function loadBundle(slug: string): Promise<DemoBundle> {
|
|
if (slug === MOCK_SLUG) {
|
|
// Dynamic, so the mock lands in its own chunk and production never fetches
|
|
// it. A static import would ship several hundred lines of fake trace to
|
|
// every visitor of every real demo.
|
|
const mock = await import('./mock');
|
|
return { demo: mock.mockDemo, runs: mock.mockRuns, episodes: mock.mockEpisodes };
|
|
}
|
|
|
|
const demo = await loadDemoModule(slug);
|
|
const runs = await listRuns(slug).catch(() => [] as RunRef[]);
|
|
const settled = await Promise.allSettled(runs.map((run) => loadEpisode(run)));
|
|
|
|
const episodes: Record<string, DemoEpisode> = {};
|
|
settled.forEach((outcome, index) => {
|
|
const run = runs[index];
|
|
if (!run) return;
|
|
if (outcome.status === 'fulfilled') episodes[run.id] = outcome.value;
|
|
else console.error(`[pig-demo] dropped run "${run.id}":`, outcome.reason);
|
|
});
|
|
|
|
return { demo, runs: runs.filter((run) => episodes[run.id] !== undefined), episodes };
|
|
}
|
|
|
|
export interface DemoShellProps {
|
|
/** Overrides the route param. Useful for previews and tests. */
|
|
slug?: string;
|
|
/** Skips loading entirely when the caller already has the bundle. */
|
|
bundle?: DemoBundle;
|
|
}
|
|
|
|
/**
|
|
* The route component every demo is rendered through.
|
|
*
|
|
* It owns four things and no more: loading, which tabs exist, the URL state,
|
|
* and the page's single polite live region. Everything visual is delegated to
|
|
* the surfaces in this directory, and the demo module is never reached into —
|
|
* the shell only ever calls `adapt` and renders `Surface`.
|
|
*/
|
|
export function DemoShell({ slug: slugProp, bundle }: DemoShellProps) {
|
|
const params = useParams();
|
|
const slug = slugProp ?? params['slug'] ?? '';
|
|
const [state, setState] = useState<LoadState>(
|
|
bundle ? { status: 'ready', bundle } : { status: 'loading' },
|
|
);
|
|
|
|
useEffect(() => {
|
|
if (bundle) {
|
|
setState({ status: 'ready', bundle });
|
|
return;
|
|
}
|
|
let live = true;
|
|
setState({ status: 'loading' });
|
|
loadBundle(slug)
|
|
.then((loaded) => {
|
|
if (live) setState({ status: 'ready', bundle: loaded });
|
|
})
|
|
.catch((error: unknown) => {
|
|
if (!live) return;
|
|
setState({
|
|
status: 'error',
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
});
|
|
return () => {
|
|
live = false;
|
|
};
|
|
}, [slug, bundle]);
|
|
|
|
if (state.status === 'loading') return <ShellSkeleton />;
|
|
if (state.status === 'error') {
|
|
return (
|
|
<main className={cn(st.shell, 'py-16')}>
|
|
<div role="alert" className="card max-w-xl p-6">
|
|
<h1 className={st.h2}>That demo is not here</h1>
|
|
<p className={cn(st.prose, 'mt-3')}>{state.message}</p>
|
|
<a href="/gallery" className={cn(st.btnSecondary, 'mt-6')}>
|
|
Back to the demos
|
|
</a>
|
|
</div>
|
|
</main>
|
|
);
|
|
}
|
|
|
|
return (
|
|
// A second boundary inside the route's own: this one is keyed to the demo
|
|
// so a crash names it, and resetting re-renders the surfaces rather than
|
|
// re-navigating.
|
|
<DemoErrorBoundary demoTitle={state.bundle.demo.meta.title}>
|
|
<DemoBody bundle={state.bundle} />
|
|
</DemoErrorBoundary>
|
|
);
|
|
}
|
|
|
|
function DemoBody({ bundle }: { bundle: DemoBundle }) {
|
|
const { demo, runs, episodes } = bundle;
|
|
const isDesktop = useIsDesktop();
|
|
|
|
const [runParam, setRunParam] = useRunParam();
|
|
const [stepParam, setStepParam] = useStepParam();
|
|
const [speedParam, setSpeedParam] = useSpeedParam();
|
|
|
|
const run = useMemo(
|
|
() => runs.find((candidate) => candidate.id === runParam) ?? runs[0],
|
|
[runs, runParam],
|
|
);
|
|
const episode = run ? episodes[run.id] : undefined;
|
|
|
|
const steps = useMemo<DemoStep<unknown>[]>(
|
|
() => (episode ? demo.adapt(episode) : []),
|
|
[demo, episode],
|
|
);
|
|
|
|
const player = usePlayer(steps, {
|
|
initialIndex: stepParam,
|
|
initialSpeed: speedParam,
|
|
onIndexChange: setStepParam,
|
|
});
|
|
|
|
// The URL is the other writer of this state — Back, a pasted permalink, the
|
|
// run switcher. The player is the source of truth while it is running, so it
|
|
// only follows the URL when the two have actually diverged.
|
|
const { seek } = player;
|
|
useEffect(() => {
|
|
if (stepParam !== player.index) seek(stepParam);
|
|
// Intentionally keyed on the URL only: including `player.index` here would
|
|
// re-run the effect on the player's own advance and fight it.
|
|
}, [stepParam, seek]);
|
|
|
|
// Derived, never declared. A demo that ships no interactive mode has no Play
|
|
// tab and opens on Watch; one whose traces failed to load has no Watch tab
|
|
// and opens on Reward.
|
|
const hasRecording = Boolean(run && episode && steps.length > 0);
|
|
const tabs = useMemo(
|
|
() => visibleTabs({ play: Boolean(demo.interactive), watch: hasRecording }),
|
|
[demo.interactive, hasRecording],
|
|
);
|
|
// `visibleTabs` always keeps `reward` and `evidence`, so index 0 exists; the
|
|
// fallback is here only so the type does not need an assertion.
|
|
const defaultTab: DemoTabId = tabs[0] ?? 'evidence';
|
|
const [tabParam, setTabParam] = useTabParam(defaultTab);
|
|
const activeTab = resolveTab(tabParam, tabs, defaultTab);
|
|
|
|
// React-only, not a URL param. See DEFAULT_DETAIL_PANEL.
|
|
const [detailPanel, setDetailPanel] = useState(DEFAULT_DETAIL_PANEL);
|
|
|
|
const arms = useMemo<RewardArm[]>(
|
|
() =>
|
|
runs.map((candidate) => {
|
|
const armEpisode = episodes[candidate.id];
|
|
const arm: RewardArm = {
|
|
id: candidate.id,
|
|
label: candidate.label,
|
|
values: armEpisode?.rewards ?? {},
|
|
};
|
|
if (candidate.intervention) arm.note = candidate.intervention;
|
|
return arm;
|
|
}),
|
|
[runs, episodes],
|
|
);
|
|
|
|
const blindPair = useMemo(() => {
|
|
for (let i = 0; i < runs.length; i += 1) {
|
|
for (let j = i + 1; j < runs.length; j += 1) {
|
|
const left = runs[i];
|
|
const right = runs[j];
|
|
if (!left || !right || left.seed !== right.seed) continue;
|
|
const leftEpisode = episodes[left.id];
|
|
const rightEpisode = episodes[right.id];
|
|
if (!leftEpisode || !rightEpisode) continue;
|
|
return { left, right, leftEpisode, rightEpisode };
|
|
}
|
|
}
|
|
// Two runs on different seeds are two different puzzles; showing them side
|
|
// by side would be a comparison of luck.
|
|
return null;
|
|
}, [runs, episodes]);
|
|
|
|
const Surface = demo.Surface as ComponentType<{ state: unknown; compact?: boolean }>;
|
|
const current = steps[player.index];
|
|
const claims = demo.narrative.claims;
|
|
const extras = demo.tabs ?? [];
|
|
|
|
// The seed is the shared coordinate between the visitor's board and the
|
|
// agent's. With no recording to match, the demo's own first board will do.
|
|
const playSeed = run?.seed ?? 0;
|
|
|
|
const headerStats: Stat[] = episode
|
|
? [
|
|
{
|
|
label: 'Outcome',
|
|
value: episode.outcome,
|
|
tone: episode.outcome === 'solved' ? 'positive' : 'warning',
|
|
...(episode.truncated ? { title: 'Truncated before a terminal state' } : {}),
|
|
},
|
|
{
|
|
label: 'Reward',
|
|
value: formatOrDash(rewardTotal(episode.rewards, demo.reward.components)),
|
|
tone: 'brand',
|
|
title: 'Total under the shipped weights',
|
|
},
|
|
{ label: 'Steps', value: steps.length, title: 'Model calls in this run' },
|
|
{
|
|
label: 'Seed',
|
|
value: episode.seed,
|
|
title: 'The same seed reproduces this board',
|
|
},
|
|
]
|
|
: [];
|
|
|
|
const detailPanels: { id: string; label: string; content: ReactNode }[] = [
|
|
{
|
|
id: 'reasoning',
|
|
label: 'Reasoning',
|
|
content: isDesktop ? (
|
|
<ReasoningPanel
|
|
reasoning={current?.reasoning ?? null}
|
|
durationMs={current?.call?.durationMs ?? null}
|
|
playing={player.isPlaying}
|
|
speed={player.speed}
|
|
stepIndex={player.index}
|
|
/>
|
|
) : (
|
|
// Under `lg` there is no column for this, and putting it below the
|
|
// board means watching the run with the thinking off-screen. The sheet
|
|
// is mounted only here, so vaul never locks body scroll on desktop.
|
|
<ReasoningDrawer
|
|
reasoning={current?.reasoning ?? null}
|
|
durationMs={current?.call?.durationMs ?? null}
|
|
playing={player.isPlaying}
|
|
speed={player.speed}
|
|
stepIndex={player.index}
|
|
/>
|
|
),
|
|
},
|
|
{
|
|
id: 'call',
|
|
label: 'Model call',
|
|
content: (
|
|
<div className="space-y-3">
|
|
<ModelCallPanel call={current?.call ?? null} />
|
|
{current?.reply ? (
|
|
<div className="card p-3">
|
|
<h3 className="text-sm font-semibold">Reply</h3>
|
|
<p className="mt-1 whitespace-pre-wrap font-mono text-xs leading-relaxed">
|
|
{current.reply}
|
|
</p>
|
|
</div>
|
|
) : null}
|
|
</div>
|
|
),
|
|
},
|
|
// A demo's own extra panels ride alongside the step detail, where they sit
|
|
// next to the step they are almost always about. With no recording there is
|
|
// no step detail, so the Evidence tab picks them up instead.
|
|
...(hasRecording
|
|
? extras.map((tab) => ({ id: tab.id, label: tab.label, content: <tab.Component /> }))
|
|
: []),
|
|
];
|
|
const activeDetail = detailPanels.some((panel) => panel.id === detailPanel)
|
|
? detailPanel
|
|
: DEFAULT_DETAIL_PANEL;
|
|
|
|
const timeline = (
|
|
<StepTimeline
|
|
steps={steps}
|
|
current={player.index}
|
|
onSelect={(next) => {
|
|
player.pause();
|
|
player.seek(next);
|
|
}}
|
|
Surface={Surface}
|
|
onTogglePlay={player.toggle}
|
|
/>
|
|
);
|
|
|
|
return (
|
|
<main className={cn(st.shell, 'pb-16')}>
|
|
{/*
|
|
The page's ONE live region. Every step change lands here and nowhere
|
|
else: with reduced motion the board animation is gone, so this sentence
|
|
is the only thing that tells a screen-reader user what just happened.
|
|
*/}
|
|
<div aria-live="polite" aria-atomic="true" className="sr-only">
|
|
{current?.announce ?? ''}
|
|
</div>
|
|
|
|
<header className="pt-6 sm:pt-8">
|
|
<p className={cn(st.eyebrow, 'text-accent-fg')}>For {demo.meta.persona}</p>
|
|
<h1 className={cn(st.h2, 'mt-1')}>{demo.meta.title}</h1>
|
|
<p className={cn(st.lede, 'mt-2 max-w-prose')}>{demo.meta.tagline}</p>
|
|
{/* The buyer's question, kept quiet on purpose: it is the thing they
|
|
walked in with, not the thing this page is asserting. */}
|
|
<p className="mt-2 max-w-prose text-sm italic leading-relaxed text-muted">
|
|
{demo.narrative.anxiety}
|
|
</p>
|
|
{/*
|
|
The stats describe the RECORDED RUN, so they only belong on the tabs
|
|
whose subject is that run. On Play they sat above the visitor's own
|
|
empty board reading "Outcome: failed", which parses as *your* game
|
|
having already failed before you have touched a key.
|
|
*/}
|
|
{run && headerStats.length > 0 && activeTab !== 'play' ? (
|
|
// One line that scrolls itself rather than a block that wraps: every
|
|
// row this header spends is a row of the interactive board pushed
|
|
// below the fold on a 390px phone.
|
|
<div className="mt-3 flex items-center gap-2 overflow-x-auto pb-1 sm:flex-wrap sm:overflow-visible sm:pb-0">
|
|
<StatStrip stats={headerStats} />
|
|
<RecordedBadge
|
|
model={run.model}
|
|
capturedAt={run.capturedAt}
|
|
{...(run.intervention ? { intervention: run.intervention } : {})}
|
|
className="ml-0 shrink-0 flex-nowrap"
|
|
/>
|
|
</div>
|
|
) : null}
|
|
<SlotRegion id="hero-aside" />
|
|
</header>
|
|
|
|
<Tabs value={activeTab} onValueChange={setTabParam} className="mt-4 sm:mt-5">
|
|
<DemoTabBar tabs={tabs} />
|
|
|
|
{tabs.includes('play') ? (
|
|
// `forceMount` keeps the visitor's half-finished board alive while
|
|
// they read the other tabs, so a game in progress survives a trip to
|
|
// Reward and back. Radix leaves the hiding to the author under
|
|
// `forceMount`, which is what the `data-[state=inactive]` class does —
|
|
// it is load-bearing, not belt-and-braces.
|
|
<TabsContent
|
|
value="play"
|
|
forceMount
|
|
className="mt-5 space-y-5 data-[state=inactive]:hidden sm:mt-6 sm:space-y-6"
|
|
>
|
|
<TabClaim>{claims.play}</TabClaim>
|
|
<PlayYourself demo={demo} seed={playSeed} />
|
|
<SlotRegion id="below-board" />
|
|
<div className="space-y-2">
|
|
<h2 className={st.h3}>The machine you are inside</h2>
|
|
<EnvAnatomy anatomy={demo.anatomy} rewardLine={demo.meta.rewardLine} compact />
|
|
</div>
|
|
</TabsContent>
|
|
) : null}
|
|
|
|
{tabs.includes('watch') && run && episode ? (
|
|
<TabsContent value="watch" className="mt-6 space-y-4">
|
|
<TabClaim>{claims.watch}</TabClaim>
|
|
|
|
{runs.length > 1 ? (
|
|
<RunSwitcher
|
|
runs={runs}
|
|
activeId={run.id}
|
|
onSelect={(id) => {
|
|
player.pause();
|
|
// The run param setter also zeroes `step`: step 6 of a
|
|
// nine-turn rollout is not step 6 of a three-turn one.
|
|
setRunParam(id);
|
|
}}
|
|
/>
|
|
) : null}
|
|
|
|
<TracePlayer
|
|
playing={player.isPlaying}
|
|
onPlayingChange={(next) => (next ? player.play() : player.pause())}
|
|
speed={player.speed}
|
|
onSpeedChange={(next) => {
|
|
player.setSpeed(next);
|
|
setSpeedParam(next);
|
|
// `instant` is a destination, not a rate. The player only
|
|
// consumes it while running, so choosing it from a paused
|
|
// transport has to start the run — otherwise the button
|
|
// visibly does nothing, which reads as broken.
|
|
if (next === 'instant') player.play();
|
|
}}
|
|
onRestart={player.restart}
|
|
step={player.index}
|
|
stepCount={steps.length}
|
|
onStepChange={(next) => {
|
|
player.pause();
|
|
player.seek(next);
|
|
}}
|
|
progress={player.progress}
|
|
timingIsReal={player.timingIsReal}
|
|
model={run.model}
|
|
capturedAt={run.capturedAt}
|
|
{...(run.intervention ? { intervention: run.intervention } : {})}
|
|
/>
|
|
|
|
<div className="grid grid-cols-1 gap-3 lg:grid-cols-[auto_minmax(0,1fr)] lg:items-start">
|
|
<div className="card w-fit p-4">
|
|
{current ? <Surface state={current.state} /> : null}
|
|
</div>
|
|
|
|
<div className="min-w-0">
|
|
<Tabs value={activeDetail} onValueChange={setDetailPanel}>
|
|
<TabsList aria-label="Details for this step" className="w-full overflow-x-auto">
|
|
{detailPanels.map((panel) => (
|
|
<TabsTrigger key={panel.id} value={panel.id}>
|
|
{panel.label}
|
|
</TabsTrigger>
|
|
))}
|
|
</TabsList>
|
|
{detailPanels.map((panel) => (
|
|
<TabsContent key={panel.id} value={panel.id}>
|
|
{panel.content}
|
|
</TabsContent>
|
|
))}
|
|
</Tabs>
|
|
</div>
|
|
</div>
|
|
|
|
{timeline}
|
|
{player.timingIsReal ? null : (
|
|
<p className="text-xs text-muted">
|
|
Some steps in this run carried no recorded latency, so their dwell on the timeline
|
|
is the player's fallback rather than a measurement.
|
|
</p>
|
|
)}
|
|
<SlotRegion id="below-timeline" />
|
|
|
|
{blindPair ? (
|
|
<BlindCompare
|
|
seed={blindPair.left.seed}
|
|
Surface={Surface}
|
|
a={{
|
|
runId: blindPair.left.id,
|
|
label: blindPair.left.label,
|
|
model: blindPair.left.model,
|
|
...(blindPair.left.intervention
|
|
? { intervention: blindPair.left.intervention }
|
|
: {}),
|
|
steps: demo.adapt(blindPair.leftEpisode),
|
|
total: rewardTotal(blindPair.leftEpisode.rewards, demo.reward.components),
|
|
}}
|
|
b={{
|
|
runId: blindPair.right.id,
|
|
label: blindPair.right.label,
|
|
model: blindPair.right.model,
|
|
...(blindPair.right.intervention
|
|
? { intervention: blindPair.right.intervention }
|
|
: {}),
|
|
steps: demo.adapt(blindPair.rightEpisode),
|
|
total: rewardTotal(blindPair.rightEpisode.rewards, demo.reward.components),
|
|
}}
|
|
/>
|
|
) : null}
|
|
</TabsContent>
|
|
) : null}
|
|
|
|
<TabsContent value="reward" className="mt-6 space-y-4">
|
|
<TabClaim>{claims.reward}</TabClaim>
|
|
|
|
{episode ? (
|
|
<RewardBreakdown
|
|
spec={demo.reward}
|
|
values={episode.rewards}
|
|
{...(episode.metrics ? { metrics: episode.metrics } : {})}
|
|
/>
|
|
) : (
|
|
<>
|
|
<p className={cn(st.prose, 'max-w-prose')}>
|
|
No recorded run has been scored for this environment yet, so every term below
|
|
reads as not scored rather than as zero. The weights are the ones the environment
|
|
ships.
|
|
</p>
|
|
<RewardBreakdown spec={demo.reward} values={{}} />
|
|
</>
|
|
)}
|
|
|
|
{/* The editor carries the ranking it re-orders in its own right-hand
|
|
column, so the two are never on screen apart. */}
|
|
{arms.length > 1 ? <RewardEditor spec={demo.reward} arms={arms} /> : null}
|
|
|
|
{episode ? <VerifyBadge demo={demo} episode={episode} /> : null}
|
|
|
|
{run && episode ? (
|
|
<HeadlineMetric
|
|
label={`Total reward — ${run.label}`}
|
|
arms={arms}
|
|
components={demo.reward.components}
|
|
currentRewards={episode.rewards}
|
|
/>
|
|
) : null}
|
|
|
|
<SlotRegion id="beside-reward" />
|
|
</TabsContent>
|
|
|
|
<TabsContent value="evidence" className="mt-6 space-y-6">
|
|
<TabClaim>{claims.evidence}</TabClaim>
|
|
|
|
{/*
|
|
`narrative.thesis` is a required field of the contract and the only
|
|
paragraph on a demo that argues for the environment as a whole
|
|
rather than for one tab. It has to be SOMEWHERE, and this is the
|
|
tab a visitor opens to read rather than to do — Play stays a board
|
|
above the fold, which is the one thing a paragraph here would cost.
|
|
*/}
|
|
<p className={cn(st.prose, 'max-w-prose')}>{demo.narrative.thesis}</p>
|
|
|
|
<EnvAnatomy anatomy={demo.anatomy} rewardLine={demo.meta.rewardLine} />
|
|
|
|
<div className="grid grid-cols-1 gap-4 lg:grid-cols-2 lg:items-start">
|
|
<ProvenanceCard provenance={demo.provenance} {...(run ? { run } : {})} />
|
|
<CodeReceipt
|
|
code={demo.reward.source.code}
|
|
path={demo.reward.source.path}
|
|
{...(demo.reward.source.marker ? { marker: demo.reward.source.marker } : {})}
|
|
href={`${REPO_BLOB}${demo.reward.source.path}`}
|
|
/>
|
|
<SlotRegion id="after-receipts" className="lg:col-span-2" />
|
|
</div>
|
|
|
|
{!hasRecording && extras.length > 0 ? (
|
|
<div className="space-y-4">
|
|
{extras.map((tab) => (
|
|
<tab.Component key={tab.id} />
|
|
))}
|
|
</div>
|
|
) : null}
|
|
|
|
<SlotRegion id="before-limits" />
|
|
<LimitsCallout limits={demo.narrative.limits} />
|
|
</TabsContent>
|
|
</Tabs>
|
|
</main>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The headline metric, with every recorded arm on the same line.
|
|
*
|
|
* Arms that were never scored are dropped rather than plotted at zero — the
|
|
* difference between "scored badly" and "not scored" is the site's whole
|
|
* argument, and a chart is the easiest place in the world to lose it.
|
|
*/
|
|
function HeadlineMetric({
|
|
label,
|
|
arms,
|
|
components,
|
|
currentRewards,
|
|
}: {
|
|
label: string;
|
|
arms: RewardArm[];
|
|
components: AnyDemoModule['reward']['components'];
|
|
currentRewards: DemoEpisode['rewards'];
|
|
}) {
|
|
const points = arms
|
|
.map((arm) => ({ x: arm.label, y: rewardTotal(arm.values, components) }))
|
|
.filter((point): point is { x: string; y: number } => point.y !== null);
|
|
const baselineArm = arms[0];
|
|
const baselineValue = baselineArm ? rewardTotal(baselineArm.values, components) : null;
|
|
|
|
return (
|
|
<MetricMover
|
|
label={label}
|
|
value={rewardTotal(currentRewards, components) ?? 0}
|
|
{...(baselineArm && baselineValue !== null && arms.length > 1
|
|
? { baseline: { value: baselineValue, label: baselineArm.label } }
|
|
: {})}
|
|
series={points}
|
|
caption="Every point is a recorded run scored by the same grader. Nothing here is a projection."
|
|
/>
|
|
);
|
|
}
|
|
|
|
function RunSwitcher({
|
|
runs,
|
|
activeId,
|
|
onSelect,
|
|
}: {
|
|
runs: RunRef[];
|
|
activeId: string;
|
|
onSelect: (id: string) => void;
|
|
}) {
|
|
// Two axes, not one list. With four arms over eight seeds a flat control is
|
|
// thirty buttons carrying four distinct labels, which reads as a bug. Arms
|
|
// are grouped by `label` because that is what an arm IS in the manifest —
|
|
// the shell has no other notion of one, and inventing a field for it would
|
|
// put demo-specific structure into the contract.
|
|
const arms = useMemo(() => {
|
|
const byLabel = new Map<string, RunRef[]>();
|
|
for (const run of runs) {
|
|
const list = byLabel.get(run.label);
|
|
if (list) list.push(run);
|
|
else byLabel.set(run.label, [run]);
|
|
}
|
|
return [...byLabel.entries()].map(([label, group]) => ({ label, runs: group }));
|
|
}, [runs]);
|
|
|
|
const active = runs.find((r) => r.id === activeId) ?? runs[0];
|
|
if (!active) return null;
|
|
const activeArm = arms.find((a) => a.label === active.label) ?? arms[0];
|
|
if (!activeArm) return null;
|
|
|
|
const pickArm = (label: string) => {
|
|
const arm = arms.find((a) => a.label === label);
|
|
if (!arm) return;
|
|
// Hold the seed across an arm change where the arm has it. Comparing two
|
|
// agents means comparing them on the SAME hidden word; silently jumping to
|
|
// a different seed would make the comparison meaningless while looking fine.
|
|
const sameSeed = arm.runs.find((r) => r.seed === active.seed);
|
|
onSelect((sameSeed ?? arm.runs[0])!.id);
|
|
};
|
|
|
|
return (
|
|
<div className="flex flex-wrap items-center gap-x-4 gap-y-2">
|
|
<SegmentedControl
|
|
label="Agent"
|
|
options={arms.map((arm) => {
|
|
const intervention = arm.runs.find((r) => r.intervention)?.intervention;
|
|
return {
|
|
value: arm.label,
|
|
label: arm.label,
|
|
...(intervention ? { title: intervention } : {}),
|
|
};
|
|
})}
|
|
value={activeArm.label}
|
|
onChange={pickArm}
|
|
className="border border-border p-1"
|
|
optionClassName="tap px-3 text-sm"
|
|
/>
|
|
{activeArm.runs.length > 1 ? (
|
|
<SegmentedControl
|
|
label="Hidden word"
|
|
options={activeArm.runs.map((run) => ({
|
|
value: run.id,
|
|
label: `#${run.seed}`,
|
|
}))}
|
|
value={active.id}
|
|
onChange={onSelect}
|
|
className="border border-border p-1"
|
|
optionClassName="tap px-2.5 text-xs nums"
|
|
/>
|
|
) : null}
|
|
</div>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The loading state. Shaped like the page it becomes, and with no spinner: a
|
|
* spinner here would imply a live model call, which is the one thing the whole
|
|
* page is at pains to say is not happening.
|
|
*/
|
|
function ShellSkeleton() {
|
|
return (
|
|
<div className={cn(st.shell, 'py-10')} aria-busy="true">
|
|
<p className="sr-only">Loading the recorded run.</p>
|
|
<Skeleton className="h-8 w-64" />
|
|
<Skeleton className="mt-3 h-4 w-full max-w-md" />
|
|
<Skeleton className="mt-4 h-10 w-full max-w-sm" />
|
|
<Skeleton className="mt-6 h-11 w-full max-w-md" />
|
|
<Skeleton className="mt-6 h-80" />
|
|
</div>
|
|
);
|
|
}
|