import { useState } from 'react'; import type { ComponentType } from 'react'; import { Eye, Trophy } from 'lucide-react'; import type { DemoStep } from '@/lib/demo-kit/types'; import { Button } from '@/components/ui/button'; import { cn } from '@/lib/utils'; import { formatOrDash } from './format'; export interface BlindCompareRun { runId: string; /** The identity, revealed only after the visitor commits. */ label: string; model: string; /** Required by the contract on an `intervened` run; shown at reveal. */ intervention?: string; steps: DemoStep[]; total: number | null; } export type BlindVote = 'A' | 'B' | 'tie'; export interface BlindCompareProps { /** Both runs must be the same seed or the comparison is meaningless. */ seed: number; a: BlindCompareRun; b: BlindCompareRun; Surface: ComponentType<{ state: T; compact?: boolean }>; question?: string; onVote?: (vote: BlindVote) => void; className?: string; } /** * Two runs on the same seed, unlabelled, until you commit. * * The point is not the vote. The point is that the visitor forms an opinion * from the behaviour BEFORE they learn which one had the better prompt, the * bigger model or the training run — because once they know, they cannot * unknow it, and every "obviously the trained one looks better" is worthless * after the fact. * * There is a "reveal without voting" escape on purpose: a visitor who does not * want to play should not be held hostage by a modal-shaped page. */ export function BlindCompare({ seed, a, b, Surface, question = 'Which agent would you rather have running this?', onVote, className, }: BlindCompareProps) { const [vote, setVote] = useState(null); const [revealed, setRevealed] = useState(false); const commit = (choice: BlindVote) => { setVote(choice); setRevealed(true); onVote?.(choice); }; const winner: 'A' | 'B' | 'tie' = a.total === null || b.total === null ? 'tie' : a.total > b.total ? 'A' : b.total > a.total ? 'B' : 'tie'; const sides: { id: 'A' | 'B'; run: BlindCompareRun }[] = [ { id: 'A', run: a }, { id: 'B', run: b }, ]; return (

{question}

Same puzzle, same seed ({seed}).

{sides.map(({ id, run }) => { const last = run.steps[run.steps.length - 1]; const picked = vote === id; return (

Agent {id}

{run.steps.length} steps
{last ? ( ) : (

This run recorded no steps.

)}
{!revealed ? ( ) : (
Identity
{run.label}
Model
{run.model}
Intervention
{run.intervention ?? ( none — plain rollout )}
Reward
{revealed && winner === id ? (
)}
); })}
{!revealed ? (
) : (

{vote === null ? ( <>Revealed without a vote. ) : vote === winner ? ( <> You picked the higher-scoring run.{' '} ) : ( <> You picked the lower-scoring run. {' '} )} The environment scored these two with the same grader, on the same seed. The difference between them is stated above — and if it is a prompt change rather than a training run, it says so, because a prompt change presented as a training result is the oldest trick in this business.

)}
); }