import { useMemo, useState } from 'react'; import { ArrowDown, ArrowUp, Minus, RotateCcw } from 'lucide-react'; import { isEdited, pruneOverrides, reweight, weightsAreDegenerate, } from '@/lib/demo-kit/reward'; import type { WeightOverrides } from '@/lib/demo-kit/reward'; import { rewardTotal } from '@/lib/demo-kit/episode'; import type { RewardSpec, RewardValues } from '@/lib/demo-kit/types'; import { Button } from '@/components/ui/button'; import { Slider } from '@/components/ui/slider'; import { cn } from '@/lib/utils'; import { formatNumber, formatOrDash } from './format'; import { EditedChip } from './StatStrip'; /** One recorded arm — a run, or a group of runs already reduced to one score. */ export interface RewardArm { id: string; label: string; /** The environment's per-component scores. Re-weighted, never re-run. */ values: RewardValues; note?: string; } export interface RewardPreset { id: string; label: string; description?: string; /** Raw weights, before normalisation. Keyed by component. */ weights: Record; } export interface RewardEditorProps { spec: RewardSpec; arms: RewardArm[]; /** Two is the right number. More and the visitor reads instead of playing. */ presets?: RewardPreset[]; onWeightsChange?: (overrides: WeightOverrides, edited: boolean) => void; className?: string; } const STEP = 0.05; /** * Presets built from the spec's own labels, for a demo that does not supply its * own. Each is phrased as a position someone would actually argue for in a * meeting, not as "preset A" and "preset B". */ function derivePresets(spec: RewardSpec): RewardPreset[] { const shipped: Record = {}; for (const component of spec.components) shipped[component.key] = component.weight; const counterweights = spec.components.filter((c) => c.role === 'counterweight'); const objectives = spec.components.filter((c) => c.role === 'objective'); const presets: RewardPreset[] = [ { id: 'shipped', label: 'What we ship', description: 'The weights in the environment as committed.', weights: shipped, }, ]; const firstObjective = objectives[0]; const firstCounterweight = counterweights[0]; if (firstCounterweight && firstObjective) { const objectiveOnly = { ...shipped }; for (const component of counterweights) objectiveOnly[component.key] = 0; presets.push({ id: 'objective-only', label: `${firstObjective.label} at any cost`, description: `Drops ${counterweights.map((c) => c.label.toLowerCase()).join(' and ')} to zero.`, weights: objectiveOnly, }); const doubled = { ...shipped }; for (const component of counterweights) doubled[component.key] = component.weight * 2; presets.push({ id: 'counterweight-heavy', // Quoted, because a component label is a phrase written for a table cell // ("Found it early") and reads as gibberish spliced into a sentence. label: `Twice as much "${firstCounterweight.label}"`, description: 'What a risk-averse buyer would ask for.', weights: doubled, }); } return presets; } function sameWeights(spec: RewardSpec, overrides: WeightOverrides, preset: RewardPreset): boolean { return spec.components.every((component) => { const current = overrides[component.key] ?? component.weight; const target = preset.weights[component.key] ?? component.weight; return Math.abs(current - target) < 1e-9; }); } /** * Change what "good" means and watch the ranking move. * * The honesty problem this component has to solve: re-weighting recorded scores * is NOT training. It shows the ranking a different reward would have produced * over these exact attempts; it cannot show the different attempts a model * trained on that reward would have made. That distinction is the permanent * caption at the bottom, and it is not collapsible. * * Weights are normalised to sum to 1 before scoring — `reweight` does it — so * dragging one slider up trades weight away from the others instead of lifting * every arm at once. Without that, the totals all rise together and the ranking * appears to move when only the scale did. */ export function RewardEditor({ spec, arms, presets, onWeightsChange, className, }: RewardEditorProps) { const [overrides, setOverrides] = useState({}); const effectivePresets = useMemo(() => presets ?? derivePresets(spec), [presets, spec]); const edited = isEdited(overrides, spec.components); const inForce = useMemo(() => reweight(spec.components, overrides), [spec, overrides]); const shippedNormalised = useMemo(() => reweight(spec.components, {}), [spec]); const degenerate = weightsAreDegenerate(inForce); const max = useMemo( () => Math.max(2, ...spec.components.map((c) => Math.ceil(Math.abs(c.weight) * 2))), [spec], ); const apply = (next: WeightOverrides) => { // Pruned before it goes into state: an override that equals the shipped // weight is not an edit, and leaving it in makes the "edited" chip stick // after the visitor drags a slider back where it started. const pruned = pruneOverrides(next, spec.components); setOverrides(pruned); onWeightsChange?.(pruned, isEdited(pruned, spec.components)); }; const ranked = useMemo(() => { const rows = arms.map((arm) => ({ arm, total: degenerate ? null : rewardTotal(arm.values, inForce), shippedTotal: rewardTotal(arm.values, shippedNormalised), })); // Nulls sort last: an unscored arm is not a zero-scoring arm. const byTotal = (a: { total: number | null }, b: { total: number | null }) => { if (a.total === null && b.total === null) return 0; if (a.total === null) return 1; if (b.total === null) return -1; return b.total - a.total; }; const shippedOrder = [...rows] .sort((a, b) => byTotal({ total: a.shippedTotal }, { total: b.shippedTotal })) .map((row) => row.arm.id); return [...rows].sort(byTotal).map((row, index) => ({ ...row, rank: index + 1, shippedRank: shippedOrder.indexOf(row.arm.id) + 1, })); }, [arms, inForce, shippedNormalised, degenerate]); const span = useMemo(() => { const totals = ranked.map((row) => row.total).filter((total): total is number => total !== null); if (totals.length === 0) return { lo: 0, hi: 1 }; const lo = Math.min(0, ...totals); const hi = Math.max(...totals); return { lo, hi: hi === lo ? lo + 1 : hi }; }, [ranked]); const leader = ranked[0]; return (

Change what good means

{edited ? : null}
{effectivePresets.map((preset) => { const active = sameWeights(spec, overrides, preset); return ( ); })}
{spec.components.map((component, index) => { const raw = overrides[component.key] ?? component.weight; const changed = Math.abs(raw - component.weight) > 1e-9; const share = inForce[index]?.weight ?? 0; return (
{/* A

{component.description}

apply({ ...overrides, [component.key]: next[0] ?? component.weight }) } aria-labelledby={`weight-label-${component.key}`} />
); })}

The percentage is the share of the reward each term carries once the weights are normalised. Raising one lowers the others — that is the trade a reward designer actually makes.

Ranking under this reward {edited ? : null}

{degenerate ? (

Every weight is zero, so there is no reward left to rank by. That is not a score of nought — it is a reward that expresses no preference at all.

) : (
    {ranked.map((row) => { const moved = row.rank - row.shippedRank; const width = row.total === null ? 0 : Math.max(2, ((row.total - span.lo) / (span.hi - span.lo)) * 100); return (
  1. {row.rank} {row.arm.label} {formatOrDash(row.total)}
  2. ); })}
)} {/* Safe to announce here without fighting the shell's step-change region: a slider drag and a step advance never fire from one action. */}

{leader && leader.total !== null ? `Leading under this reward: ${leader.arm.label}, ${formatOrDash(leader.total)}.` : ''}

We re-scored the same recorded attempts under your reward. Training on it would change the behaviour, not just the ranking.

); } function RankMove({ moved }: { moved: number }) { if (moved === 0) { return ( ); } const up = moved < 0; const places = Math.abs(moved); return ( {up ? ); }