diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7679d46..7b80ba7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,11 +51,20 @@ jobs: # A blob-backed worker is blocked in production and nowhere else: the # site's CSP has no worker-src, so it falls back to default-src 'self'. - # The failure is silent — the solver simply never boots. - - name: no inline workers + # The failure is silent — the worker simply never boots. + # + # Grep for worker construction FROM a blob, not for the string `blob:` + # anywhere. React's own bundle contains that string in a URL-scheme check, + # so the broad version fails on a risk that is not present, which teaches + # everyone to ignore it. + - name: no blob-backed workers run: | - ! grep -rqE "createObjectURL|blob:" dist/assets/*.js \ - || { echo "a blob: URL reached the bundle; CSP will block it in prod"; exit 1; } + if grep -rEo "new (Shared)?Worker\([^)]{0,80}" dist/assets/*.js \ + | grep -E "blob:|createObjectURL"; then + echo "a worker is constructed from a blob URL; production CSP blocks it silently" + exit 1 + fi + echo "ok — no blob-backed worker construction in the bundle" python: runs-on: ubuntu-latest diff --git a/envs/build_manifest.py b/envs/build_manifest.py index 707f4af..8aeb7eb 100644 --- a/envs/build_manifest.py +++ b/envs/build_manifest.py @@ -17,6 +17,7 @@ ARMS = { "base-off": ("Out of the box", "recorded"), "base-on": ("Allowed to think", "intervened"), "solver": ("Best-known play", "generated"), + "cautious": ("Never wastes a guess", "generated"), } # What was done to the run, for arms that had something done to them. Required @@ -26,7 +27,7 @@ INTERVENTIONS = { "base-on": "Same model, same seeds, sampled with thinking enabled. No training, no fine-tuning.", } -ORDER = ["base-off", "base-on", "solver"] +ORDER = ["base-off", "base-on", "solver", "cautious"] def main() -> int: diff --git a/envs/capture.py b/envs/capture.py index 94070ef..4b1539f 100644 --- a/envs/capture.py +++ b/envs/capture.py @@ -18,6 +18,7 @@ from __future__ import annotations import argparse import json +import math import os import sys import time @@ -47,6 +48,7 @@ ARMS = { "base-off": {"label": "Out of the box", "thinking": False}, "base-on": {"label": "Allowed to think", "thinking": True}, "solver": {"label": "Best-known play", "thinking": None}, + "cautious": {"label": "Never wastes a guess", "thinking": None}, } @@ -105,6 +107,53 @@ def call_model(messages: list[dict], thinking: bool) -> dict: } +def cautious_turn(game: Game) -> dict: + """Only ever guesses a word that could still be the answer. + + This arm exists because the reward's counterweight is a claim about a + trade-off, and a trade-off with only one policy on the board is a claim + nobody can check. It never spends a turn on a word that cannot win, so it + takes `consistency` outright — and it pays for that in turns, because it + cannot buy information with a guess that has no chance. Weight the reward + one way and it beats the entropy solver; weight it the other and it loses. + That is the whole argument, made with two recorded runs instead of a claim. + """ + pool = answers() + alive = [w for w in S.consistent_candidates(game.history) if w not in {g for g, _ in game.history}] + if not alive: + guess = "tares" + else: + # Most informative CANDIDATE, not first alphabetically. Same objective as + # the solver, restricted action set — which is the honest contrast. A + # policy that opens on whatever sorts first looks incompetent rather than + # cautious, and would make the trade-off it exists to demonstrate look + # like a straw man. + index = np.array([pool.index(w) for w in alive]) + best, best_bits = alive[0], -1.0 + for word in alive: + counts: dict[int, int] = {} + row = S.pattern_matrix()[pool.index(word)] + for j in index: + code = int(row[j]) + counts[code] = counts.get(code, 0) + 1 + total = len(alive) + bits = -sum((c / total) * math.log2(c / total) for c in counts.values()) + if bits > best_bits: + best, best_bits = word, bits + guess = best + return { + "reply": f"[{guess}]", + "reasoning": None, + "call": { + "promptTokens": None, + "completionTokens": None, + "reasoningTokens": None, + "durationMs": None, + "finishReason": "generated", + }, + } + + def solver_turn(game: Game) -> dict: """The reference player, recorded in the same shape as a model turn. @@ -145,6 +194,8 @@ def capture(arm: str, seed: int) -> dict: while not game.over and len(turns) < MAX_GUESSES * 2: if arm == "solver": result = solver_turn(game) + elif arm == "cautious": + result = cautious_turn(game) else: result = call_model(messages, bool(config["thinking"])) @@ -183,7 +234,7 @@ def capture(arm: str, seed: int) -> dict: return { "runId": f"{arm}-s{seed}", "seed": seed, - "model": MODEL if arm != "solver" else "entropy-solver", + "model": {"solver": "entropy-solver", "cautious": "candidate-only-solver"}.get(arm, MODEL), "capturedAt": time.strftime("%Y-%m-%d"), "rewards": score(episode), "metrics": metrics(episode), diff --git a/src/components/demo/DemoShell.tsx b/src/components/demo/DemoShell.tsx index d008fe3..e160da1 100644 --- a/src/components/demo/DemoShell.tsx +++ b/src/components/demo/DemoShell.tsx @@ -19,6 +19,7 @@ import { EnvAnatomy } from './EnvAnatomy'; import { LimitsCallout } from './LimitsCallout'; import { MetricMover } from './MetricMover'; import { ModelCallPanel } from './ModelCallPanel'; +import { PlayYourself } from './PlayYourself'; import { ProvenanceCard } from './ProvenanceCard'; import { ReasoningDrawer } from './ReasoningDrawer'; import { ReasoningPanel } from './ReasoningPanel'; @@ -351,6 +352,19 @@ function DemoBody({ bundle }: { bundle: DemoBundle }) { case 'split-play': return (
+ {demo.interactive ? ( +
+ +
+

What the model did

+

+ Same hidden answer, same rules, same budget. Its attempt is + below, replayed at the speed it actually happened. +

+
+
+ ) : null} + {runs.length > 1 ? ( void; }) { + // Two axes, not one list. With four arms over eight seeds a flat control is + // thirty buttons carrying four distinct labels, which reads as a bug. Arms + // are grouped by `label` because that is what an arm IS in the manifest — + // the shell has no other notion of one, and inventing a field for it would + // put demo-specific structure into the contract. + const arms = useMemo(() => { + const byLabel = new Map(); + for (const run of runs) { + const list = byLabel.get(run.label); + if (list) list.push(run); + else byLabel.set(run.label, [run]); + } + return [...byLabel.entries()].map(([label, group]) => ({ label, runs: group })); + }, [runs]); + + const active = runs.find((r) => r.id === activeId) ?? runs[0]; + if (!active) return null; + const activeArm = arms.find((a) => a.label === active.label) ?? arms[0]; + if (!activeArm) return null; + + const pickArm = (label: string) => { + const arm = arms.find((a) => a.label === label); + if (!arm) return; + // Hold the seed across an arm change where the arm has it. Comparing two + // agents means comparing them on the SAME hidden word; silently jumping to + // a different seed would make the comparison meaningless while looking fine. + const sameSeed = arm.runs.find((r) => r.seed === active.seed); + onSelect((sameSeed ?? arm.runs[0])!.id); + }; + return ( - ({ - value: run.id, - label: run.label, - ...(run.intervention ? { title: run.intervention } : {}), - }))} - value={activeId} - onChange={onSelect} - className="border border-border p-1" - optionClassName="tap px-3 text-sm" - /> +
+ { + const intervention = arm.runs.find((r) => r.intervention)?.intervention; + return { + value: arm.label, + label: arm.label, + ...(intervention ? { title: intervention } : {}), + }; + })} + value={activeArm.label} + onChange={pickArm} + className="border border-border p-1" + optionClassName="tap px-3 text-sm" + /> + {activeArm.runs.length > 1 ? ( + ({ + value: run.id, + label: `#${run.seed}`, + }))} + value={active.id} + onChange={onSelect} + className="border border-border p-1" + optionClassName="tap px-2.5 text-xs nums" + /> + ) : null} +
); } + /** * The loading state. Shaped like the page it becomes, and with no spinner: a * spinner here would imply a live model call, which is the one thing the whole diff --git a/src/components/demo/PlayYourself.tsx b/src/components/demo/PlayYourself.tsx new file mode 100644 index 0000000..4c2cc1d --- /dev/null +++ b/src/components/demo/PlayYourself.tsx @@ -0,0 +1,75 @@ +import { useEffect, useMemo, useState } from 'react'; +import { RotateCcw } from 'lucide-react'; + +import type { DemoModule } from '@/lib/demo-kit'; +import { cn } from '@/lib/utils'; + +import * as st from '@/content/styles'; + +/** + * The visitor's own attempt, on the same seed as the run beside it. + * + * Generic over the demo: it knows only `interactive.init`, `interactive.Controls` + * and `Surface` from the contract, so a demo that ships an interactive mode gets + * this for free and one that does not renders nothing rather than a broken pane. + * + * The seed is the shared coordinate. Both boards derive from it, which is what + * makes "you and the model got the same word" a fact about the page rather than + * a sentence on it. + */ +export function PlayYourself({ + demo, + seed, + className, +}: { + demo: DemoModule; + seed: number; + className?: string; +}) { + const interactive = demo.interactive; + const Surface = demo.Surface; + + // `init` is pure and seeded, so remounting on a seed change is the whole of + // "start a new game" — there is no other state to reset. + const initial = useMemo( + () => (interactive ? interactive.init(seed) : null), + [interactive, seed], + ); + const [state, setState] = useState(initial); + const [generation, setGeneration] = useState(0); + + useEffect(() => setState(initial), [initial]); + + if (!interactive || state === null) return null; + const Controls = interactive.Controls; + + return ( +
+
+

Your attempt

+ +
+ +
+ +
+ + {/* Remounting on `generation` clears any state the controls hold of their + own — a half-typed word survives a reset otherwise, and the board and + the keyboard then disagree about what is on the current row. */} + +
+ ); +} + +export default PlayYourself; diff --git a/src/demos/wordle/keyboard.tsx b/src/demos/wordle/keyboard.tsx index d8fed50..97bf093 100644 --- a/src/demos/wordle/keyboard.tsx +++ b/src/demos/wordle/keyboard.tsx @@ -3,7 +3,8 @@ import { useEffect, useMemo } from 'react'; import { cn } from '@/lib/utils'; import { play, type BoardState } from './engine'; -import { allowedNow, primeGuessList } from './words'; +import { useSolver } from './useSolver'; +import { allowedNow, ANSWERS, primeGuessList } from './words'; const ROWS = ['qwertyuiop', 'asdfghjkl', 'zxcvbnm']; @@ -37,6 +38,7 @@ export function Keyboard({ }) { const states = useMemo(() => letterStates(state), [state]); const done = state.status !== 'playing'; + const solver = useSolver(ANSWERS); useEffect(() => { void primeGuessList(); @@ -78,8 +80,46 @@ export function Keyboard({ return () => window.removeEventListener('keydown', handler); }); + const hint = solver.state; + return (
+ {/* The solver, live, on whatever word you picked — not a recording. This + is the difference between a demo and a video: it answers for boards no + rollout on this site ever covered. */} + {hint.status !== 'unavailable' && !done ? ( +
+ + + {hint.status === 'ready' ? ( + <> + + {hint.suggestion.guess} + {' '} + — {hint.suggestion.bits.toFixed(2)} bits from{' '} + {hint.suggestion.candidatesBefore.toLocaleString('en-US')}{' '} + remaining + {hint.suggestion.viable ? null : ( + <> + {' '} + ·{' '} + + cannot win — it is buying information + + + )} + + ) : null} + +
+ ) : null} {ROWS.map((row, index) => (
{index === 2 ? ( diff --git a/src/demos/wordle/useSolver.ts b/src/demos/wordle/useSolver.ts new file mode 100644 index 0000000..6caf7dd --- /dev/null +++ b/src/demos/wordle/useSolver.ts @@ -0,0 +1,90 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; + +import type { Pattern } from './engine'; +import type { SolverRequest, SolverResponse } from './solver.worker'; + +/** + * The entropy solver, on a worker, with a main-thread fallback. + * + * This is what lets the page answer for ANY word the visitor picks rather than + * only the handful the recordings cover — the difference between a demo and a + * video. The opening scan is roughly 4,600 x 4,600 pattern computations, which + * visibly freezes a phone if it runs on the main thread at the exact moment + * somebody first touches the board. + * + * `new URL(..., import.meta.url)` gives a same-origin module in the build. + * Never Vite's `?worker&inline`: that yields a blob: URL, and production CSP has + * no `worker-src`, so it falls back to `default-src 'self'` and the worker is + * blocked with no console error — in production only. CI greps the bundle for + * exactly that construction. + */ +export interface Suggestion { + guess: string; + bits: number; + viable: boolean; + candidatesBefore: number; +} + +export type SolverState = + | { status: 'idle' } + | { status: 'thinking' } + | { status: 'ready'; suggestion: Suggestion } + | { status: 'unavailable' }; + +export function useSolver(pool: readonly string[]) { + const workerRef = useRef(null); + const nextId = useRef(0); + const pending = useRef(null); + const [state, setState] = useState({ status: 'idle' }); + + useEffect(() => { + let worker: Worker | null = null; + try { + worker = new Worker(new URL('./solver.worker.ts', import.meta.url), { type: 'module' }); + worker.onmessage = (event: MessageEvent) => { + // Ignore everything but the most recent request, or a slow first scan + // lands after a later, faster one and the hint goes backwards. + if (event.data.id !== pending.current) return; + const { guess, bits, viable, candidatesBefore } = event.data; + setState({ status: 'ready', suggestion: { guess, bits, viable, candidatesBefore } }); + }; + worker.onerror = () => setState({ status: 'unavailable' }); + workerRef.current = worker; + } catch { + // Some embedded browsers refuse module workers outright. The button is + // hidden rather than shown broken. + setState({ status: 'unavailable' }); + } + return () => { + worker?.terminate(); + workerRef.current = null; + }; + }, []); + + const ask = useCallback( + (history: readonly { guess: string; pattern: Pattern }[]) => { + const worker = workerRef.current; + if (!worker) { + setState({ status: 'unavailable' }); + return; + } + const id = (nextId.current += 1); + pending.current = id; + setState({ status: 'thinking' }); + const request: SolverRequest = { + id, + pool: pool as string[], + history: history.map((h) => ({ guess: h.guess, pattern: h.pattern })), + }; + worker.postMessage(request); + }, + [pool], + ); + + const reset = useCallback(() => { + pending.current = null; + setState({ status: 'idle' }); + }, []); + + return { state, ask, reset }; +}