karti-ai
331b46b114
Three things the build was quietly missing. The interactive board was never mounted. The contract has `interactive.init` and `interactive.Controls`, the demo implemented both, and the shell's `split-play` beat rendered only the replay — so the beat titled "you and the model get the same word" showed one board. PlayYourself now renders the visitor's attempt from the same seed as the run beside it, generically: it knows only the contract, so any demo shipping an interactive mode gets it and one that does not renders nothing rather than an empty pane. solver.worker.ts was dead code — nothing constructed it, which is how CI caught it: `new Worker(` appeared nowhere in the bundle. It is wired now behind "what would the best player guess?", and it answers in 92ms from a real worker on boards no recording covers. That is the difference between a demo and a video. It also surfaces the moment the solver picks a word that CANNOT win, which is the counterweight visible in one line instead of explained in a paragraph. The CI check that found it was itself wrong: it grepped every bundled file for `blob:`, which React's own code contains in a scheme check, so it failed on a risk that was not present. It now greps for worker construction from a blob, which is the thing production CSP actually blocks in silence. And the run switcher was thirty buttons carrying four distinct labels. Split into arm and seed, holding the seed across an arm change — comparing two agents means comparing them on the same hidden word, and silently jumping seeds would break that while looking fine. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
80 lines
2.8 KiB
Python
80 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Build public/traces/manifest.json from the captured fixtures.
|
|
|
|
The manifest is what the browser reads to know which runs exist. It is
|
|
generated rather than hand-written so a fixture can never be referenced without
|
|
existing, or exist without being referenced.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
TRACES = Path(__file__).parent.parent / "public" / "traces"
|
|
|
|
ARMS = {
|
|
"base-off": ("Out of the box", "recorded"),
|
|
"base-on": ("Allowed to think", "intervened"),
|
|
"solver": ("Best-known play", "generated"),
|
|
"cautious": ("Never wastes a guess", "generated"),
|
|
}
|
|
|
|
# What was done to the run, for arms that had something done to them. Required
|
|
# by the contract on any `intervened` run so that a sampling change can never be
|
|
# presented as a training result by omitting to mention it.
|
|
INTERVENTIONS = {
|
|
"base-on": "Same model, same seeds, sampled with thinking enabled. No training, no fine-tuning.",
|
|
}
|
|
|
|
ORDER = ["base-off", "base-on", "solver", "cautious"]
|
|
|
|
|
|
def main() -> int:
|
|
manifest: dict[str, list[dict]] = {}
|
|
|
|
for demo_dir in sorted(p for p in TRACES.iterdir() if p.is_dir()):
|
|
runs = []
|
|
for path in sorted(demo_dir.glob("*.json")):
|
|
data = json.loads(path.read_text())
|
|
arm = data["runId"].rsplit("-s", 1)[0]
|
|
label, kind = ARMS.get(arm, (arm, "recorded"))
|
|
run = {
|
|
"id": data["runId"],
|
|
"label": label,
|
|
"path": f"/traces/{demo_dir.name}/{path.name}",
|
|
"kind": kind,
|
|
"model": data["model"],
|
|
"capturedAt": data["capturedAt"],
|
|
"seed": data["seed"],
|
|
}
|
|
if arm in INTERVENTIONS:
|
|
run["intervention"] = INTERVENTIONS[arm]
|
|
runs.append(run)
|
|
|
|
runs.sort(key=lambda r: (ORDER.index(r["id"].rsplit("-s", 1)[0])
|
|
if r["id"].rsplit("-s", 1)[0] in ORDER else 99,
|
|
r["seed"]))
|
|
if runs:
|
|
manifest[demo_dir.name] = runs
|
|
|
|
out = TRACES / "manifest.json"
|
|
out.write_text(json.dumps(manifest, indent=2) + "\n")
|
|
|
|
for slug, runs in manifest.items():
|
|
by_arm: dict[str, list[dict]] = {}
|
|
for r in runs:
|
|
by_arm.setdefault(r["id"].rsplit("-s", 1)[0], []).append(r)
|
|
print(f"{slug}: {len(runs)} runs")
|
|
for arm, group in by_arm.items():
|
|
solved = 0
|
|
for r in group:
|
|
data = json.loads((TRACES.parent / r["path"].lstrip("/")).read_text())
|
|
solved += 1 if data["outcome"] == "solved" else 0
|
|
print(f" {arm:<10} {len(group)} runs, solved {solved}/{len(group)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|