karti-ai
2dfa96939e
The first environment shipped through .claude/workflows/new-environment.js: specification, three adversarial reviews (all 'fixable', none fatal), the Python environment, the TypeScript port, captured rollouts, and the demo page. Eleven agents, no errors. The proof that the platform scales is one line long. Alert Triage has a completely different shape from Word Five — JSON actions, priced lookups, an analyst screen instead of a grid — and the only change under src/components/demo/ is a comment edit, because the isolation lint refused the word "wordle" there. Zero shell code changed. 415 contract checks now pass against two demos, up from 206 against one. The environment is honest by construction. Every alert is synthetic, generated from the seed, and the banner saying so sits inside the board surface. Two of the eleven scenario templates are hidden-suspicious: generated by the same code as their benign twin with the signal overlaid only in lookup data, so the free screen is identically distributed and a screen-only policy STRUCTURALLY cannot tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden seeds. That is the counterweight made real rather than asserted. Twelve policies, thirteen ladder assertions, a genuine three-way trade: fast 0.846 wins hours (0.85), misses every hidden case targeted 0.894 wins the shipped total thorough 0.820 wins evidence (1.00), spends 2.9 hours None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at delta 0, and conformance gated on world + scorer + protocol so the browser shows the same alert for ?seed= that Python generated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
89 lines
3.4 KiB
Python
89 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Build public/traces/manifest.json from the captured fixtures.
|
|
|
|
The manifest is what the browser reads to know which runs exist. It is
|
|
generated rather than hand-written so a fixture can never be referenced without
|
|
existing, or exist without being referenced.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
TRACES = Path(__file__).parent.parent / "public" / "traces"
|
|
|
|
# Arm ids are shared across tasksets where they mean the same thing (the two
|
|
# model arms); the generated arms are per-taskset and simply do not collide.
|
|
ARMS = {
|
|
"base-off": ("Out of the box", "recorded"),
|
|
"base-on": ("Allowed to think", "intervened"),
|
|
# wordle
|
|
"solver": ("Best-known play", "generated"),
|
|
"cautious": ("Never wastes a guess", "generated"),
|
|
# alert-triage: the shipped scripted analysts from alert_triage/policies.py
|
|
"fast": ("Reads the screen", "generated"),
|
|
"targeted": ("Checks the hidden tells", "generated"),
|
|
"thorough": ("Runs the full procedure", "generated"),
|
|
}
|
|
|
|
# What was done to the run, for arms that had something done to them. Required
|
|
# by the contract on any `intervened` run so that a sampling change can never be
|
|
# presented as a training result by omitting to mention it.
|
|
INTERVENTIONS = {
|
|
"base-on": "Same model, same seeds, sampled with thinking enabled. No training, no fine-tuning.",
|
|
}
|
|
|
|
ORDER = ["base-off", "base-on", "solver", "cautious", "fast", "targeted", "thorough"]
|
|
|
|
|
|
def main() -> int:
|
|
manifest: dict[str, list[dict]] = {}
|
|
|
|
for demo_dir in sorted(p for p in TRACES.iterdir() if p.is_dir()):
|
|
runs = []
|
|
for path in sorted(demo_dir.glob("*.json")):
|
|
data = json.loads(path.read_text())
|
|
arm = data["runId"].rsplit("-s", 1)[0]
|
|
label, kind = ARMS.get(arm, (arm, "recorded"))
|
|
run = {
|
|
"id": data["runId"],
|
|
"label": label,
|
|
"path": f"/traces/{demo_dir.name}/{path.name}",
|
|
"kind": kind,
|
|
"model": data["model"],
|
|
"capturedAt": data["capturedAt"],
|
|
"seed": data["seed"],
|
|
}
|
|
if arm in INTERVENTIONS:
|
|
run["intervention"] = INTERVENTIONS[arm]
|
|
runs.append(run)
|
|
|
|
runs.sort(key=lambda r: (ORDER.index(r["id"].rsplit("-s", 1)[0])
|
|
if r["id"].rsplit("-s", 1)[0] in ORDER else 99,
|
|
r["seed"]))
|
|
if runs:
|
|
manifest[demo_dir.name] = runs
|
|
|
|
out = TRACES / "manifest.json"
|
|
out.write_text(json.dumps(manifest, indent=2) + "\n")
|
|
|
|
for slug, runs in manifest.items():
|
|
by_arm: dict[str, list[dict]] = {}
|
|
for r in runs:
|
|
by_arm.setdefault(r["id"].rsplit("-s", 1)[0], []).append(r)
|
|
print(f"{slug}: {len(runs)} runs")
|
|
for arm, group in by_arm.items():
|
|
outcomes: dict[str, int] = {}
|
|
for r in group:
|
|
data = json.loads((TRACES.parent / r["path"].lstrip("/")).read_text())
|
|
outcomes[data["outcome"]] = outcomes.get(data["outcome"], 0) + 1
|
|
solved = outcomes.get("solved", 0)
|
|
rest = ", ".join(f"{k} {v}" for k, v in sorted(outcomes.items()) if k != "solved")
|
|
print(f" {arm:<10} {len(group)} runs, solved {solved}/{len(group)}" + (f" ({rest})" if rest else ""))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|