Alert Triage: environment #2, built end to end by the pipeline

The first environment shipped through .claude/workflows/new-environment.js:
specification, three adversarial reviews (all 'fixable', none fatal), the
Python environment, the TypeScript port, captured rollouts, and the demo page.
Eleven agents, no errors.

The proof that the platform scales is one line long. Alert Triage has a
completely different shape from Word Five — JSON actions, priced lookups, an
analyst screen instead of a grid — and the only change under
src/components/demo/ is a comment edit, because the isolation lint refused the
word "wordle" there. Zero shell code changed. 415 contract checks now pass
against two demos, up from 206 against one.

The environment is honest by construction. Every alert is synthetic, generated
from the seed, and the banner saying so sits inside the board surface. Two of
the eleven scenario templates are hidden-suspicious: generated by the same code
as their benign twin with the signal overlaid only in lookup data, so the free
screen is identically distributed and a screen-only policy STRUCTURALLY cannot
tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden
seeds. That is the counterweight made real rather than asserted.

Twelve policies, thirteen ladder assertions, a genuine three-way trade:

  fast      0.846   wins hours (0.85), misses every hidden case
  targeted  0.894   wins the shipped total
  thorough  0.820   wins evidence (1.00), spends 2.9 hours

None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at
delta 0, and conformance gated on world + scorer + protocol so the browser shows
the same alert for ?seed= that Python generated.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB

envs/verify_fixtures.py

@@ -16,17 +16,20 @@ import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent / "wordle_five"))
+sys.path.insert(0, str(Path(__file__).parent / "alert_triage"))
 
+from alert_triage import taskset as AT  # noqa: E402
+from alert_triage.generator import is_held_out  # noqa: E402
 from wordle_five import solver as S  # noqa: E402
 from wordle_five.engine import Game  # noqa: E402
 from wordle_five.protocol import parse_guess  # noqa: E402
 from wordle_five.reward import Episode, score  # noqa: E402
 
-TRACES = Path(__file__).parent.parent / "public" / "traces" / "wordle"
+TRACES = Path(__file__).parent.parent / "public" / "traces"
 TOLERANCE = 1e-9
 
 
-def replay(fixture: dict) -> dict[str, float]:
+def replay_wordle(fixture: dict) -> dict[str, float]:
     game = Game(seed=fixture["seed"])
     if game.answer != fixture["answer"]:
         raise AssertionError(
@@ -49,27 +52,61 @@ def replay(fixture: dict) -> dict[str, float]:
     return score(episode)
 
 
+def replay_alert_triage(fixture: dict) -> dict[str, float | None]:
+    """The world comes from the seed, the score from the reply strings. Nothing
+    else in the fixture is trusted — the same rule the browser's verify uses."""
+    seed = fixture["seed"]
+    if is_held_out(seed):
+        raise AssertionError(f"seed {seed} is in the held-out bucket and must never be captured")
+    played = AT.replay(seed, [turn["reply"] for turn in fixture["turns"]])
+    if played["outcome"] != fixture["outcome"]:
+        raise AssertionError(f"replay gives outcome {played['outcome']!r}, fixture claims {fixture['outcome']!r}")
+    return played["rewards"]
+
+
+REPLAY = {"wordle": replay_wordle, "alert-triage": replay_alert_triage}
+
+
+def _delta(recomputed, recorded) -> float:
+    """None is "not scored". It matches only None; against a number it is a mismatch, not a zero."""
+    if recomputed is None or recorded is None:
+        return 0.0 if recomputed is recorded else float("inf")
+    return abs(recomputed - recorded)
+
+
 def main() -> int:
-    fixtures = sorted(TRACES.glob("*.json"))
-    fixtures = [f for f in fixtures if f.name != "manifest.json"]
+    fixtures = [
+        f for demo in sorted(p for p in TRACES.iterdir() if p.is_dir())
+        for f in sorted(demo.glob("*.json"))
+    ]
     if not fixtures:
         print("no fixtures found — nothing to verify")
         return 0
 
     failures = 0
     for path in fixtures:
+        demo = path.parent.name
+        if demo not in REPLAY:
+            failures += 1
+            print(f"{path.parent.name}/{path.name:<20}  NO REPLAYER for demo '{demo}'")
+            continue
         data = json.loads(path.read_text())
-        recomputed = replay(data)
+        try:
+            recomputed = REPLAY[demo](data)
+        except AssertionError as exc:
+            failures += 1
+            print(f"{path.parent.name}/{path.name:<20}  MISMATCH  {exc}")
+            continue
         recorded = data["rewards"]
-        deltas = {k: abs(recomputed[k] - recorded[k]) for k in recorded}
+        deltas = {k: _delta(recomputed[k], recorded[k]) for k in recorded}
         worst = max(deltas.values())
         status = "ok" if worst <= TOLERANCE else "MISMATCH"
         if worst > TOLERANCE:
             failures += 1
             culprit = max(deltas, key=deltas.get)
-            print(f"{path.name:>20}  {status}  worst delta {worst:.3g} on '{culprit}'")
+            print(f"{path.parent.name}/{path.name:<20}  {status}  worst delta {worst:.3g} on '{culprit}'")
         else:
-            print(f"{path.name:>20}  {status}  delta {worst:.1e}")
+            print(f"{path.parent.name}/{path.name:<20}  {status}  delta {worst:.1e}")
 
     print(f"\n{len(fixtures)} fixtures, {failures} mismatched")
     return 1 if failures else 0