Alert Triage: environment #2, built end to end by the pipeline

The first environment shipped through .claude/workflows/new-environment.js:
specification, three adversarial reviews (all 'fixable', none fatal), the
Python environment, the TypeScript port, captured rollouts, and the demo page.
Eleven agents, no errors.

The proof that the platform scales is one line long. Alert Triage has a
completely different shape from Word Five — JSON actions, priced lookups, an
analyst screen instead of a grid — and the only change under
src/components/demo/ is a comment edit, because the isolation lint refused the
word "wordle" there. Zero shell code changed. 415 contract checks now pass
against two demos, up from 206 against one.

The environment is honest by construction. Every alert is synthetic, generated
from the seed, and the banner saying so sits inside the board surface. Two of
the eleven scenario templates are hidden-suspicious: generated by the same code
as their benign twin with the signal overlaid only in lookup data, so the free
screen is identically distributed and a screen-only policy STRUCTURALLY cannot
tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden
seeds. That is the counterweight made real rather than asserted.

Twelve policies, thirteen ladder assertions, a genuine three-way trade:

  fast      0.846   wins hours (0.85), misses every hidden case
  targeted  0.894   wins the shipped total
  thorough  0.820   wins evidence (1.00), spends 2.9 hours

None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at
delta 0, and conformance gated on world + scorer + protocol so the browser shows
the same alert for ?seed= that Python generated.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB

envs/capture.py

@@ -12,6 +12,13 @@ number it shows from the recorded moves, and says so. See verify.ts.
 Usage:
   uv run python envs/capture.py --arm base-off --seeds 0-7
   uv run python envs/capture.py --arm solver   --seeds 0-7   # no model needed
+  uv run python envs/capture.py --taskset alert-triage --arm base-on --seeds 1-7,9
+  uv run python envs/capture.py --taskset alert-triage --arm targeted --seeds 1-7,9
+
+spark-1 serves one model, single-stream: run the model arms one after another,
+never concurrently. A thinking arm on alert-triage takes minutes per seed —
+run it in the background with stdout redirected to a file and poll the
+fixtures being written rather than the log.
 """
 
 from __future__ import annotations
@@ -27,9 +34,14 @@ import urllib.request
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent / "wordle_five"))
+sys.path.insert(0, str(Path(__file__).parent / "alert_triage"))
 
 import numpy as np  # noqa: E402
 
+from alert_triage import policies as AT  # noqa: E402
+from alert_triage import taskset as AT_taskset  # noqa: E402
+from alert_triage.generator import is_held_out, world_for_seed  # noqa: E402
+
 from wordle_five import solver as S  # noqa: E402
 from wordle_five.engine import MAX_GUESSES, Game, answers  # noqa: E402
 from wordle_five.protocol import (  # noqa: E402
@@ -40,7 +52,8 @@ from wordle_five.protocol import (  # noqa: E402
 )
 from wordle_five.reward import Episode, metrics, score  # noqa: E402
 
-OUT = Path(__file__).parent.parent / "public" / "traces" / "wordle"
+TRACES = Path(__file__).parent.parent / "public" / "traces"
+OUT = TRACES / "wordle"
 ENDPOINT = os.environ.get("PIG_DEMO_INFERENCE", "http://100.127.247.67:8001/v1/chat/completions")
 MODEL = os.environ.get("PIG_DEMO_MODEL", "brain-qwen38-dspark")
 
@@ -52,7 +65,7 @@ ARMS = {
 }
 
 
-def call_model(messages: list[dict], thinking: bool) -> dict:
+def call_model(messages: list[dict], thinking: bool, max_tokens: int | None = None, timeout: int = 300) -> dict:
     """One completion. Returns reply, reasoning and the real call metrics.
 
     Never raises on an upstream failure — a dropped call becomes a turn with a
@@ -64,7 +77,7 @@ def call_model(messages: list[dict], thinking: bool) -> dict:
         "model": MODEL,
         "messages": messages,
         "temperature": 0.7,
-        "max_tokens": 2048 if thinking else 512,
+        "max_tokens": max_tokens or (2048 if thinking else 512),
         "chat_template_kwargs": {"enable_thinking": bool(thinking)},
     }
     request = urllib.request.Request(
@@ -74,7 +87,7 @@ def call_model(messages: list[dict], thinking: bool) -> dict:
     )
     started = time.time()
     try:
-        with urllib.request.urlopen(request, timeout=300) as response:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
             payload = json.loads(response.read())
     except (urllib.error.URLError, TimeoutError, OSError) as exc:
         return {
@@ -245,33 +258,158 @@ def capture(arm: str, seed: int) -> dict:
     }
 
 
+# ------------------------------------------------------------ alert-triage --
+#
+# The same fixture shape as wordle — runId, seed, model, capturedAt, rewards,
+# metrics, truncated, outcome, turns[{reply, reasoning, call, info}] — driven
+# through `alert_triage.taskset.play_episode`, the loop the probe and the
+# tests share. The browser regenerates the world from the seed and replays the
+# reply strings; nothing else in the fixture is trusted by the page.
+
+AT_ARMS = {
+    "base-off": {"label": "Out of the box", "thinking": False, "max_tokens": 1024, "timeout": 300},
+    "base-on": {"label": "Allowed to think", "thinking": True, "max_tokens": 4096, "timeout": 900},
+    "fast": {"label": "Reads the screen", "thinking": None, "policy": "fast"},
+    "targeted": {"label": "Checks the hidden tells", "thinking": None, "policy": "targeted"},
+    "thorough": {"label": "Runs the full procedure", "thinking": None, "policy": "thorough"},
+}
+AT_MODEL_NAME = {"fast": "fast-analyst", "targeted": "targeted-analyst", "thorough": "thorough-analyst"}
+
+# A thinking budget the model exhausts is recorded as finishReason "length"
+# with whatever content survived (usually none, which the engine rejects).
+# That is a thing the model did under the budget it was given, not a capture
+# error, and the fixture says so rather than retrying until it looks better.
+
+
+def _messages_from(prompt: str, transcript: list[dict]) -> list[dict]:
+    """The chat a model sees: the system prompt, the screen, then each reply and what it got back."""
+    messages = [{"role": "system", "content": prompt}, {"role": "user", "content": transcript[0]["observation"]}]
+    for entry in transcript[1:]:
+        messages.append({"role": "assistant", "content": entry["reply"] or ""})
+        messages.append({"role": "user", "content": entry["observation"]})
+    return messages
+
+
+def _generated_call() -> dict:
+    return {"promptTokens": None, "completionTokens": None, "reasoningTokens": None, "durationMs": None, "finishReason": "generated"}
+
+
+def capture_alert_triage(arm: str, seed: int) -> dict:
+    config = AT_ARMS[arm]
+    calls: list[dict] = []
+
+    if config["thinking"] is None:
+        policy = AT.POLICIES[config["policy"]]
+
+        def respond(prompt: str, transcript: list[dict], view: dict) -> str | None:
+            reply = policy(view)
+            calls.append({"reply": reply, "reasoning": None, "call": _generated_call()})
+            return reply
+    else:
+        def respond(prompt: str, transcript: list[dict], view: dict) -> str | None:
+            result = call_model(_messages_from(prompt, transcript), bool(config["thinking"]),
+                                max_tokens=config["max_tokens"], timeout=config["timeout"])
+            calls.append(result)
+            return result["reply"]
+
+    world = world_for_seed(seed)
+    played = AT_taskset.play_episode(seed, respond, world)
+    steps = played["transcript"][1:]
+    assert len(steps) == len(calls), "one model call per engine step"
+
+    turns = [
+        {
+            **call,
+            "info": {"action": step["action"], "rejection": step["rejection"], "observation": step["observation"]},
+        }
+        for call, step in zip(calls, steps)
+    ]
+    return {
+        "runId": f"{arm}-s{seed}",
+        "seed": seed,
+        "model": AT_MODEL_NAME.get(arm, MODEL),
+        "capturedAt": time.strftime("%Y-%m-%d"),
+        "rewards": played["rewards"],
+        "metrics": played["metrics"],
+        "truncated": played["truncated"],
+        "outcome": played["outcome"],
+        "info": {**played["info"], "tier": world["tier"], "screen": played["transcript"][0]["observation"]},
+        "turns": turns,
+    }
+
+
+def _at_summary(episode: dict) -> str:
+    r = episode["rewards"]
+    fmt = lambda v: "  -- " if v is None else f"{v:.2f}"  # noqa: E731
+    actions = []
+    for t in episode["turns"]:
+        a = t["info"]["action"]
+        if a is None:
+            actions.append("REJ")
+        elif a["action"] == "lookup":
+            actions.append(a.get("month") or a.get("id") or a["what"])
+        else:
+            actions.append(a["action"].upper())
+    return (
+        f"{episode['info']['tier']:<8}{episode['info']['template']:<3} {episode['outcome']:<8}"
+        f" caught={fmt(r['caught'])} hours={fmt(r['hours'])} evid={fmt(r['evidence'])}"
+        f"  {episode['metrics']['hours_spent']:.2f}h  {actions}"
+    )
+
+
+TASKSETS = {
+    "wordle-five": {"arms": ARMS, "out": TRACES / "wordle", "capture": capture},
+    "alert-triage": {"arms": AT_ARMS, "out": TRACES / "alert-triage", "capture": capture_alert_triage},
+}
+
+
 def parse_seeds(spec: str) -> list[int]:
-    if "-" in spec:
-        lo, hi = spec.split("-")
-        return list(range(int(lo), int(hi) + 1))
-    return [int(s) for s in spec.split(",")]
+    """"0-7", "1,3", or a mix: "1-7,9"."""
+    seeds: list[int] = []
+    for part in spec.split(","):
+        if "-" in part:
+            lo, hi = part.split("-")
+            seeds.extend(range(int(lo), int(hi) + 1))
+        else:
+            seeds.append(int(part))
+    return seeds
 
 
 def main() -> int:
     parser = argparse.ArgumentParser()
-    parser.add_argument("--arm", required=True, choices=sorted(ARMS))
+    parser.add_argument("--taskset", default="wordle-five", choices=sorted(TASKSETS))
+    parser.add_argument("--arm", required=True)
     parser.add_argument("--seeds", default="0-7")
     args = parser.parse_args()
 
-    OUT.mkdir(parents=True, exist_ok=True)
+    taskset = TASKSETS[args.taskset]
+    if args.arm not in taskset["arms"]:
+        parser.error(f"--arm must be one of {sorted(taskset['arms'])} for {args.taskset}")
+
+    out: Path = taskset["out"]
+    out.mkdir(parents=True, exist_ok=True)
     for seed in parse_seeds(args.seeds):
+        if args.taskset == "alert-triage" and is_held_out(seed):
+            # The held-out bucket is never captured, probed or trained on. A
+            # fixture for one would put a held-out alert behind a permalink.
+            print(f"{args.arm}-s{seed}: seed {seed} is held out — skipped", flush=True)
+            continue
         started = time.time()
-        episode = capture(args.arm, seed)
-        path = OUT / f"{episode['runId']}.json"
+        episode = taskset["capture"](args.arm, seed)
+        path = out / f"{episode['runId']}.json"
         path.write_text(json.dumps(episode, indent=2) + "\n")
-        guesses = [t["info"]["guess"] for t in episode["turns"]]
-        print(
-            f"{episode['runId']:>16}  {episode['answer']}  {episode['outcome']:<7}"
-            f"  solved={episode['rewards']['solved']:.0f}"
-            f" econ={episode['rewards']['economy']:.2f}"
-            f" cons={episode['rewards']['consistency']:.2f}"
-            f"  {time.time()-started:5.1f}s  {guesses}"
-        )
+        if args.taskset == "wordle-five":
+            guesses = [t["info"]["guess"] for t in episode["turns"]]
+            print(
+                f"{episode['runId']:>16}  {episode['answer']}  {episode['outcome']:<7}"
+                f"  solved={episode['rewards']['solved']:.0f}"
+                f" econ={episode['rewards']['economy']:.2f}"
+                f" cons={episode['rewards']['consistency']:.2f}"
+                f"  {time.time()-started:5.1f}s  {guesses}",
+                flush=True,
+            )
+        else:
+            print(f"{episode['runId']:>16}  {_at_summary(episode)}  {time.time()-started:6.1f}s", flush=True)
     return 0