karti-ai
2dfa96939e
The first environment shipped through .claude/workflows/new-environment.js: specification, three adversarial reviews (all 'fixable', none fatal), the Python environment, the TypeScript port, captured rollouts, and the demo page. Eleven agents, no errors. The proof that the platform scales is one line long. Alert Triage has a completely different shape from Word Five — JSON actions, priced lookups, an analyst screen instead of a grid — and the only change under src/components/demo/ is a comment edit, because the isolation lint refused the word "wordle" there. Zero shell code changed. 415 contract checks now pass against two demos, up from 206 against one. The environment is honest by construction. Every alert is synthetic, generated from the seed, and the banner saying so sits inside the board surface. Two of the eleven scenario templates are hidden-suspicious: generated by the same code as their benign twin with the signal overlaid only in lookup data, so the free screen is identically distributed and a screen-only policy STRUCTURALLY cannot tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden seeds. That is the counterweight made real rather than asserted. Twelve policies, thirteen ladder assertions, a genuine three-way trade: fast 0.846 wins hours (0.85), misses every hidden case targeted 0.894 wins the shipped total thorough 0.820 wins evidence (1.00), spends 2.9 hours None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at delta 0, and conformance gated on world + scorer + protocol so the browser shows the same alert for ?seed= that Python generated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
85 lines
3.2 KiB
Python
85 lines
3.2 KiB
Python
"""The reference analysts are players, not oracles, and the reference exists everywhere."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import inspect
|
|
|
|
from alert_triage.engine import Engine
|
|
from alert_triage.generator import MAX_TURNS, canonical_json, generate, world_for_seed
|
|
from alert_triage.policies import POLICIES, SHIPPED, reference_for, run_policy
|
|
|
|
ALL = range(4096)
|
|
|
|
|
|
def test_every_policy_takes_only_the_view() -> None:
|
|
for name, policy in POLICIES.items():
|
|
params = list(inspect.signature(policy).parameters)
|
|
assert params == ["view"], name
|
|
|
|
|
|
def test_policies_never_mutate_the_world() -> None:
|
|
for seed in range(32):
|
|
world = world_for_seed(seed)
|
|
before = canonical_json(world)
|
|
for policy in POLICIES.values():
|
|
run_policy(policy, world)
|
|
assert canonical_json(world) == before
|
|
|
|
|
|
def test_shipped_policies_are_never_rejected() -> None:
|
|
for seed in range(256):
|
|
world = world_for_seed(seed)
|
|
for name in SHIPPED + ("thorough_wasteful",):
|
|
engine = run_policy(POLICIES[name], world)
|
|
assert engine.rejected == 0, (seed, name)
|
|
assert engine.disposition is not None, (seed, name)
|
|
|
|
|
|
def test_turn_budgets_are_as_documented() -> None:
|
|
for seed in range(128):
|
|
world = world_for_seed(seed)
|
|
assert run_policy(POLICIES["fast"], world).turns <= 2
|
|
assert run_policy(POLICIES["targeted"], world).turns <= 5
|
|
thorough = run_policy(POLICIES["thorough"], world).turns
|
|
assert thorough <= MAX_TURNS - 1
|
|
assert run_policy(POLICIES["thorough_wasteful"], world).turns == thorough + 1
|
|
|
|
|
|
def test_reference_exists_and_the_investigators_catch_everything_on_every_seed() -> None:
|
|
"""Lens 0: `reference_minutes` is never null. Lens 1/2: caught 1.0 over 4096, not 24."""
|
|
for seed in ALL:
|
|
world = generate(seed)
|
|
minutes, policy = reference_for(world)
|
|
assert minutes is not None and policy is not None, seed
|
|
for name in ("targeted", "thorough"):
|
|
assert run_policy(POLICIES[name], world).outcome == "solved", (seed, name)
|
|
|
|
|
|
def test_fast_misses_exactly_the_hidden_tier() -> None:
|
|
for seed in range(512):
|
|
world = world_for_seed(seed)
|
|
outcome = run_policy(POLICIES["fast"], world).outcome
|
|
assert (outcome == "solved") == (world["tier"] != "hidden"), seed
|
|
|
|
|
|
def test_reference_is_fast_where_the_screen_suffices() -> None:
|
|
for seed in range(512):
|
|
world = world_for_seed(seed)
|
|
assert world["reference_policy"] == ("targeted" if world["tier"] == "hidden" else "fast"), seed
|
|
|
|
|
|
def test_policies_cannot_see_the_answer() -> None:
|
|
"""Run every policy against a view whose world has the hidden fields deleted: identical replies."""
|
|
world = world_for_seed(9)
|
|
stripped = copy.deepcopy(world)
|
|
for key in ("label", "typology", "planted", "overlay", "tier", "template", "reference_minutes", "reference_policy"):
|
|
stripped[key] = None if key != "planted" else []
|
|
for name, policy in POLICIES.items():
|
|
a, b = Engine(world), Engine(stripped)
|
|
while not a.done:
|
|
ra, rb = policy(a.view()), policy(b.view())
|
|
assert ra == rb, name
|
|
a.step(ra)
|
|
b.step(rb)
|