Files
PIG-Demo/envs/alert_triage/tests/test_triage_engine.py
T
karti-ai 2dfa96939e
ci / web (push) Successful in 2m43s
ci / python (push) Successful in 2m36s
Alert Triage: environment #2, built end to end by the pipeline
The first environment shipped through .claude/workflows/new-environment.js:
specification, three adversarial reviews (all 'fixable', none fatal), the
Python environment, the TypeScript port, captured rollouts, and the demo page.
Eleven agents, no errors.

The proof that the platform scales is one line long. Alert Triage has a
completely different shape from Word Five — JSON actions, priced lookups, an
analyst screen instead of a grid — and the only change under
src/components/demo/ is a comment edit, because the isolation lint refused the
word "wordle" there. Zero shell code changed. 415 contract checks now pass
against two demos, up from 206 against one.

The environment is honest by construction. Every alert is synthetic, generated
from the seed, and the banner saying so sits inside the board surface. Two of
the eleven scenario templates are hidden-suspicious: generated by the same code
as their benign twin with the signal overlaid only in lookup data, so the free
screen is identically distributed and a screen-only policy STRUCTURALLY cannot
tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden
seeds. That is the counterweight made real rather than asserted.

Twelve policies, thirteen ladder assertions, a genuine three-way trade:

  fast      0.846   wins hours (0.85), misses every hidden case
  targeted  0.894   wins the shipped total
  thorough  0.820   wins evidence (1.00), spends 2.9 hours

None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at
delta 0, and conformance gated on world + scorer + protocol so the browser shows
the same alert for ?seed= that Python generated.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
2026-08-28 19:48:36 -07:00

125 lines
5.5 KiB
Python

"""Rejections, charges, shown ids, and the abort/truncate line."""
from __future__ import annotations
import json
from alert_triage.engine import Engine
from alert_triage.generator import MAX_TURNS, world_for_seed
from alert_triage.reward import DOC_MINUTES, LOOKUP_MINUTES, SCREEN_MINUTES, TURN_MINUTES, score
JUNK = ["", " ", None, "no json here", "{", "}", "{}", "[]", "null", "{\"action\":null}", "{\"action\":\"lookup\"}",
"{\"action\":\"lookup\",\"what\":\"history\"}", "{\"action\":\"lookup\",\"what\":\"history\",\"month\":7}",
"{\"action\":\"lookup\",\"what\":\"counterparty\"}", "{\"action\":\"lookup\",\"what\":\"counterparty\",\"id\":[]}",
"{\"action\":\"close\"}", "{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\"}",
"{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\",\"cites\":{}}", "{\"action\":\"escalate\",\"cites\":[\"kyc.country\"]}",
"{\"action\":\"lookup\",\"what\":\"documents\",\"x\":NaN}", "\x00\x01", "{" * 50]
def _e(seed: int = 0) -> Engine:
return Engine(world_for_seed(seed))
def test_screen_costs_and_first_turn() -> None:
e = _e()
assert e.minutes == SCREEN_MINUTES and e.turns == 0 and not e.done
def test_junk_never_raises_and_costs_one_turn_each() -> None:
for junk in JUNK:
e = _e()
step = e.step(junk)
assert not step.accepted and step.reason and e.turns == 1 and e.rejected == 1
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES
def test_trailing_garbage_after_a_balanced_object_is_fine() -> None:
e = _e()
assert e.step('{"action":"lookup","what":"documents"}' + "}" * 50 + " and more prose").accepted
def test_lookup_charges_its_price_only_when_accepted() -> None:
e = _e()
e.step('{"action":"lookup","what":"documents"}')
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES + DOC_MINUTES
e.step('{"action":"lookup","what":"documents"}') # repeat: rejected
assert e.minutes == SCREEN_MINUTES + 2 * TURN_MINUTES + DOC_MINUTES and e.rejected == 1
e.step('{"action":"lookup","what":"prior_alerts"}')
assert e.minutes == SCREEN_MINUTES + 3 * TURN_MINUTES + DOC_MINUTES + LOOKUP_MINUTES
def test_history_window_is_the_twelve_months_ending_in_the_fire_month() -> None:
e = _e()
months = e.world["months"]
assert len(months) == 12 and months[-1] == e.world["alert"]["fired"][:7]
assert e.step(json.dumps({"action": "lookup", "what": "history", "month": months[0]})).accepted
before = months[0][:4] + "-" + f"{int(months[0][5:]) - 1:02d}" if months[0][5:] != "01" else f"{int(months[0][:4]) - 1}-12"
assert not e.step(json.dumps({"action": "lookup", "what": "history", "month": before})).accepted
def test_counterparty_must_have_been_shown() -> None:
e = _e()
on_screen = e.world["screen_counterparties"][0]["id"]
assert e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
assert not e.step('{"action":"lookup","what":"counterparty","id":"CP-999"}').accepted
assert not e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
def test_cites_must_be_shown_and_are_deduplicated() -> None:
e = _e()
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","doc.D-0000"]}')
assert not step.accepted and "not been shown" in step.reason
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","kyc.country","kyc.pep"]}')
assert step.accepted and step.action["cites"] == ["kyc.country", "kyc.pep"] and e.done
def test_documents_become_citable_after_the_lookup() -> None:
e = _e()
doc = e.world["documents"][0]["id"]
assert not e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
assert e.step('{"action":"lookup","what":"documents"}').accepted
assert e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
def test_enums_are_exact_case() -> None:
e = _e()
assert not e.step('{"action":"close","reason":"rule_artefact","cites":["kyc.country"]}').accepted
assert not e.step('{"action":"ESCALATE","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
assert e.step('{"action":"escalate","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
def test_eight_rejections_abort_and_score_zero() -> None:
e = _e()
for _ in range(MAX_TURNS):
e.step("nothing")
assert e.done and e.aborted and e.outcome == "aborted"
assert e.step("{}").reason == "the episode is over" and e.turns == MAX_TURNS
assert score(e.episode()) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
def test_a_disposition_on_the_eighth_turn_is_accepted() -> None:
e = _e()
for _ in range(MAX_TURNS - 1):
e.step("nothing")
assert e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country"]}').accepted
assert e.outcome in ("solved", "failed")
def test_stopping_early_is_truncated_not_aborted() -> None:
e = _e()
e.step("nothing")
e.step('{"action":"lookup","what":"documents"}')
assert not e.done and e.outcome is None
ep = e.episode()
assert ep.truncated
assert score(ep) == {"caught": None, "hours": None, "evidence": None}
def test_view_never_shows_hidden_fields() -> None:
e = _e(5)
e.step('{"action":"lookup","what":"documents"}')
e.step('{"action":"lookup","what":"prior_alerts"}')
text = json.dumps(e.view())
for key in ("label", "planted", "typology", "overlay", "tier", "template", "reference_minutes", "reference_policy"):
assert f'"{key}"' not in text, key