2dfa96939e
The first environment shipped through .claude/workflows/new-environment.js: specification, three adversarial reviews (all 'fixable', none fatal), the Python environment, the TypeScript port, captured rollouts, and the demo page. Eleven agents, no errors. The proof that the platform scales is one line long. Alert Triage has a completely different shape from Word Five — JSON actions, priced lookups, an analyst screen instead of a grid — and the only change under src/components/demo/ is a comment edit, because the isolation lint refused the word "wordle" there. Zero shell code changed. 415 contract checks now pass against two demos, up from 206 against one. The environment is honest by construction. Every alert is synthetic, generated from the seed, and the banner saying so sits inside the board surface. Two of the eleven scenario templates are hidden-suspicious: generated by the same code as their benign twin with the signal overlaid only in lookup data, so the free screen is identically distributed and a screen-only policy STRUCTURALLY cannot tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden seeds. That is the counterweight made real rather than asserted. Twelve policies, thirteen ladder assertions, a genuine three-way trade: fast 0.846 wins hours (0.85), misses every hidden case targeted 0.894 wins the shipped total thorough 0.820 wins evidence (1.00), spends 2.9 hours None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at delta 0, and conformance gated on world + scorer + protocol so the browser shows the same alert for ?seed= that Python generated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
125 lines
5.5 KiB
Python
125 lines
5.5 KiB
Python
"""Rejections, charges, shown ids, and the abort/truncate line."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from alert_triage.engine import Engine
|
|
from alert_triage.generator import MAX_TURNS, world_for_seed
|
|
from alert_triage.reward import DOC_MINUTES, LOOKUP_MINUTES, SCREEN_MINUTES, TURN_MINUTES, score
|
|
|
|
JUNK = ["", " ", None, "no json here", "{", "}", "{}", "[]", "null", "{\"action\":null}", "{\"action\":\"lookup\"}",
|
|
"{\"action\":\"lookup\",\"what\":\"history\"}", "{\"action\":\"lookup\",\"what\":\"history\",\"month\":7}",
|
|
"{\"action\":\"lookup\",\"what\":\"counterparty\"}", "{\"action\":\"lookup\",\"what\":\"counterparty\",\"id\":[]}",
|
|
"{\"action\":\"close\"}", "{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\"}",
|
|
"{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\",\"cites\":{}}", "{\"action\":\"escalate\",\"cites\":[\"kyc.country\"]}",
|
|
"{\"action\":\"lookup\",\"what\":\"documents\",\"x\":NaN}", "\x00\x01", "{" * 50]
|
|
|
|
|
|
def _e(seed: int = 0) -> Engine:
|
|
return Engine(world_for_seed(seed))
|
|
|
|
|
|
def test_screen_costs_and_first_turn() -> None:
|
|
e = _e()
|
|
assert e.minutes == SCREEN_MINUTES and e.turns == 0 and not e.done
|
|
|
|
|
|
def test_junk_never_raises_and_costs_one_turn_each() -> None:
|
|
for junk in JUNK:
|
|
e = _e()
|
|
step = e.step(junk)
|
|
assert not step.accepted and step.reason and e.turns == 1 and e.rejected == 1
|
|
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES
|
|
|
|
|
|
def test_trailing_garbage_after_a_balanced_object_is_fine() -> None:
|
|
e = _e()
|
|
assert e.step('{"action":"lookup","what":"documents"}' + "}" * 50 + " and more prose").accepted
|
|
|
|
|
|
def test_lookup_charges_its_price_only_when_accepted() -> None:
|
|
e = _e()
|
|
e.step('{"action":"lookup","what":"documents"}')
|
|
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES + DOC_MINUTES
|
|
e.step('{"action":"lookup","what":"documents"}') # repeat: rejected
|
|
assert e.minutes == SCREEN_MINUTES + 2 * TURN_MINUTES + DOC_MINUTES and e.rejected == 1
|
|
e.step('{"action":"lookup","what":"prior_alerts"}')
|
|
assert e.minutes == SCREEN_MINUTES + 3 * TURN_MINUTES + DOC_MINUTES + LOOKUP_MINUTES
|
|
|
|
|
|
def test_history_window_is_the_twelve_months_ending_in_the_fire_month() -> None:
|
|
e = _e()
|
|
months = e.world["months"]
|
|
assert len(months) == 12 and months[-1] == e.world["alert"]["fired"][:7]
|
|
assert e.step(json.dumps({"action": "lookup", "what": "history", "month": months[0]})).accepted
|
|
before = months[0][:4] + "-" + f"{int(months[0][5:]) - 1:02d}" if months[0][5:] != "01" else f"{int(months[0][:4]) - 1}-12"
|
|
assert not e.step(json.dumps({"action": "lookup", "what": "history", "month": before})).accepted
|
|
|
|
|
|
def test_counterparty_must_have_been_shown() -> None:
|
|
e = _e()
|
|
on_screen = e.world["screen_counterparties"][0]["id"]
|
|
assert e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
|
|
assert not e.step('{"action":"lookup","what":"counterparty","id":"CP-999"}').accepted
|
|
assert not e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
|
|
|
|
|
|
def test_cites_must_be_shown_and_are_deduplicated() -> None:
|
|
e = _e()
|
|
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","doc.D-0000"]}')
|
|
assert not step.accepted and "not been shown" in step.reason
|
|
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","kyc.country","kyc.pep"]}')
|
|
assert step.accepted and step.action["cites"] == ["kyc.country", "kyc.pep"] and e.done
|
|
|
|
|
|
def test_documents_become_citable_after_the_lookup() -> None:
|
|
e = _e()
|
|
doc = e.world["documents"][0]["id"]
|
|
assert not e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
|
|
assert e.step('{"action":"lookup","what":"documents"}').accepted
|
|
assert e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
|
|
|
|
|
|
def test_enums_are_exact_case() -> None:
|
|
e = _e()
|
|
assert not e.step('{"action":"close","reason":"rule_artefact","cites":["kyc.country"]}').accepted
|
|
assert not e.step('{"action":"ESCALATE","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
|
|
assert e.step('{"action":"escalate","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
|
|
|
|
|
|
def test_eight_rejections_abort_and_score_zero() -> None:
|
|
e = _e()
|
|
for _ in range(MAX_TURNS):
|
|
e.step("nothing")
|
|
assert e.done and e.aborted and e.outcome == "aborted"
|
|
assert e.step("{}").reason == "the episode is over" and e.turns == MAX_TURNS
|
|
assert score(e.episode()) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
|
|
|
|
|
def test_a_disposition_on_the_eighth_turn_is_accepted() -> None:
|
|
e = _e()
|
|
for _ in range(MAX_TURNS - 1):
|
|
e.step("nothing")
|
|
assert e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country"]}').accepted
|
|
assert e.outcome in ("solved", "failed")
|
|
|
|
|
|
def test_stopping_early_is_truncated_not_aborted() -> None:
|
|
e = _e()
|
|
e.step("nothing")
|
|
e.step('{"action":"lookup","what":"documents"}')
|
|
assert not e.done and e.outcome is None
|
|
ep = e.episode()
|
|
assert ep.truncated
|
|
assert score(ep) == {"caught": None, "hours": None, "evidence": None}
|
|
|
|
|
|
def test_view_never_shows_hidden_fields() -> None:
|
|
e = _e(5)
|
|
e.step('{"action":"lookup","what":"documents"}')
|
|
e.step('{"action":"lookup","what":"prior_alerts"}')
|
|
text = json.dumps(e.view())
|
|
for key in ("label", "planted", "typology", "overlay", "tier", "template", "reference_minutes", "reference_policy"):
|
|
assert f'"{key}"' not in text, key
|