Alert Triage: environment #2, built end to end by the pipeline
The first environment shipped through .claude/workflows/new-environment.js: specification, three adversarial reviews (all 'fixable', none fatal), the Python environment, the TypeScript port, captured rollouts, and the demo page. Eleven agents, no errors. The proof that the platform scales is one line long. Alert Triage has a completely different shape from Word Five — JSON actions, priced lookups, an analyst screen instead of a grid — and the only change under src/components/demo/ is a comment edit, because the isolation lint refused the word "wordle" there. Zero shell code changed. 415 contract checks now pass against two demos, up from 206 against one. The environment is honest by construction. Every alert is synthetic, generated from the seed, and the banner saying so sits inside the board surface. Two of the eleven scenario templates are hidden-suspicious: generated by the same code as their benign twin with the signal overlaid only in lookup data, so the free screen is identically distributed and a screen-only policy STRUCTURALLY cannot tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden seeds. That is the counterweight made real rather than asserted. Twelve policies, thirteen ladder assertions, a genuine three-way trade: fast 0.846 wins hours (0.85), misses every hidden case targeted 0.894 wins the shipped total thorough 0.820 wins evidence (1.00), spends 2.9 hours None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at delta 0, and conformance gated on world + scorer + protocol so the browser shows the same alert for ?seed= that Python generated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
karti-ai
@@ -0,0 +1,32 @@
|
||||
"""The cross-language gates. If any of these changes, the TypeScript port must be re-verified."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alert_triage.conformance import committed, protocol_digest, protocol_trace, scorer_digest
|
||||
from alert_triage.generator import world_digest
|
||||
|
||||
|
||||
def test_scorer_digest_matches_the_committed_value() -> None:
|
||||
assert scorer_digest() == committed()["scorer"]
|
||||
|
||||
|
||||
def test_protocol_digest_matches_the_committed_value() -> None:
|
||||
assert protocol_digest() == committed()["protocol"]
|
||||
|
||||
|
||||
def test_protocol_corpus_exercises_every_rejection_path() -> None:
|
||||
trace = protocol_trace()
|
||||
reasons = {t["reason"] for case in trace for t in case["turns"] if not t["accepted"]}
|
||||
fragments = ["no JSON object", "could not be parsed", "not a JSON object", "unknown action", "unknown lookup",
|
||||
"not in the twelve-month window", "already been looked up", "has not appeared", "cites must be",
|
||||
"not been shown", "unknown close reason", "unknown typology"]
|
||||
for fragment in fragments:
|
||||
assert any(fragment in r for r in reasons), fragment
|
||||
outcomes = {case["outcome"] for case in trace}
|
||||
assert "aborted" in outcomes
|
||||
assert any(case["truncated"] for case in trace)
|
||||
assert any(len(case["turns"]) == 8 and case["outcome"] in ("solved", "failed") for case in trace)
|
||||
|
||||
|
||||
def test_world_digest_matches_the_committed_value() -> None:
|
||||
assert world_digest() == committed()["world"]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Rejections, charges, shown ids, and the abort/truncate line."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from alert_triage.engine import Engine
|
||||
from alert_triage.generator import MAX_TURNS, world_for_seed
|
||||
from alert_triage.reward import DOC_MINUTES, LOOKUP_MINUTES, SCREEN_MINUTES, TURN_MINUTES, score
|
||||
|
||||
JUNK = ["", " ", None, "no json here", "{", "}", "{}", "[]", "null", "{\"action\":null}", "{\"action\":\"lookup\"}",
|
||||
"{\"action\":\"lookup\",\"what\":\"history\"}", "{\"action\":\"lookup\",\"what\":\"history\",\"month\":7}",
|
||||
"{\"action\":\"lookup\",\"what\":\"counterparty\"}", "{\"action\":\"lookup\",\"what\":\"counterparty\",\"id\":[]}",
|
||||
"{\"action\":\"close\"}", "{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\"}",
|
||||
"{\"action\":\"close\",\"reason\":\"RULE_ARTEFACT\",\"cites\":{}}", "{\"action\":\"escalate\",\"cites\":[\"kyc.country\"]}",
|
||||
"{\"action\":\"lookup\",\"what\":\"documents\",\"x\":NaN}", "\x00\x01", "{" * 50]
|
||||
|
||||
|
||||
def _e(seed: int = 0) -> Engine:
|
||||
return Engine(world_for_seed(seed))
|
||||
|
||||
|
||||
def test_screen_costs_and_first_turn() -> None:
|
||||
e = _e()
|
||||
assert e.minutes == SCREEN_MINUTES and e.turns == 0 and not e.done
|
||||
|
||||
|
||||
def test_junk_never_raises_and_costs_one_turn_each() -> None:
|
||||
for junk in JUNK:
|
||||
e = _e()
|
||||
step = e.step(junk)
|
||||
assert not step.accepted and step.reason and e.turns == 1 and e.rejected == 1
|
||||
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES
|
||||
|
||||
|
||||
def test_trailing_garbage_after_a_balanced_object_is_fine() -> None:
|
||||
e = _e()
|
||||
assert e.step('{"action":"lookup","what":"documents"}' + "}" * 50 + " and more prose").accepted
|
||||
|
||||
|
||||
def test_lookup_charges_its_price_only_when_accepted() -> None:
|
||||
e = _e()
|
||||
e.step('{"action":"lookup","what":"documents"}')
|
||||
assert e.minutes == SCREEN_MINUTES + TURN_MINUTES + DOC_MINUTES
|
||||
e.step('{"action":"lookup","what":"documents"}') # repeat: rejected
|
||||
assert e.minutes == SCREEN_MINUTES + 2 * TURN_MINUTES + DOC_MINUTES and e.rejected == 1
|
||||
e.step('{"action":"lookup","what":"prior_alerts"}')
|
||||
assert e.minutes == SCREEN_MINUTES + 3 * TURN_MINUTES + DOC_MINUTES + LOOKUP_MINUTES
|
||||
|
||||
|
||||
def test_history_window_is_the_twelve_months_ending_in_the_fire_month() -> None:
|
||||
e = _e()
|
||||
months = e.world["months"]
|
||||
assert len(months) == 12 and months[-1] == e.world["alert"]["fired"][:7]
|
||||
assert e.step(json.dumps({"action": "lookup", "what": "history", "month": months[0]})).accepted
|
||||
before = months[0][:4] + "-" + f"{int(months[0][5:]) - 1:02d}" if months[0][5:] != "01" else f"{int(months[0][:4]) - 1}-12"
|
||||
assert not e.step(json.dumps({"action": "lookup", "what": "history", "month": before})).accepted
|
||||
|
||||
|
||||
def test_counterparty_must_have_been_shown() -> None:
|
||||
e = _e()
|
||||
on_screen = e.world["screen_counterparties"][0]["id"]
|
||||
assert e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
|
||||
assert not e.step('{"action":"lookup","what":"counterparty","id":"CP-999"}').accepted
|
||||
assert not e.step(json.dumps({"action": "lookup", "what": "counterparty", "id": on_screen})).accepted
|
||||
|
||||
|
||||
def test_cites_must_be_shown_and_are_deduplicated() -> None:
|
||||
e = _e()
|
||||
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","doc.D-0000"]}')
|
||||
assert not step.accepted and "not been shown" in step.reason
|
||||
step = e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country","kyc.country","kyc.pep"]}')
|
||||
assert step.accepted and step.action["cites"] == ["kyc.country", "kyc.pep"] and e.done
|
||||
|
||||
|
||||
def test_documents_become_citable_after_the_lookup() -> None:
|
||||
e = _e()
|
||||
doc = e.world["documents"][0]["id"]
|
||||
assert not e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
|
||||
assert e.step('{"action":"lookup","what":"documents"}').accepted
|
||||
assert e.step(json.dumps({"action": "close", "reason": "DOCUMENTED_SOURCE_OF_FUNDS", "cites": [doc]})).accepted
|
||||
|
||||
|
||||
def test_enums_are_exact_case() -> None:
|
||||
e = _e()
|
||||
assert not e.step('{"action":"close","reason":"rule_artefact","cites":["kyc.country"]}').accepted
|
||||
assert not e.step('{"action":"ESCALATE","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
|
||||
assert e.step('{"action":"escalate","typology":"UNKNOWN","cites":["kyc.country"]}').accepted
|
||||
|
||||
|
||||
def test_eight_rejections_abort_and_score_zero() -> None:
|
||||
e = _e()
|
||||
for _ in range(MAX_TURNS):
|
||||
e.step("nothing")
|
||||
assert e.done and e.aborted and e.outcome == "aborted"
|
||||
assert e.step("{}").reason == "the episode is over" and e.turns == MAX_TURNS
|
||||
assert score(e.episode()) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
||||
|
||||
|
||||
def test_a_disposition_on_the_eighth_turn_is_accepted() -> None:
|
||||
e = _e()
|
||||
for _ in range(MAX_TURNS - 1):
|
||||
e.step("nothing")
|
||||
assert e.step('{"action":"close","reason":"RULE_ARTEFACT","cites":["kyc.country"]}').accepted
|
||||
assert e.outcome in ("solved", "failed")
|
||||
|
||||
|
||||
def test_stopping_early_is_truncated_not_aborted() -> None:
|
||||
e = _e()
|
||||
e.step("nothing")
|
||||
e.step('{"action":"lookup","what":"documents"}')
|
||||
assert not e.done and e.outcome is None
|
||||
ep = e.episode()
|
||||
assert ep.truncated
|
||||
assert score(ep) == {"caught": None, "hours": None, "evidence": None}
|
||||
|
||||
|
||||
def test_view_never_shows_hidden_fields() -> None:
|
||||
e = _e(5)
|
||||
e.step('{"action":"lookup","what":"documents"}')
|
||||
e.step('{"action":"lookup","what":"prior_alerts"}')
|
||||
text = json.dumps(e.view())
|
||||
for key in ("label", "planted", "typology", "overlay", "tier", "template", "reference_minutes", "reference_policy"):
|
||||
assert f'"{key}"' not in text, key
|
||||
@@ -0,0 +1,164 @@
|
||||
"""What the generator promises about every world, checked over every digested seed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
|
||||
import pytest
|
||||
|
||||
from alert_triage.generator import (
|
||||
BENIGN_PCT,
|
||||
RULE_IDS,
|
||||
VISIBLE_PCT,
|
||||
canonical_json,
|
||||
generate,
|
||||
is_held_out,
|
||||
screen_of,
|
||||
)
|
||||
from alert_triage.rng import days_from_civil
|
||||
|
||||
ALL = range(4096)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def worlds():
|
||||
return [generate(s) for s in ALL]
|
||||
|
||||
|
||||
def _day(iso: str) -> int:
|
||||
y, m, d = (int(x) for x in iso.split("-"))
|
||||
return days_from_civil(y, m, d)
|
||||
|
||||
|
||||
def test_deterministic() -> None:
|
||||
assert canonical_json(generate(17)) == canonical_json(generate(17))
|
||||
assert canonical_json(generate(17)) != canonical_json(generate(18))
|
||||
|
||||
|
||||
def test_canonical_json_has_no_floats(worlds) -> None:
|
||||
def walk(v):
|
||||
if isinstance(v, float):
|
||||
raise AssertionError("float in world")
|
||||
if isinstance(v, dict):
|
||||
for x in v.values():
|
||||
walk(x)
|
||||
if isinstance(v, list):
|
||||
for x in v:
|
||||
walk(x)
|
||||
for w in worlds[:512]:
|
||||
walk(w)
|
||||
assert json.loads(canonical_json(w)) == w
|
||||
|
||||
|
||||
def test_mix_by_seed_is_close_to_declared(worlds) -> None:
|
||||
tiers = Counter(w["tier"] for w in worlds)
|
||||
n = len(worlds)
|
||||
assert abs(tiers["benign"] / n - BENIGN_PCT / 100) < 0.03
|
||||
assert abs(tiers["visible"] / n - VISIBLE_PCT / 100) < 0.03
|
||||
assert abs(tiers["hidden"] / n - (100 - BENIGN_PCT - VISIBLE_PCT) / 100) < 0.03
|
||||
|
||||
|
||||
def test_every_rule_carries_both_labels(worlds) -> None:
|
||||
"""CHEAT 3: no rule id may be a label. Each fires on benign AND suspicious seeds."""
|
||||
seen = Counter((w["alert"]["rule"], w["label"]) for w in worlds)
|
||||
for table in RULE_IDS.values():
|
||||
for rid in table.values():
|
||||
assert seen[(rid, "benign")] > 0, rid
|
||||
assert seen[(rid, "suspicious")] > 0, rid
|
||||
|
||||
|
||||
def test_every_hidden_signal_lands_on_several_templates(worlds) -> None:
|
||||
"""CHEAT 4: no fixed template -> lookup mapping to memorise."""
|
||||
pairs = Counter((w["overlay"], w["template"]) for w in worlds if w["overlay"])
|
||||
by_kind: dict[str, set[str]] = {}
|
||||
for (kind, template), _ in pairs.items():
|
||||
by_kind.setdefault(kind, set()).add(template)
|
||||
assert len(by_kind["funnel"]) == 6
|
||||
assert len(by_kind["serial_closer"]) >= 4
|
||||
assert len(by_kind["doc_mismatch"]) == 3
|
||||
|
||||
|
||||
def test_overlay_only_on_benign_screens(worlds) -> None:
|
||||
for w in worlds:
|
||||
if w["overlay"]:
|
||||
assert w["template"].startswith("B") and w["tier"] == "hidden" and w["label"] == "suspicious"
|
||||
if w["template"].startswith("S"):
|
||||
assert w["label"] == "suspicious" and w["overlay"] is None
|
||||
|
||||
|
||||
def test_nothing_is_dated_on_or_after_the_fire_date(worlds) -> None:
|
||||
for w in worlds:
|
||||
fired = _day(w["alert"]["fired"])
|
||||
assert all(_day(t["date"]) < fired for t in w["transactions"]), w["seed"]
|
||||
|
||||
|
||||
def test_transaction_ids_are_chronological_and_dense(worlds) -> None:
|
||||
for w in worlds[:512]:
|
||||
ids = [t["id"] for t in w["transactions"]]
|
||||
assert ids == [f"T-{i + 1}" for i in range(len(ids))]
|
||||
days = [_day(t["date"]) for t in w["transactions"]]
|
||||
assert days == sorted(days)
|
||||
|
||||
|
||||
def test_triggering_set_is_non_empty_and_shown(worlds) -> None:
|
||||
for w in worlds:
|
||||
ids = {t["id"] for t in w["transactions"]}
|
||||
assert w["alert"]["triggering"], w["seed"]
|
||||
assert set(w["alert"]["triggering"]) <= ids
|
||||
|
||||
|
||||
def test_planted_is_citable_after_the_right_lookups(worlds) -> None:
|
||||
"""Every planted id exists in the world: a kyc field, a T-id, a CP-id, a doc or a prior alert."""
|
||||
for w in worlds:
|
||||
universe = {f"kyc.{k}" for k in w["customer"]} | {t["id"] for t in w["transactions"]}
|
||||
universe |= {c["id"] for c in w["counterparties"]} | {d["id"] for d in w["documents"]} | {p["id"] for p in w["prior_alerts"]}
|
||||
assert w["planted"], w["seed"]
|
||||
for alt in w["planted"]:
|
||||
assert alt and set(alt) <= universe, (w["seed"], alt)
|
||||
|
||||
|
||||
def test_benign_screens_always_carry_an_outbound_counterparty(worlds) -> None:
|
||||
for w in worlds:
|
||||
if w["template"].startswith("B"):
|
||||
assert any(c["direction"] == "out" for c in w["screen_counterparties"]), w["seed"]
|
||||
assert len(w["screen_counterparties"]) == 2
|
||||
assert 1 <= len(w["screen_counterparties"]) <= 2
|
||||
|
||||
|
||||
def test_prior_alert_count_on_screen_matches_the_lookup(worlds) -> None:
|
||||
for w in worlds:
|
||||
assert len(w["prior_alerts"]) == w["customer"]["prior_alerts"]
|
||||
|
||||
|
||||
def test_held_out_slice_moves_the_threshold_and_the_screen(worlds) -> None:
|
||||
held = [w for w in worlds if w["held_out"]]
|
||||
main = [w for w in worlds if not w["held_out"]]
|
||||
assert 0.10 < len(held) / len(worlds) < 0.15
|
||||
for w in held:
|
||||
assert is_held_out(w["seed"])
|
||||
assert w["currency"] == "KRN" and w["threshold"] == 2_000_000 and w["screen_order"] == "kyc_first"
|
||||
assert w["alert"]["rule"] in RULE_IDS["held_out"].values()
|
||||
assert w["customer"]["country"] == "Karnland"
|
||||
for w in main:
|
||||
assert w["currency"] == "USD" and w["threshold"] == 1_000_000 and w["screen_order"] == "alert_first"
|
||||
assert w["alert"]["rule"] in RULE_IDS["main"].values()
|
||||
|
||||
|
||||
def test_structuring_band_is_threshold_relative(worlds) -> None:
|
||||
for w in worlds:
|
||||
if w["template"] == "S1":
|
||||
lo, hi = w["threshold"] * 92 // 100, w["threshold"] - 1
|
||||
deposits = [t for t in w["transactions"] if t["id"] in w["planted"][0]]
|
||||
assert len(deposits) == 8
|
||||
assert all(lo <= t["amount"] <= hi for t in deposits)
|
||||
assert len({t["branch"] for t in deposits}) == 3
|
||||
|
||||
|
||||
def test_screen_carries_no_hidden_field(worlds) -> None:
|
||||
hidden = {"label", "typology", "planted", "overlay", "tier", "template", "reference_minutes", "reference_policy",
|
||||
"prior_alerts_list", "documents", "counterparties", "transactions", "other_alerted_accounts", "party", "rationale"}
|
||||
for w in worlds[:256]:
|
||||
text = canonical_json(screen_of(w))
|
||||
for key in hidden:
|
||||
assert f'"{key}"' not in text, key
|
||||
@@ -0,0 +1,84 @@
|
||||
"""The reference analysts are players, not oracles, and the reference exists everywhere."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
|
||||
from alert_triage.engine import Engine
|
||||
from alert_triage.generator import MAX_TURNS, canonical_json, generate, world_for_seed
|
||||
from alert_triage.policies import POLICIES, SHIPPED, reference_for, run_policy
|
||||
|
||||
ALL = range(4096)
|
||||
|
||||
|
||||
def test_every_policy_takes_only_the_view() -> None:
|
||||
for name, policy in POLICIES.items():
|
||||
params = list(inspect.signature(policy).parameters)
|
||||
assert params == ["view"], name
|
||||
|
||||
|
||||
def test_policies_never_mutate_the_world() -> None:
|
||||
for seed in range(32):
|
||||
world = world_for_seed(seed)
|
||||
before = canonical_json(world)
|
||||
for policy in POLICIES.values():
|
||||
run_policy(policy, world)
|
||||
assert canonical_json(world) == before
|
||||
|
||||
|
||||
def test_shipped_policies_are_never_rejected() -> None:
|
||||
for seed in range(256):
|
||||
world = world_for_seed(seed)
|
||||
for name in SHIPPED + ("thorough_wasteful",):
|
||||
engine = run_policy(POLICIES[name], world)
|
||||
assert engine.rejected == 0, (seed, name)
|
||||
assert engine.disposition is not None, (seed, name)
|
||||
|
||||
|
||||
def test_turn_budgets_are_as_documented() -> None:
|
||||
for seed in range(128):
|
||||
world = world_for_seed(seed)
|
||||
assert run_policy(POLICIES["fast"], world).turns <= 2
|
||||
assert run_policy(POLICIES["targeted"], world).turns <= 5
|
||||
thorough = run_policy(POLICIES["thorough"], world).turns
|
||||
assert thorough <= MAX_TURNS - 1
|
||||
assert run_policy(POLICIES["thorough_wasteful"], world).turns == thorough + 1
|
||||
|
||||
|
||||
def test_reference_exists_and_the_investigators_catch_everything_on_every_seed() -> None:
|
||||
"""Lens 0: `reference_minutes` is never null. Lens 1/2: caught 1.0 over 4096, not 24."""
|
||||
for seed in ALL:
|
||||
world = generate(seed)
|
||||
minutes, policy = reference_for(world)
|
||||
assert minutes is not None and policy is not None, seed
|
||||
for name in ("targeted", "thorough"):
|
||||
assert run_policy(POLICIES[name], world).outcome == "solved", (seed, name)
|
||||
|
||||
|
||||
def test_fast_misses_exactly_the_hidden_tier() -> None:
|
||||
for seed in range(512):
|
||||
world = world_for_seed(seed)
|
||||
outcome = run_policy(POLICIES["fast"], world).outcome
|
||||
assert (outcome == "solved") == (world["tier"] != "hidden"), seed
|
||||
|
||||
|
||||
def test_reference_is_fast_where_the_screen_suffices() -> None:
|
||||
for seed in range(512):
|
||||
world = world_for_seed(seed)
|
||||
assert world["reference_policy"] == ("targeted" if world["tier"] == "hidden" else "fast"), seed
|
||||
|
||||
|
||||
def test_policies_cannot_see_the_answer() -> None:
|
||||
"""Run every policy against a view whose world has the hidden fields deleted: identical replies."""
|
||||
world = world_for_seed(9)
|
||||
stripped = copy.deepcopy(world)
|
||||
for key in ("label", "typology", "planted", "overlay", "tier", "template", "reference_minutes", "reference_policy"):
|
||||
stripped[key] = None if key != "planted" else []
|
||||
for name, policy in POLICIES.items():
|
||||
a, b = Engine(world), Engine(stripped)
|
||||
while not a.done:
|
||||
ra, rb = policy(a.view()), policy(b.view())
|
||||
assert ra == rb, name
|
||||
a.step(ra)
|
||||
b.step(rb)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""The pinned parse rules — the ones two runtimes disagree on."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alert_triage.protocol import extract_candidate, parse_reply
|
||||
|
||||
FENCE = "```"
|
||||
|
||||
|
||||
def test_fence_wins_over_an_earlier_bare_brace() -> None:
|
||||
text = 'first {not it} then ' + FENCE + 'json\n{"action":"lookup","what":"documents"}\n' + FENCE
|
||||
assert extract_candidate(text) == '{"action":"lookup","what":"documents"}\n'
|
||||
obj, reason = parse_reply(text)
|
||||
assert obj == {"action": "lookup", "what": "documents"} and reason is None
|
||||
|
||||
|
||||
def test_first_balanced_span_is_string_aware() -> None:
|
||||
text = 'ok {"note":"a } inside \\" quotes","action":"close"} trailing {"x":1}'
|
||||
obj, _ = parse_reply(text)
|
||||
assert obj == {"note": 'a } inside " quotes', "action": "close"}
|
||||
|
||||
|
||||
def test_invalid_first_span_is_rejected_without_further_scanning() -> None:
|
||||
obj, reason = parse_reply('{"a":1,} {"action":"lookup","what":"documents"}')
|
||||
assert obj is None and reason
|
||||
|
||||
|
||||
def test_non_object_json_is_rejected() -> None:
|
||||
assert parse_reply("[1,2]")[0] is None
|
||||
assert parse_reply(FENCE + "json\n[{\"action\":\"lookup\"}]\n" + FENCE)[0] is None
|
||||
assert parse_reply("42")[0] is None
|
||||
|
||||
|
||||
def test_nan_and_infinity_are_rejected_like_json_parse_does() -> None:
|
||||
assert parse_reply('{"x":NaN}')[0] is None
|
||||
assert parse_reply('{"x":Infinity}')[0] is None
|
||||
|
||||
|
||||
def test_unbalanced_or_absent_braces() -> None:
|
||||
assert parse_reply('{"action":"lookup"')[0] is None
|
||||
assert parse_reply("no braces at all")[0] is None
|
||||
assert parse_reply("")[0] is None
|
||||
assert parse_reply(None)[0] is None
|
||||
|
||||
|
||||
def test_nested_object_is_the_outer_one() -> None:
|
||||
obj, _ = parse_reply('{"outer":{"action":"lookup"}}')
|
||||
assert obj == {"outer": {"action": "lookup"}}
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Worked examples of the reward, and the marker region the page quotes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from alert_triage.reward import (
|
||||
FALSE_ESCALATION_CREDIT,
|
||||
WEIGHTS,
|
||||
Episode,
|
||||
escalate_break_even,
|
||||
metrics,
|
||||
score,
|
||||
score_exact,
|
||||
total,
|
||||
)
|
||||
|
||||
|
||||
def ep(**kw) -> Episode:
|
||||
base = dict(label="benign", planted=[["kyc.a"]], disposition="close", cites=["kyc.a"], minutes_spent=33,
|
||||
reference_minutes=33, turns_spent=1, rejected=0)
|
||||
base.update(kw)
|
||||
return Episode(**base)
|
||||
|
||||
|
||||
def test_weights_sum_to_one_and_have_a_counterweight() -> None:
|
||||
assert abs(sum(WEIGHTS.values()) - 1.0) < 1e-12
|
||||
from alert_triage.reward import ROLES
|
||||
assert "counterweight" in ROLES.values()
|
||||
|
||||
|
||||
def test_perfect_close() -> None:
|
||||
assert score(ep()) == {"caught": 1.0, "hours": 1.0, "evidence": 1.0}
|
||||
assert abs(total(ep()) - 1.0) < 1e-12
|
||||
|
||||
|
||||
def test_a_miss_scores_exactly_zero() -> None:
|
||||
e = ep(label="suspicious", planted=[["T-1", "T-2"]], disposition="close", cites=["kyc.a"])
|
||||
assert score(e) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
||||
assert total(e) == 0.0
|
||||
|
||||
|
||||
def test_a_false_escalation_takes_half_the_counterweight_and_nothing_else() -> None:
|
||||
e = ep(disposition="escalate", cites=["kyc.a"])
|
||||
assert score(e) == {"caught": FALSE_ESCALATION_CREDIT, "hours": 0.0, "evidence": 0.0}
|
||||
|
||||
|
||||
def test_no_disposition_is_zero_and_truncated_is_none() -> None:
|
||||
assert score(ep(disposition=None, cites=[])) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
||||
assert score(ep(truncated=True)) == {"caught": None, "hours": None, "evidence": None}
|
||||
assert total(ep(truncated=True)) is None
|
||||
|
||||
|
||||
def test_hours_is_capped_and_ratio_against_the_reference() -> None:
|
||||
assert score(ep(minutes_spent=20))["hours"] == 1.0
|
||||
assert abs(score(ep(minutes_spent=66))["hours"] - 0.5) < 1e-12
|
||||
assert score(ep(reference_minutes=None))["hours"] is None
|
||||
assert total(ep(reference_minutes=None)) is None
|
||||
|
||||
|
||||
def test_evidence_is_f1_over_the_best_alternate() -> None:
|
||||
e = ep(planted=[["a", "b"], ["a"]], cites=["a"])
|
||||
assert score(e)["evidence"] == 1.0
|
||||
e = ep(planted=[["a", "b"]], cites=["a", "x", "y"])
|
||||
assert abs(score(e)["evidence"] - 0.4) < 1e-12 # 2·1 / (3+2)
|
||||
e = ep(planted=[["a"]], cites=["a", "b", "c", "d", "e", "f", "g", "h"])
|
||||
assert abs(score(e)["evidence"] - 2 / 9) < 1e-12 # one planted among eight cited
|
||||
assert score_exact(e)["evidence"] == [2, 9]
|
||||
|
||||
|
||||
def test_exact_and_float_scores_agree() -> None:
|
||||
for e in (ep(), ep(minutes_spent=66), ep(planted=[["a", "b"]], cites=["a", "x", "y"]), ep(disposition="escalate")):
|
||||
exact = score_exact(e)
|
||||
approx = score(e)
|
||||
for k in approx:
|
||||
assert abs(exact[k][0] / exact[k][1] - approx[k]) < 1e-12
|
||||
|
||||
|
||||
def test_break_even_is_between_zero_and_one() -> None:
|
||||
assert 0.3 < escalate_break_even() < 0.7
|
||||
|
||||
|
||||
def test_metrics_are_diagnostics() -> None:
|
||||
m = metrics(ep(disposition="escalate", typology="UNKNOWN", true_typology="UNKNOWN", label="suspicious"))
|
||||
assert m["typology_match"] == 1.0 and m["false_escalation"] == 0.0
|
||||
assert metrics(ep())["typology_match"] is None
|
||||
|
||||
|
||||
def test_exactly_one_marker_pair() -> None:
|
||||
text = (Path(__file__).parent.parent / "alert_triage" / "reward.py").read_text()
|
||||
assert text.count("# region: pig-demo/reward") == 1
|
||||
assert text.count("# endregion: pig-demo/reward") == 1
|
||||
@@ -0,0 +1,52 @@
|
||||
"""The portable primitives, pinned so the TypeScript port has vectors to hit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alert_triage.rng import (
|
||||
XorShift32,
|
||||
civil_from_days,
|
||||
days_from_civil,
|
||||
days_in_month,
|
||||
fnv1a32,
|
||||
iso_date,
|
||||
)
|
||||
|
||||
# fnv1a32 over the decimal seed — identical to wordle_five's and to engine.ts.
|
||||
FNV_VECTORS = {"0": 0x350CA8AF, "1": 0x340CA71C, "42": 0x87E38583, "4095": 0x43875F5F}
|
||||
|
||||
|
||||
def test_fnv1a_vectors() -> None:
|
||||
for text, expect in FNV_VECTORS.items():
|
||||
assert fnv1a32(text) == expect, text
|
||||
|
||||
|
||||
def test_xorshift_sequence_is_pinned() -> None:
|
||||
rng = XorShift32(fnv1a32("0"))
|
||||
first = [rng.next() for _ in range(5)]
|
||||
assert first == XORSHIFT_FROM_SEED_0
|
||||
|
||||
|
||||
XORSHIFT_FROM_SEED_0 = [2738490563, 3068243922, 3765331391, 3085691315, 2439018365]
|
||||
|
||||
|
||||
def test_zero_seed_is_replaced() -> None:
|
||||
assert XorShift32(0).state != 0
|
||||
|
||||
|
||||
def test_sample_is_distinct_and_partial_shuffle() -> None:
|
||||
rng = XorShift32(7)
|
||||
out = rng.sample(list(range(10)), 4)
|
||||
assert len(out) == 4 and len(set(out)) == 4
|
||||
|
||||
|
||||
def test_civil_dates_round_trip() -> None:
|
||||
for day in range(days_from_civil(1999, 12, 25), days_from_civil(2030, 3, 2)):
|
||||
y, m, d = civil_from_days(day)
|
||||
assert days_from_civil(y, m, d) == day
|
||||
|
||||
|
||||
def test_known_dates() -> None:
|
||||
assert days_from_civil(1970, 1, 1) == 0
|
||||
assert iso_date(days_from_civil(2026, 3, 1)) == "2026-03-01"
|
||||
assert days_in_month(2024, 2) == 29 and days_in_month(2026, 2) == 28
|
||||
assert days_in_month(2026, 12) == 31
|
||||
@@ -0,0 +1,87 @@
|
||||
"""The hidden tier is hidden only if the generator keeps it so. This is the gate.
|
||||
|
||||
Two checks. The first is structural and exact: for every seed the free screen
|
||||
is byte-identical with the overlay on or off, for every overlay the screen can
|
||||
carry. The second is distributional: across seeds, the screen features a
|
||||
policy could key on do not separate hidden from benign beyond noise. The
|
||||
generator draws the tier BEFORE the template and the screen, so the two
|
||||
populations are the same draw; this test is what would catch a regression that
|
||||
made them differ.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from alert_triage.generator import canonical_json, compatible_overlays, generate, screen_of
|
||||
|
||||
ALL = range(4096)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pairs():
|
||||
out = []
|
||||
for s in ALL:
|
||||
base = generate(s, overlay=None)
|
||||
if base["template"].startswith("S"):
|
||||
continue
|
||||
out.append((s, base))
|
||||
return out
|
||||
|
||||
|
||||
def test_overlay_never_touches_the_screen(pairs) -> None:
|
||||
for seed, base in pairs:
|
||||
before = canonical_json(screen_of(base))
|
||||
for kind in compatible_overlays(base):
|
||||
overlaid = generate(seed, overlay=kind)
|
||||
assert overlaid["label"] == "suspicious" and overlaid["overlay"] == kind
|
||||
assert canonical_json(screen_of(overlaid)) == before, (seed, kind)
|
||||
|
||||
|
||||
def test_the_seed_overlay_is_one_of_the_compatible_kinds(pairs) -> None:
|
||||
for seed, base in pairs:
|
||||
w = generate(seed)
|
||||
if w["overlay"]:
|
||||
assert w["overlay"] in compatible_overlays(base)
|
||||
|
||||
|
||||
def _features(w) -> list[float]:
|
||||
c = w["customer"]
|
||||
screen = screen_of(w)
|
||||
trig = screen["triggering_transactions"]
|
||||
amounts = [t["amount"] for t in trig]
|
||||
return [
|
||||
float(c["prior_alerts"]),
|
||||
float(c["prior_sars"]),
|
||||
float(c["pep"]),
|
||||
float(sum(1 for cp in screen["screen_counterparties"] if cp["direction"] == "out")),
|
||||
float(sum(1 for a in amounts if a % 100000 == 0)) / max(1, len(amounts)),
|
||||
float(len(trig)),
|
||||
float(len({t["branch"] for t in trig if t["branch"]})),
|
||||
float(sum(r["wires_out"] > 0 for r in w["summary"])),
|
||||
float(screen["alert"]["rule"] == "R-STR-01"),
|
||||
float(c["expected_monthly_cash"] > 0),
|
||||
]
|
||||
|
||||
|
||||
def test_screen_features_do_not_separate_hidden_from_benign() -> None:
|
||||
"""Per-feature z-test between the two populations; nothing beyond 4 sigma."""
|
||||
benign, hidden = [], []
|
||||
for s in ALL:
|
||||
w = generate(s)
|
||||
if w["tier"] == "benign":
|
||||
benign.append(_features(w))
|
||||
elif w["tier"] == "hidden":
|
||||
hidden.append(_features(w))
|
||||
assert len(hidden) > 400
|
||||
for j in range(len(benign[0])):
|
||||
b = [f[j] for f in benign]
|
||||
h = [f[j] for f in hidden]
|
||||
mb, mh = sum(b) / len(b), sum(h) / len(h)
|
||||
vb = sum((x - mb) ** 2 for x in b) / max(1, len(b) - 1)
|
||||
vh = sum((x - mh) ** 2 for x in h) / max(1, len(h) - 1)
|
||||
se = math.sqrt(vb / len(b) + vh / len(h)) or 1e-9
|
||||
z = abs(mb - mh) / se
|
||||
assert z < 4.0, f"feature {j}: benign {mb:.3f} vs hidden {mh:.3f}, z={z:.1f}"
|
||||
Reference in New Issue
Block a user