karti-ai
2dfa96939e
The first environment shipped through .claude/workflows/new-environment.js: specification, three adversarial reviews (all 'fixable', none fatal), the Python environment, the TypeScript port, captured rollouts, and the demo page. Eleven agents, no errors. The proof that the platform scales is one line long. Alert Triage has a completely different shape from Word Five — JSON actions, priced lookups, an analyst screen instead of a grid — and the only change under src/components/demo/ is a comment edit, because the isolation lint refused the word "wordle" there. Zero shell code changed. 415 contract checks now pass against two demos, up from 206 against one. The environment is honest by construction. Every alert is synthetic, generated from the seed, and the banner saying so sits inside the board surface. Two of the eleven scenario templates are hidden-suspicious: generated by the same code as their benign twin with the signal overlaid only in lookup data, so the free screen is identically distributed and a screen-only policy STRUCTURALLY cannot tell them apart. The probe ladder measures it: `fast` catches 0.0 of hidden seeds. That is the counterweight made real rather than asserted. Twelve policies, thirteen ladder assertions, a genuine three-way trade: fast 0.846 wins hours (0.85), misses every hidden case targeted 0.894 wins the shipped total thorough 0.820 wins evidence (1.00), spends 2.9 hours None dominates. 92 Python tests, 35 TypeScript tests, 65 fixtures replaying at delta 0, and conformance gated on world + scorer + protocol so the browser shows the same alert for ?seed= that Python generated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
95 lines
3.4 KiB
Python
95 lines
3.4 KiB
Python
"""Worked examples of the reward, and the marker region the page quotes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from alert_triage.reward import (
|
|
FALSE_ESCALATION_CREDIT,
|
|
WEIGHTS,
|
|
Episode,
|
|
escalate_break_even,
|
|
metrics,
|
|
score,
|
|
score_exact,
|
|
total,
|
|
)
|
|
|
|
|
|
def ep(**kw) -> Episode:
|
|
base = dict(label="benign", planted=[["kyc.a"]], disposition="close", cites=["kyc.a"], minutes_spent=33,
|
|
reference_minutes=33, turns_spent=1, rejected=0)
|
|
base.update(kw)
|
|
return Episode(**base)
|
|
|
|
|
|
def test_weights_sum_to_one_and_have_a_counterweight() -> None:
|
|
assert abs(sum(WEIGHTS.values()) - 1.0) < 1e-12
|
|
from alert_triage.reward import ROLES
|
|
assert "counterweight" in ROLES.values()
|
|
|
|
|
|
def test_perfect_close() -> None:
|
|
assert score(ep()) == {"caught": 1.0, "hours": 1.0, "evidence": 1.0}
|
|
assert abs(total(ep()) - 1.0) < 1e-12
|
|
|
|
|
|
def test_a_miss_scores_exactly_zero() -> None:
|
|
e = ep(label="suspicious", planted=[["T-1", "T-2"]], disposition="close", cites=["kyc.a"])
|
|
assert score(e) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
|
assert total(e) == 0.0
|
|
|
|
|
|
def test_a_false_escalation_takes_half_the_counterweight_and_nothing_else() -> None:
|
|
e = ep(disposition="escalate", cites=["kyc.a"])
|
|
assert score(e) == {"caught": FALSE_ESCALATION_CREDIT, "hours": 0.0, "evidence": 0.0}
|
|
|
|
|
|
def test_no_disposition_is_zero_and_truncated_is_none() -> None:
|
|
assert score(ep(disposition=None, cites=[])) == {"caught": 0.0, "hours": 0.0, "evidence": 0.0}
|
|
assert score(ep(truncated=True)) == {"caught": None, "hours": None, "evidence": None}
|
|
assert total(ep(truncated=True)) is None
|
|
|
|
|
|
def test_hours_is_capped_and_ratio_against_the_reference() -> None:
|
|
assert score(ep(minutes_spent=20))["hours"] == 1.0
|
|
assert abs(score(ep(minutes_spent=66))["hours"] - 0.5) < 1e-12
|
|
assert score(ep(reference_minutes=None))["hours"] is None
|
|
assert total(ep(reference_minutes=None)) is None
|
|
|
|
|
|
def test_evidence_is_f1_over_the_best_alternate() -> None:
|
|
e = ep(planted=[["a", "b"], ["a"]], cites=["a"])
|
|
assert score(e)["evidence"] == 1.0
|
|
e = ep(planted=[["a", "b"]], cites=["a", "x", "y"])
|
|
assert abs(score(e)["evidence"] - 0.4) < 1e-12 # 2·1 / (3+2)
|
|
e = ep(planted=[["a"]], cites=["a", "b", "c", "d", "e", "f", "g", "h"])
|
|
assert abs(score(e)["evidence"] - 2 / 9) < 1e-12 # one planted among eight cited
|
|
assert score_exact(e)["evidence"] == [2, 9]
|
|
|
|
|
|
def test_exact_and_float_scores_agree() -> None:
|
|
for e in (ep(), ep(minutes_spent=66), ep(planted=[["a", "b"]], cites=["a", "x", "y"]), ep(disposition="escalate")):
|
|
exact = score_exact(e)
|
|
approx = score(e)
|
|
for k in approx:
|
|
assert abs(exact[k][0] / exact[k][1] - approx[k]) < 1e-12
|
|
|
|
|
|
def test_break_even_is_between_zero_and_one() -> None:
|
|
assert 0.3 < escalate_break_even() < 0.7
|
|
|
|
|
|
def test_metrics_are_diagnostics() -> None:
|
|
m = metrics(ep(disposition="escalate", typology="UNKNOWN", true_typology="UNKNOWN", label="suspicious"))
|
|
assert m["typology_match"] == 1.0 and m["false_escalation"] == 0.0
|
|
assert metrics(ep())["typology_match"] is None
|
|
|
|
|
|
def test_exactly_one_marker_pair() -> None:
|
|
text = (Path(__file__).parent.parent / "alert_triage" / "reward.py").read_text()
|
|
assert text.count("# region: pig-demo/reward") == 1
|
|
assert text.count("# endregion: pig-demo/reward") == 1
|