Alert Triage: the thinking-on model arm, all eight seeds
base-on solved 6/8 with 2 aborted — the model ran out its eight-turn budget without reaching a disposition, which scores exactly 0.000 and is recorded as such rather than dropped. base-off solved 7/8. Both arms sit below the scripted policies on the shipped weights, and above them on nothing, which is the honest shape of a model that has not been trained on this task. 72 fixtures across both environments replay at delta 0.
This commit is contained in:
karti-ai
@@ -0,0 +1,173 @@
|
||||
{
|
||||
"runId": "base-on-s9",
|
||||
"seed": 9,
|
||||
"model": "brain-qwen38-dspark",
|
||||
"capturedAt": "2026-08-28",
|
||||
"rewards": {
|
||||
"caught": 0.0,
|
||||
"hours": 0.0,
|
||||
"evidence": 0.0
|
||||
},
|
||||
"metrics": {
|
||||
"hours_spent": 0.9,
|
||||
"reference_hours": 2.05,
|
||||
"lookups": 0.0,
|
||||
"turns_used": 8.0,
|
||||
"rejected_replies": 8.0,
|
||||
"false_escalation": 0.0,
|
||||
"missed": 0.0,
|
||||
"typology_match": null
|
||||
},
|
||||
"truncated": false,
|
||||
"outcome": "aborted",
|
||||
"info": {
|
||||
"label": "suspicious",
|
||||
"template": "B3",
|
||||
"overlay": "funnel",
|
||||
"typology": "FUNNEL_ACCOUNT",
|
||||
"planted": [
|
||||
[
|
||||
"CP-3",
|
||||
"T-52"
|
||||
],
|
||||
[
|
||||
"T-52"
|
||||
]
|
||||
],
|
||||
"reference_minutes": 123,
|
||||
"reference_policy": "targeted",
|
||||
"held_out": false,
|
||||
"tier": "hidden",
|
||||
"screen": "ALERT A-9537 rule R-VEL-02 \u2014 Funds received and moved out again within 48 hours\n fired 2026-04-19\n triggering transactions:\n T-51 2026-04-16 wire_in 462,000.00 USD CP-2\n T-52 2026-04-17 wire_out 448,100.00 USD CP-3\nKYC FILE C-9673 Samir Tanaka\n kyc.account_opened = 2019-12-21\n kyc.country = United States\n kyc.customer_type = individual\n kyc.expected_activity = Salaried individual; payroll deposits and routine spending.\n kyc.expected_monthly_cash = 0.00 USD\n kyc.kyc_refreshed = 2025-04-04\n kyc.occupation = electrician\n kyc.pep = False\n kyc.prior_alerts = 4\n kyc.prior_sars = 1\n kyc.remittance_declared = none declared\n kyc.stated_income = 84,000.00 USD\nTWELVE-MONTH SUMMARY (USD)\n month cash in cash out wires in wires out ach in ach out txns\n 2025-05 0 1,090 0 0 6,958 0 5\n 2025-06 0 630 0 0 7,009 0 5\n 2025-07 0 780 0 0 6,957 0 4\n 2025-08 0 470 0 0 6,989 0 3\n 2025-09 0 1,040 0 0 6,959 0 5\n 2025-10 0 740 0 0 6,996 0 4\n 2025-11 0 860 0 0 6,947 0 5\n 2025-12 0 900 0 0 6,940 0 5\n 2026-01 0 650 0 0 7,025 0 3\n 2026-02 0 460 0 0 7,026 0 3\n 2026-03 0 620 0 0 6,962 0 3\n 2026-04 0 960 462,000 448,100 6,944 0 7\nCOUNTERPARTIES on screen (top by twelve-month volume):\n CP-2 in 1 txns 462,000.00 USD\n CP-3 out 1 txns 448,100.00 USD\nReporting threshold in this jurisdiction: 10,000.00 USD\nSYNTHETIC DATA \u2014 no real customer, account or transaction.\n\nHours spent so far: 0.50. You have 8 replies left."
|
||||
},
|
||||
"turns": [
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 7 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 6 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 5 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 4 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 3 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 2 replies left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 1 reply left."
|
||||
}
|
||||
},
|
||||
{
|
||||
"reply": null,
|
||||
"reasoning": null,
|
||||
"call": {
|
||||
"promptTokens": null,
|
||||
"completionTokens": null,
|
||||
"reasoningTokens": null,
|
||||
"durationMs": 0,
|
||||
"finishReason": "error: URLError"
|
||||
},
|
||||
"info": {
|
||||
"action": null,
|
||||
"rejection": "no JSON object found in the reply",
|
||||
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 0 replies left."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user