karti-ai
af15ba584e
base-on solved 6/8 with 2 aborted — the model ran out its eight-turn budget without reaching a disposition, which scores exactly 0.000 and is recorded as such rather than dropped. base-off solved 7/8. Both arms sit below the scripted policies on the shipped weights, and above them on nothing, which is the honest shape of a model that has not been trained on this task. 72 fixtures across both environments replay at delta 0.
170 lines
7.0 KiB
JSON
170 lines
7.0 KiB
JSON
{
|
|
"runId": "base-on-s7",
|
|
"seed": 7,
|
|
"model": "brain-qwen38-dspark",
|
|
"capturedAt": "2026-08-28",
|
|
"rewards": {
|
|
"caught": 0.0,
|
|
"hours": 0.0,
|
|
"evidence": 0.0
|
|
},
|
|
"metrics": {
|
|
"hours_spent": 0.9,
|
|
"reference_hours": 0.7,
|
|
"lookups": 0.0,
|
|
"turns_used": 8.0,
|
|
"rejected_replies": 8.0,
|
|
"false_escalation": 0.0,
|
|
"missed": 0.0,
|
|
"typology_match": null
|
|
},
|
|
"truncated": false,
|
|
"outcome": "aborted",
|
|
"info": {
|
|
"label": "benign",
|
|
"template": "B6",
|
|
"overlay": null,
|
|
"typology": null,
|
|
"planted": [
|
|
[
|
|
"doc.D-b931"
|
|
]
|
|
],
|
|
"reference_minutes": 42,
|
|
"reference_policy": "fast",
|
|
"held_out": false,
|
|
"tier": "benign",
|
|
"screen": "ALERT A-5644 rule R-DORM-05 \u2014 Large credit to an account with no material activity for nine months\n fired 2026-06-14\n triggering transactions:\n T-4 2026-06-07 wire_in 249,000.00 USD CP-1\n T-5 2026-06-10 wire_out 134,400.00 USD CP-2\nKYC FILE C-1843 Leila Bergstrom\n kyc.account_opened = 2017-03-20\n kyc.country = United States\n kyc.customer_type = individual\n kyc.expected_activity = Retired; low activity expected.\n kyc.expected_monthly_cash = 0.00 USD\n kyc.kyc_refreshed = 2026-01-13\n kyc.occupation = retired\n kyc.pep = False\n kyc.prior_alerts = 0\n kyc.prior_sars = 0\n kyc.remittance_declared = none declared\n kyc.stated_income = 33,000.00 USD\nTWELVE-MONTH SUMMARY (USD)\n month cash in cash out wires in wires out ach in ach out txns\n 2025-07 0 0 0 0 0 0 0\n 2025-08 0 0 0 0 0 0 0\n 2025-09 0 60 0 0 0 0 1\n 2025-10 0 0 0 0 0 0 0\n 2025-11 0 150 0 0 0 0 1\n 2025-12 0 0 0 0 0 0 0\n 2026-01 0 0 0 0 0 0 0\n 2026-02 0 0 0 0 0 0 0\n 2026-03 0 50 0 0 0 0 1\n 2026-04 0 0 0 0 0 0 0\n 2026-05 0 0 0 0 0 0 0\n 2026-06 0 0 249,000 134,400 0 0 2\nCOUNTERPARTIES on screen (top by twelve-month volume):\n CP-1 in 1 txns 249,000.00 USD\n CP-2 out 1 txns 134,400.00 USD\nReporting threshold in this jurisdiction: 10,000.00 USD\nSYNTHETIC DATA \u2014 no real customer, account or transaction.\n\nHours spent so far: 0.50. You have 8 replies left."
|
|
},
|
|
"turns": [
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 36706,
|
|
"finishReason": "error: RemoteDisconnected"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 7 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 1,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 6 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 1,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 5 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 1,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 4 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 1,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 3 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 0,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 2 replies left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 0,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 1 reply left."
|
|
}
|
|
},
|
|
{
|
|
"reply": null,
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": 0,
|
|
"finishReason": "error: URLError"
|
|
},
|
|
"info": {
|
|
"action": null,
|
|
"rejection": "no JSON object found in the reply",
|
|
"observation": "That reply was not accepted: no JSON object found in the reply. It still cost a turn. You have 0 replies left."
|
|
}
|
|
}
|
|
]
|
|
}
|