karti-ai
2ef62c5670
32 recorded rollouts, 8 seeds x 4 agents, every one replaying through the Python engine to a delta of exactly 0. arm solved economy consistency total base-off 0/8 0.000 0.122 0.0244 base-on 4/8 0.469 0.479 0.4865 cautious 8/8 0.944 1.000 0.9831 solver 8/8 1.000 0.719 0.9437 Thinking-off solves none of eight. The same model on the same seeds, sampled with thinking on, solves four. That is the headline, it is ours, and it needed no training — which is also why the run is labelled an intervention and names what was done to it, so a sampling change can never read as a training result. `cautious` is new and it exists to make the reward editor honest. Until now the page invited you to move a slider and watch the ranking change, and no slider changed anything, because the solver dominated a model that solved nothing. A candidate-only player — most informative guess among words that could still win — takes consistency outright and pays for it in turns. Now: only speed matters solver 0.9859 cautious 0.9606 -> solver shipped weights solver 0.9437 cautious 0.9831 -> cautious punish contradictions solver 0.8312 cautious 0.9972 -> cautious The ranking really does flip, on recorded data, with one slider. The first version of this arm picked the alphabetically-first candidate and opened on 'abaci', which made the policy the counterweight exists to reward look like a straw man. 190 contract checks pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
99 lines
2.2 KiB
JSON
99 lines
2.2 KiB
JSON
{
|
|
"runId": "cautious-s0",
|
|
"seed": 0,
|
|
"model": "candidate-only-solver",
|
|
"capturedAt": "2026-08-28",
|
|
"rewards": {
|
|
"solved": 1.0,
|
|
"economy": 0.8,
|
|
"consistency": 1.0
|
|
},
|
|
"metrics": {
|
|
"guesses_used": 5.0,
|
|
"rejected_replies": 0.0,
|
|
"reference_depth": 4.0,
|
|
"turns_granted": 6.0,
|
|
"inconsistent_guesses": 0.0
|
|
},
|
|
"truncated": false,
|
|
"outcome": "solved",
|
|
"answer": "wants",
|
|
"turns": [
|
|
{
|
|
"reply": "[tares]",
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": null,
|
|
"finishReason": "generated"
|
|
},
|
|
"info": {
|
|
"guess": "tares",
|
|
"observation": "T A R E S\nY G X X G\n\nYou have 5 guesses left."
|
|
}
|
|
},
|
|
{
|
|
"reply": "[casts]",
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": null,
|
|
"finishReason": "generated"
|
|
},
|
|
"info": {
|
|
"guess": "casts",
|
|
"observation": "C A S T S\nX G X G G\n\nYou have 4 guesses left."
|
|
}
|
|
},
|
|
{
|
|
"reply": "[waits]",
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": null,
|
|
"finishReason": "generated"
|
|
},
|
|
"info": {
|
|
"guess": "waits",
|
|
"observation": "W A I T S\nG G X G G\n\nYou have 3 guesses left."
|
|
}
|
|
},
|
|
{
|
|
"reply": "[wafts]",
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": null,
|
|
"finishReason": "generated"
|
|
},
|
|
"info": {
|
|
"guess": "wafts",
|
|
"observation": "W A F T S\nG G X G G\n\nYou have 2 guesses left."
|
|
}
|
|
},
|
|
{
|
|
"reply": "[wants]",
|
|
"reasoning": null,
|
|
"call": {
|
|
"promptTokens": null,
|
|
"completionTokens": null,
|
|
"reasoningTokens": null,
|
|
"durationMs": null,
|
|
"finishReason": "generated"
|
|
},
|
|
"info": {
|
|
"guess": "wants",
|
|
"observation": "W A N T S\nG G G G G\n\nYou have 1 guess left."
|
|
}
|
|
}
|
|
]
|
|
}
|