/** * The reasoning trap, pinned. * * This is the one defect in the harness swap that cost real money and produced * nothing at all. `createAgentSession` defaults `thinkingLevel` to `medium`, * which is tuned for a coding agent; asked "what is our utilisation?", the * default model spent 6,195 output tokens reasoning and returned an EMPTY * answer with `finish_reason: length`. Reasoning bills as output, so the turn * was billed in full for nothing. `low` was worse. The fix is two halves and * BOTH are needed: * * 1. `PIGGY_AGENT_THINKING` defaults to `off` (apps/piggy/src/config.ts:71). * 2. The default model carries a `thinkingLevelMap` mapping `off` to the * literal `"none"` (apps/piggy/src/agent/models.json:22-30). * * Half two is the half nobody would guess, and it is why this file exists. In * `@earendil-works/pi-ai@0.84.1`, `streamSimple` turns a thinking level of * `off` into `reasoningEffort: undefined` * (dist/api/openai-completions.js:473-474), and the request builder then reads: * * else if (!options?.reasoningEffort && model.reasoning && compat.supportsReasoningEffort) { * const offValue = model.thinkingLevelMap?.off; * if (typeof offValue === "string") { params.reasoning_effort = offValue; } * } * — dist/api/openai-completions.js:661-666 * * So without a map, `off` OMITS `reasoning_effort` from the request entirely * and the endpoint's own default — thinking ON, verbosely — wins. With the map, * the request carries `reasoning_effort: "none"` and the same question answers * in 149 output tokens. Nothing about the omission is visible in TypeScript, in * the configuration, or in a passing test suite: the only symptom is a blank * reply and a bill. * * The behaviour is per-model, so the assertions below are anchored to whichever * model is the default rather than to nemotron by name. A future default that * needs its own mapping fails here rather than in production. */ import assert from 'node:assert/strict'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test, { after, before } from 'node:test'; import { defineTool, type ToolDefinition } from '@earendil-works/pi-coding-agent'; import { Type } from 'typebox'; import { piggyDefaultModelId } from '../src/agent/models'; import { loadPiggyConfig } from '../src/config'; const agentDir = mkdtempSync(join(tmpdir(), 'piggy-thinking-test-')); /** * A level that is NOT the shipped default, on purpose. * * `off` is what production runs at, and asserting that a session is at `off` * when the default is also `off` proves nothing — it passes just as happily if * the level is dropped on the floor and the harness's own default is `off` one * day. Setting `high` here means the assertion can only pass if the configured * value genuinely reached the session. */ const CONFIGURED_LEVEL = 'high'; /** Far above any model's own ceiling, to prove the clamp is real. */ const ABSURD_TOKEN_BUDGET = '999999'; before(() => { process.env.DATABASE_URL = 'postgres://pig:pig@localhost:54330/pig'; process.env.PIGGY_INTERNAL_TOKEN = 'test-internal-token-for-piggy-000000'; process.env.PRIME_API_KEY = 'test-key-not-used-offline'; process.env.PIGGY_AGENT_DIR = agentDir; process.env.PIGGY_AGENT_THINKING = CONFIGURED_LEVEL; process.env.PIGGY_AGENT_MAX_TOKENS = ABSURD_TOKEN_BUDGET; }); after(() => { rmSync(agentDir, { recursive: true, force: true }); }); /** The seven levels `PIGGY_AGENT_THINKING` accepts, per apps/piggy/src/config.ts:70. */ const CONFIGURABLE_LEVELS = [ 'off', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', ] as const; /** The OpenAI-style efforts a `reasoning_effort` field may carry. */ const EFFORTS = ['none', 'minimal', 'low', 'medium', 'high']; interface ShippedModel { id: string; reasoning: boolean; maxTokens: number; thinkingLevelMap?: Record; } interface ModelsDocument { providers: Record; } /** * The shipped file, read from disk rather than imported. * * `models.ts` validates and reshapes it, and `thinkingLevelMap` is deliberately * not part of that reshaping — the harness reads it, PIG never does. So the * only honest place to assert it is the bytes that are copied into the agent * directory and handed to `ModelRuntime.create`. */ const document = JSON.parse( readFileSync(fileURLToPath(new URL('../src/agent/models.json', import.meta.url)), 'utf8'), ) as ModelsDocument; const shippedModels = document.providers['prime-inference']?.models ?? []; function shipped(id: string): ShippedModel { const model = shippedModels.find((candidate) => candidate.id === id); assert.ok(model, `${id} is not registered in models.json`); return model; } function piggyTool(name: string): ToolDefinition { return defineTool({ name, label: name, description: `Test double for ${name}.`, promptSnippet: `${name}: test double.`, parameters: Type.Object({}), async execute() { return { content: [{ type: 'text' as const, text: '{}' }], details: {} }; }, }); } test('the default model maps every configurable thinking level to an explicit effort', () => { const model = shipped(piggyDefaultModelId()); const map = model.thinkingLevelMap; assert.ok( map, `${model.id} is the default model and has no thinkingLevelMap, so at thinking level off the ` + `request carries no reasoning_effort at all and the endpoint's own default decides how ` + `hard it thinks. That is the 6,195-token empty answer.`, ); // `off` is the one that was measured, and the one production runs at. assert.equal(map.off, 'none'); for (const level of CONFIGURABLE_LEVELS) { const mapped: string | null | undefined = map[level]; // A `null` would remove the level from the picker; `undefined` would fall // through to `?? options.reasoningEffort` and send the harness's own word // for the level, which is not one this endpoint answers to. assert.equal(typeof mapped, 'string', `thinking level ${level} is not mapped to an effort`); assert.ok( EFFORTS.includes(String(mapped)), `${level} maps to ${mapped}, which is not a reasoning effort`, ); } }); test('the shipped default configuration is the level that was measured', () => { // Read from a bare environment rather than from `process.env`, which this // file has deliberately set to something else. const config = loadPiggyConfig({ DATABASE_URL: 'postgres://pig:pig@localhost:54330/pig', PRIME_API_KEY: 'test-key', PIGGY_INTERNAL_TOKEN: 'test-internal-token-for-piggy-000000', }); assert.equal(config.PIGGY_AGENT_THINKING, 'off'); // And the level the deployment actually runs at is one the default model has // an explicit answer for. This is the pairing: either half alone is silent. assert.equal(shipped(config.PIGGY_AGENT_MODEL).thinkingLevelMap?.[config.PIGGY_AGENT_THINKING], 'none'); }); test('the default is a model that pins its own reasoning effort', () => { // Three of the five are left to the endpoint's default deliberately: they are // frontier models whose defaults are sane and whose budgets are large. The // default model is not one of those, and swapping the default to a model with // no map would reintroduce the exact failure this file documents. const pinned = shippedModels.filter((model) => model.thinkingLevelMap).map((model) => model.id); assert.ok(pinned.length > 0); assert.ok( pinned.includes(piggyDefaultModelId()), `${piggyDefaultModelId()} is the default and does not pin its reasoning effort; only ` + `${pinned.join(', ')} do.`, ); }); test('the configured thinking level reaches the session, and the map reaches the model', async () => { const { createPiggySession } = await import('../src/agent/session'); const piggy = await createPiggySession({ mode: 'read_only', tools: [piggyTool('pig_get_workspace_summary')], }); try { // The harness would otherwise answer at `medium`, which is where the money // went. `session.thinkingLevel` is what the next request is built from. assert.equal(piggy.session.thinkingLevel, CONFIGURED_LEVEL); assert.equal(piggy.session.agent.state.thinkingLevel, CONFIGURED_LEVEL); // And the map survived `ModelRuntime.create` → `getModel` → the model // override `createPiggySession` builds. It is dropped in silence if it does // not: the model still resolves, still answers, and still thinks. const model = piggy.session.agent.state.model; assert.equal(model.id, piggyDefaultModelId()); assert.equal(model.thinkingLevelMap?.off, 'none'); assert.equal(model.thinkingLevelMap?.[CONFIGURED_LEVEL], 'high'); } finally { piggy.dispose(); } }); test('the per-turn budget cannot ask for more than the model will return', async () => { const { createPiggySession } = await import('../src/agent/session'); const piggy = await createPiggySession({ mode: 'read_only', tools: [piggyTool('pig_get_workspace_summary')], }); try { // Reasoning and the answer share this budget. Asking for more than the // endpoint will give is not a bigger budget, it is a 400 on every turn. const ceiling = shipped(piggyDefaultModelId()).maxTokens; assert.equal(piggy.session.agent.state.model.maxTokens, ceiling); assert.ok(ceiling < Number(ABSURD_TOKEN_BUDGET)); } finally { piggy.dispose(); } });