pig/apps/piggy/test/agent-thinking.test.ts

/**
 * The reasoning trap, pinned.
 *
 * This is the one defect in the harness swap that cost real money and produced
 * nothing at all. `createAgentSession` defaults `thinkingLevel` to `medium`,
 * which is tuned for a coding agent; asked "what is our utilisation?", the
 * default model spent 6,195 output tokens reasoning and returned an EMPTY
 * answer with `finish_reason: length`. Reasoning bills as output, so the turn
 * was billed in full for nothing. `low` was worse. The fix is two halves and
 * BOTH are needed:
 *
 *   1. `PIGGY_AGENT_THINKING` defaults to `off` (apps/piggy/src/config.ts:71).
 *   2. The default model carries a `thinkingLevelMap` mapping `off` to the
 *      literal `"none"` (apps/piggy/src/agent/models.json:22-30).
 *
 * Half two is the half nobody would guess, and it is why this file exists. In
 * `@earendil-works/pi-ai@0.84.1`, `streamSimple` turns a thinking level of
 * `off` into `reasoningEffort: undefined`
 * (dist/api/openai-completions.js:473-474), and the request builder then reads:
 *
 *     else if (!options?.reasoningEffort && model.reasoning && compat.supportsReasoningEffort) {
 *         const offValue = model.thinkingLevelMap?.off;
 *         if (typeof offValue === "string") { params.reasoning_effort = offValue; }
 *     }
 *                                        — dist/api/openai-completions.js:661-666
 *
 * So without a map, `off` OMITS `reasoning_effort` from the request entirely
 * and the endpoint's own default — thinking ON, verbosely — wins. With the map,
 * the request carries `reasoning_effort: "none"` and the same question answers
 * in 149 output tokens. Nothing about the omission is visible in TypeScript, in
 * the configuration, or in a passing test suite: the only symptom is a blank
 * reply and a bill.
 *
 * The behaviour is per-model, so the assertions below are anchored to whichever
 * model is the default rather than to nemotron by name. A future default that
 * needs its own mapping fails here rather than in production.
 */
import assert from 'node:assert/strict';
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import test, { after, before } from 'node:test';
import { defineTool, type ToolDefinition } from '@earendil-works/pi-coding-agent';
import { Type } from 'typebox';
import { piggyDefaultModelId } from '../src/agent/models';
import { loadPiggyConfig } from '../src/config';

const agentDir = mkdtempSync(join(tmpdir(), 'piggy-thinking-test-'));

/**
 * A level that is NOT the shipped default, on purpose.
 *
 * `off` is what production runs at, and asserting that a session is at `off`
 * when the default is also `off` proves nothing — it passes just as happily if
 * the level is dropped on the floor and the harness's own default is `off` one
 * day. Setting `high` here means the assertion can only pass if the configured
 * value genuinely reached the session.
 */
const CONFIGURED_LEVEL = 'high';

/** Far above any model's own ceiling, to prove the clamp is real. */
const ABSURD_TOKEN_BUDGET = '999999';

before(() => {
  process.env.DATABASE_URL = 'postgres://pig:pig@localhost:54330/pig';
  process.env.PIGGY_INTERNAL_TOKEN = 'test-internal-token-for-piggy-000000';
  process.env.PRIME_API_KEY = 'test-key-not-used-offline';
  process.env.PIGGY_AGENT_DIR = agentDir;
  process.env.PIGGY_AGENT_THINKING = CONFIGURED_LEVEL;
  process.env.PIGGY_AGENT_MAX_TOKENS = ABSURD_TOKEN_BUDGET;
});

after(() => {
  rmSync(agentDir, { recursive: true, force: true });
});

/** The seven levels `PIGGY_AGENT_THINKING` accepts, per apps/piggy/src/config.ts:70. */
const CONFIGURABLE_LEVELS = [
  'off',
  'minimal',
  'low',
  'medium',
  'high',
  'xhigh',
  'max',
] as const;

/** The OpenAI-style efforts a `reasoning_effort` field may carry. */
const EFFORTS = ['none', 'minimal', 'low', 'medium', 'high'];

interface ShippedModel {
  id: string;
  reasoning: boolean;
  maxTokens: number;
  thinkingLevelMap?: Record<string, string | null | undefined>;
}

interface ModelsDocument {
  providers: Record<string, { models: ShippedModel[] }>;
}

/**
 * The shipped file, read from disk rather than imported.
 *
 * `models.ts` validates and reshapes it, and `thinkingLevelMap` is deliberately
 * not part of that reshaping — the harness reads it, PIG never does. So the
 * only honest place to assert it is the bytes that are copied into the agent
 * directory and handed to `ModelRuntime.create`.
 */
const document = JSON.parse(
  readFileSync(fileURLToPath(new URL('../src/agent/models.json', import.meta.url)), 'utf8'),
) as ModelsDocument;
const shippedModels = document.providers['prime-inference']?.models ?? [];

function shipped(id: string): ShippedModel {
  const model = shippedModels.find((candidate) => candidate.id === id);
  assert.ok(model, `${id} is not registered in models.json`);
  return model;
}

function piggyTool(name: string): ToolDefinition {
  return defineTool({
    name,
    label: name,
    description: `Test double for ${name}.`,
    promptSnippet: `${name}: test double.`,
    parameters: Type.Object({}),
    async execute() {
      return { content: [{ type: 'text' as const, text: '{}' }], details: {} };
    },
  });
}

test('the default model maps every configurable thinking level to an explicit effort', () => {
  const model = shipped(piggyDefaultModelId());
  const map = model.thinkingLevelMap;

  assert.ok(
    map,
    `${model.id} is the default model and has no thinkingLevelMap, so at thinking level off the ` +
      `request carries no reasoning_effort at all and the endpoint's own default decides how ` +
      `hard it thinks. That is the 6,195-token empty answer.`,
  );
  // `off` is the one that was measured, and the one production runs at.
  assert.equal(map.off, 'none');
  for (const level of CONFIGURABLE_LEVELS) {
    const mapped: string | null | undefined = map[level];
    // A `null` would remove the level from the picker; `undefined` would fall
    // through to `?? options.reasoningEffort` and send the harness's own word
    // for the level, which is not one this endpoint answers to.
    assert.equal(typeof mapped, 'string', `thinking level ${level} is not mapped to an effort`);
    assert.ok(
      EFFORTS.includes(String(mapped)),
      `${level} maps to ${mapped}, which is not a reasoning effort`,
    );
  }
});

test('the shipped default configuration is the level that was measured', () => {
  // Read from a bare environment rather than from `process.env`, which this
  // file has deliberately set to something else.
  const config = loadPiggyConfig({
    DATABASE_URL: 'postgres://pig:pig@localhost:54330/pig',
    PRIME_API_KEY: 'test-key',
    PIGGY_INTERNAL_TOKEN: 'test-internal-token-for-piggy-000000',
  });

  assert.equal(config.PIGGY_AGENT_THINKING, 'off');
  // And the level the deployment actually runs at is one the default model has
  // an explicit answer for. This is the pairing: either half alone is silent.
  assert.equal(shipped(config.PIGGY_AGENT_MODEL).thinkingLevelMap?.[config.PIGGY_AGENT_THINKING], 'none');
});

test('the default is a model that pins its own reasoning effort', () => {
  // Three of the five are left to the endpoint's default deliberately: they are
  // frontier models whose defaults are sane and whose budgets are large. The
  // default model is not one of those, and swapping the default to a model with
  // no map would reintroduce the exact failure this file documents.
  const pinned = shippedModels.filter((model) => model.thinkingLevelMap).map((model) => model.id);

  assert.ok(pinned.length > 0);
  assert.ok(
    pinned.includes(piggyDefaultModelId()),
    `${piggyDefaultModelId()} is the default and does not pin its reasoning effort; only ` +
      `${pinned.join(', ')} do.`,
  );
});

test('the configured thinking level reaches the session, and the map reaches the model', async () => {
  const { createPiggySession } = await import('../src/agent/session');
  const piggy = await createPiggySession({
    mode: 'read_only',
    tools: [piggyTool('pig_get_workspace_summary')],
  });

  try {
    // The harness would otherwise answer at `medium`, which is where the money
    // went. `session.thinkingLevel` is what the next request is built from.
    assert.equal(piggy.session.thinkingLevel, CONFIGURED_LEVEL);
    assert.equal(piggy.session.agent.state.thinkingLevel, CONFIGURED_LEVEL);

    // And the map survived `ModelRuntime.create` → `getModel` → the model
    // override `createPiggySession` builds. It is dropped in silence if it does
    // not: the model still resolves, still answers, and still thinks.
    const model = piggy.session.agent.state.model;
    assert.equal(model.id, piggyDefaultModelId());
    assert.equal(model.thinkingLevelMap?.off, 'none');
    assert.equal(model.thinkingLevelMap?.[CONFIGURED_LEVEL], 'high');
  } finally {
    piggy.dispose();
  }
});

test('the per-turn budget cannot ask for more than the model will return', async () => {
  const { createPiggySession } = await import('../src/agent/session');
  const piggy = await createPiggySession({
    mode: 'read_only',
    tools: [piggyTool('pig_get_workspace_summary')],
  });

  try {
    // Reasoning and the answer share this budget. Asking for more than the
    // endpoint will give is not a bigger budget, it is a 400 on every turn.
    const ceiling = shipped(piggyDefaultModelId()).maxTokens;
    assert.equal(piggy.session.agent.state.model.maxTokens, ceiling);
    assert.ok(ceiling < Number(ABSURD_TOKEN_BUDGET));
  } finally {
    piggy.dispose();
  }
});