pig/apps/piggy/e2e/prime-agent.test.ts

/**
 * One real turn against Prime Inference, to pin the thing money bought.
 *
 * Everything in `test/` runs offline, and everything in `test/` would have
 * passed on the day Piggy answered every question with an empty string: the
 * harness defaulted `thinkingLevel` to `medium`, the default model spent 6,195
 * output tokens reasoning, hit `finish_reason: length`, and returned nothing.
 * The configuration was valid, the tools were correct, the types checked. The
 * only way to see it is to ask a model a question and count the tokens.
 *
 * So this suite does exactly that, once, on the cheapest model in the
 * catalogue, and asserts the three properties that failure violated:
 *
 *   - the answer is not empty, and was not cut off by the budget;
 *   - the reasoning did not eat the turn (149 output tokens was the measurement
 *     after the fix, against 6,195 before it);
 *   - the tool was actually called, rather than the figures being invented.
 *
 * It is opt-in twice over — a key AND `PIGGY_E2E_LIVE=1` — because a suite that
 * spends money whenever the environment happens to be loaded is a suite that
 * spends money by accident. A turn costs about $0.0003.
 *
 *   PIGGY_E2E_LIVE=1 PRIME_API_KEY=... pnpm -F @pig/piggy run test:e2e
 */
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test, { after, before } from 'node:test';
import { defineTool, type AgentSessionEvent } from '@earendil-works/pi-coding-agent';
import { Type } from 'typebox';

const live = process.env.PIGGY_E2E_LIVE === '1' && Boolean(process.env.PRIME_API_KEY);

if (!live) {
  test.skip('the live Prime Agent E2E needs PIGGY_E2E_LIVE=1 and PRIME_API_KEY; it spends credit');
}

const agentDir = mkdtempSync(join(tmpdir(), 'piggy-live-e2e-'));

before(() => {
  // The session only needs the key; these two are required by the config schema
  // and are never read on this path.
  process.env.DATABASE_URL ??= 'postgres://pig:pig@localhost:54330/pig';
  process.env.PIGGY_INTERNAL_TOKEN ??= 'test-internal-token-for-piggy-000000';
  process.env.PIGGY_AGENT_DIR = agentDir;
});

after(() => {
  rmSync(agentDir, { recursive: true, force: true });
});

/**
 * The figures are the two that were misread in production.
 *
 * 189 has to be spoken as $1.89 and 112 as $1.12 — the units rule in the system
 * prompt exists because a small model says "$189 per GPU-hour" and "112 cents"
 * otherwise, and both readings are confidently, catastrophically wrong.
 */
const SUMMARY = {
  headline: 'Northwind Robotics H100 block, 38% sold',
  committedGpuHours: 52_000,
  allocatedGpuHours: 19_760,
  utilisation: 0.38,
  costPerGpuHourCents: 189,
  breakEvenPriceCents: 112,
  idleCostCents: 1_200_000,
};

/** Usage off a `turn_end` message, without widening anything to `any`. */
function outputTokens(event: AgentSessionEvent): number {
  if (event.type !== 'turn_end') return 0;
  const message: unknown = event.message;
  if (typeof message !== 'object' || message === null) return 0;
  const usage = (message as { usage?: { output?: unknown } }).usage;
  return typeof usage?.output === 'number' ? usage.output : 0;
}

function stopReason(event: AgentSessionEvent): string | undefined {
  if (event.type !== 'turn_end') return undefined;
  const message: unknown = event.message;
  if (typeof message !== 'object' || message === null) return undefined;
  const reason = (message as { stopReason?: unknown }).stopReason;
  return typeof reason === 'string' ? reason : undefined;
}

test('a real turn answers, calls its tool, and does not think itself out of a reply', { skip: !live }, async () => {
  const { createPiggySession } = await import('../src/agent/session');

  let toolCalls = 0;
  const tool = defineTool({
    name: 'pig_get_workspace_summary',
    label: 'Workspace summary',
    description: 'Returns the workspace-wide capacity aggregates, already computed.',
    promptSnippet: 'Workspace-wide capacity aggregates, already computed',
    parameters: Type.Object({}),
    async execute() {
      toolCalls += 1;
      return {
        content: [{ type: 'text' as const, text: JSON.stringify(SUMMARY) }],
        details: {},
      };
    },
  });

  const piggy = await createPiggySession({ mode: 'read_only', tools: [tool] });
  let answer = '';
  let spent = 0;
  let finish: string | undefined;

  const unsubscribe = piggy.session.subscribe((event) => {
    if (event.type === 'message_update' && event.assistantMessageEvent.type === 'text_delta') {
      answer += event.assistantMessageEvent.delta;
    }
    spent += outputTokens(event);
    finish = stopReason(event) ?? finish;
  });

  try {
    await piggy.session.prompt(
      'What is the break-even price per GPU-hour on this block, and how much has the idle ' +
        'capacity already cost? Use the tool.',
    );
    await piggy.session.waitForIdle();
  } finally {
    unsubscribe();
    piggy.dispose();
  }

  assert.equal(toolCalls > 0, true, 'the model answered without calling the tool');
  assert.ok(answer.trim().length > 0, 'the model returned an empty answer');
  // `length` is the signature of the failure: the budget was spent before a
  // single token of the answer was written.
  assert.notEqual(finish, 'length');
  // 149 output tokens after the fix; 6,195 before it. The bound is generous
  // enough that ordinary variation cannot trip it and tight enough that a
  // reasoning regression cannot hide under it.
  assert.ok(spent > 0 && spent < 1_500, `the turn spent ${spent} output tokens`);
  // Not a check on the model's prose: a check that the units rule survived. A
  // cents-denominated money figure is the one output that is arithmetically
  // correct and commercially useless.
  assert.doesNotMatch(answer, /\b112\s*(cents|c)\b/i);
});