/** * One real turn against Prime Inference, to pin the thing money bought. * * Everything in `test/` runs offline, and everything in `test/` would have * passed on the day Piggy answered every question with an empty string: the * harness defaulted `thinkingLevel` to `medium`, the default model spent 6,195 * output tokens reasoning, hit `finish_reason: length`, and returned nothing. * The configuration was valid, the tools were correct, the types checked. The * only way to see it is to ask a model a question and count the tokens. * * So this suite does exactly that, once, on the cheapest model in the * catalogue, and asserts the three properties that failure violated: * * - the answer is not empty, and was not cut off by the budget; * - the reasoning did not eat the turn (149 output tokens was the measurement * after the fix, against 6,195 before it); * - the tool was actually called, rather than the figures being invented. * * It is opt-in twice over — a key AND `PIGGY_E2E_LIVE=1` — because a suite that * spends money whenever the environment happens to be loaded is a suite that * spends money by accident. A turn costs about $0.0003. * * PIGGY_E2E_LIVE=1 PRIME_API_KEY=... pnpm -F @pig/piggy run test:e2e */ import assert from 'node:assert/strict'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test, { after, before } from 'node:test'; import { defineTool, type AgentSessionEvent } from '@earendil-works/pi-coding-agent'; import { Type } from 'typebox'; const live = process.env.PIGGY_E2E_LIVE === '1' && Boolean(process.env.PRIME_API_KEY); if (!live) { test.skip('the live Prime Agent E2E needs PIGGY_E2E_LIVE=1 and PRIME_API_KEY; it spends credit'); } const agentDir = mkdtempSync(join(tmpdir(), 'piggy-live-e2e-')); before(() => { // The session only needs the key; these two are required by the config schema // and are never read on this path. process.env.DATABASE_URL ??= 'postgres://pig:pig@localhost:54330/pig'; process.env.PIGGY_INTERNAL_TOKEN ??= 'test-internal-token-for-piggy-000000'; process.env.PIGGY_AGENT_DIR = agentDir; }); after(() => { rmSync(agentDir, { recursive: true, force: true }); }); /** * The figures are the two that were misread in production. * * 189 has to be spoken as $1.89 and 112 as $1.12 — the units rule in the system * prompt exists because a small model says "$189 per GPU-hour" and "112 cents" * otherwise, and both readings are confidently, catastrophically wrong. */ const SUMMARY = { headline: 'Northwind Robotics H100 block, 38% sold', committedGpuHours: 52_000, allocatedGpuHours: 19_760, utilisation: 0.38, costPerGpuHourCents: 189, breakEvenPriceCents: 112, idleCostCents: 1_200_000, }; /** Usage off a `turn_end` message, without widening anything to `any`. */ function outputTokens(event: AgentSessionEvent): number { if (event.type !== 'turn_end') return 0; const message: unknown = event.message; if (typeof message !== 'object' || message === null) return 0; const usage = (message as { usage?: { output?: unknown } }).usage; return typeof usage?.output === 'number' ? usage.output : 0; } function stopReason(event: AgentSessionEvent): string | undefined { if (event.type !== 'turn_end') return undefined; const message: unknown = event.message; if (typeof message !== 'object' || message === null) return undefined; const reason = (message as { stopReason?: unknown }).stopReason; return typeof reason === 'string' ? reason : undefined; } test('a real turn answers, calls its tool, and does not think itself out of a reply', { skip: !live }, async () => { const { createPiggySession } = await import('../src/agent/session'); let toolCalls = 0; const tool = defineTool({ name: 'pig_get_workspace_summary', label: 'Workspace summary', description: 'Returns the workspace-wide capacity aggregates, already computed.', promptSnippet: 'Workspace-wide capacity aggregates, already computed', parameters: Type.Object({}), async execute() { toolCalls += 1; return { content: [{ type: 'text' as const, text: JSON.stringify(SUMMARY) }], details: {}, }; }, }); const piggy = await createPiggySession({ mode: 'read_only', tools: [tool] }); let answer = ''; let spent = 0; let finish: string | undefined; const unsubscribe = piggy.session.subscribe((event) => { if (event.type === 'message_update' && event.assistantMessageEvent.type === 'text_delta') { answer += event.assistantMessageEvent.delta; } spent += outputTokens(event); finish = stopReason(event) ?? finish; }); try { await piggy.session.prompt( 'What is the break-even price per GPU-hour on this block, and how much has the idle ' + 'capacity already cost? Use the tool.', ); await piggy.session.waitForIdle(); } finally { unsubscribe(); piggy.dispose(); } assert.equal(toolCalls > 0, true, 'the model answered without calling the tool'); assert.ok(answer.trim().length > 0, 'the model returned an empty answer'); // `length` is the signature of the failure: the budget was spent before a // single token of the answer was written. assert.notEqual(finish, 'length'); // 149 output tokens after the fix; 6,195 before it. The bound is generous // enough that ordinary variation cannot trip it and tight enough that a // reasoning regression cannot hide under it. assert.ok(spent > 0 && spent < 1_500, `the turn spent ${spent} output tokens`); // Not a check on the model's prose: a check that the units rule survived. A // cents-denominated money figure is the one output that is arithmetically // correct and commercially useless. assert.doesNotMatch(answer, /\b112\s*(cents|c)\b/i); });