/** * The scope contract, pinned. * * This suite exists because of one production answer. Asked "How many capacity * commitments are on the book?" on /capacity, Piggy called * `pig_get_idle_capacity` — the only tool that page offers — and said "3". The * book held 5. The tool filters to blocks at least 25% unsold, so 3 was the * size of a filter, and the payload gave the model nothing else to read: the * length of the list it had been handed was the only count in front of it. * * The system prompt already forbade that, naming this exact tool. So the guard * cannot be a prompt and cannot be a convention; it has to be a test that fails * when a result stops carrying its own denominator. Three things are pinned * here and nothing else: * * 1. every result carrying a count or a collection carries a `scope`; * 2. a filtered count is never the only count in its own result; * 3. the threshold that produced a filtered count is named in the payload, * because three surfaces of this product have quoted three different idle * figures and the only way to reconcile them is to know which is which. * * The page tools are executed against a stub handle rather than Postgres. The * unit suite runs in CI BEFORE the migration step, so a query here would meet a * database with no tables; the stub answers the four reads these tools make and * nothing else, which is enough because what is under test is the shaping, not * the SQL. `e2e/page-tools.test.ts` covers the SQL against a real book. */ import assert from 'node:assert/strict'; import test from 'node:test'; import { accounts, allocations, capacityCommitments, contacts, contracts, demandDeals, supplyDeals, type Database, } from '@pig/db'; import { createInteractivePigTools } from '../src/chat-tools'; import { createPagePigTools, resultScope, type ResultScope } from '../src/page-tools'; // --------------------------------------------------------------------------- // The stub handle // --------------------------------------------------------------------------- interface StubBook { /** Live commitments. The stub does not evaluate where clauses. */ commitments: readonly Record[]; allocations: readonly Record[]; /** The OPEN deals, which is what the row reads in these tools select. */ demandDeals: readonly Record[]; supplyDeals: readonly Record[]; /** Every deal on the book, which is what the `count()` reads select. */ counts: { demandDeals: number; supplyDeals: number }; accounts?: readonly Record[]; contacts?: readonly Record[]; contracts?: readonly Record[]; } /** * A thenable that answers one read. * * Drizzle's builder is a promise you can keep calling methods on, so the stub * is the same: every chaining method returns itself and `then` resolves the * rows. The where clauses are ignored deliberately — a stub that reimplemented * them would be testing itself. */ function stubQuery(rows: readonly unknown[]): Record { const builder: Record = {}; for (const method of ['where', 'limit', 'orderBy', 'leftJoin', 'innerJoin', 'innerJoinLateral']) { builder[method] = () => builder; } builder.then = (resolve: (value: readonly unknown[]) => unknown) => resolve(rows); return builder; } function stubDatabase(book: StubBook): Database { const rowsFor = (table: unknown): readonly unknown[] => { if (table === capacityCommitments) return book.commitments; if (table === allocations) return book.allocations; if (table === demandDeals) return book.demandDeals; if (table === supplyDeals) return book.supplyDeals; if (table === accounts) return book.accounts ?? []; if (table === contacts) return book.contacts ?? []; if (table === contracts) return book.contracts ?? []; throw new Error('the stub was asked for a table this suite does not fixture'); }; const countFor = (table: unknown): number => { if (table === demandDeals) return book.counts.demandDeals; if (table === supplyDeals) return book.counts.supplyDeals; return rowsFor(table).length; }; const select = (projection?: Record) => ({ from: (table: unknown) => { // `select({ value: count() })` is the only projection with that shape, // and it is how every denominator in these tools is read. const counting = projection !== undefined && Object.keys(projection).length === 1 && Object.hasOwn(projection, 'value'); return stubQuery(counting ? [{ value: countFor(table) }] : rowsFor(table)); }, }); return { select } as unknown as Database; } const DAY = 86_400_000; const now = Date.now(); /** One live block: `sold` of `hours` bought at `costCents` per GPU-hour. */ function block(name: string, hours: number, sold: number, costCents = 100) { return { id: name, name, gpuType: 'H100_80GB', gpuCount: 8, startsAt: new Date(now - 10 * DAY), endsAt: new Date(now + 100 * DAY), // numeric columns arrive from Postgres as strings, and so must these. totalGpuHours: `${hours}.00`, costPerGpuHourCents: costCents, sold, }; } function allocation(commitmentId: string, gpuHours: number) { return { capacityCommitmentId: commitmentId, status: 'committed', gpuHours: `${gpuHours}.00`, pricePerGpuHourCents: 120, holdExpiresAt: null, }; } /** * Five live commitments, three of them at least 25% unsold. * * The production book was five and the tool returned three. Reproducing that * ratio exactly is the point: a fixture where the filter happens to keep * everything cannot fail the way production did. */ const BOOK = [ block('idle-90', 1000, 100), block('idle-50', 1000, 500), block('idle-30', 1000, 700), block('idle-10', 1000, 900), block('idle-0', 1000, 1000), ]; const LIVE_COMMITMENTS = BOOK.length; const IDLE_BLOCKS = 3; const stub = stubDatabase({ commitments: BOOK.map(({ sold: _sold, ...row }) => row), allocations: BOOK.filter((row) => row.sold > 0).map((row) => allocation(row.id, row.sold)), demandDeals: Array.from({ length: 4 }, (_, i) => ({ id: `demand-${i}`, name: `Demand ${i}`, stage: 'proposal', acvCents: 1_000_000, tcvCents: 2_500_000, expectedCloseDate: null, })), supplyDeals: Array.from({ length: 2 }, (_, i) => ({ id: `supply-${i}`, name: `Supply ${i}`, stage: 'sourced', gpuType: 'H200', gpuCount: 64, targetCostPerGpuHourCents: 189, })), counts: { demandDeals: 13, supplyDeals: 8 }, accounts: [{ id: 'acct', name: 'DEMO — Halcyon Research' }], contacts: [{ id: 'contact-1', accountId: 'acct', fullName: 'A Person' }], contracts: [{ id: 'contract-1', accountId: 'acct', title: 'DEMO — MSA' }], }); type Reading = Record & { headline?: string; scope?: ResultScope }; async function read(route: '/margin' | '/capacity' | '/demand' | '/'): Promise { const [tool] = createPagePigTools(stub, route); assert.ok(tool, `no tool for ${route}`); return (await tool.execute({})) as Reading; } /** Every `scope` object anywhere in a result, however deeply it is nested. */ function scopes(value: unknown, found: ResultScope[] = []): ResultScope[] { if (Array.isArray(value)) { for (const entry of value) scopes(entry, found); return found; } if (value === null || typeof value !== 'object') return found; for (const [key, entry] of Object.entries(value)) { if (key === 'scope' || key.endsWith('Scope')) found.push(entry as ResultScope); else scopes(entry, found); } return found; } // --------------------------------------------------------------------------- // The shape itself // --------------------------------------------------------------------------- test('an unfiltered scope says so, rather than hedging a figure that is exact', () => { const scope = resultScope({ covers: 'are live', matched: 5, total: 5, totalLabel: 'live capacity commitment(s) on the book', listed: 5, }); assert.equal(scope.summary, 'All 5 live capacity commitment(s) on the book; 5 listed here.'); assert.equal(scope.matched, scope.total); }); test('a filtered scope states both figures and names the filtered one as filtered', () => { const scope = resultScope({ covers: 'are at least 25% unsold', matched: 3, total: 5, totalLabel: 'live capacity commitment(s) on the book', listed: 3, filters: { idleThresholdPct: 0.25 }, }); // The sentence a small model quotes has to carry the denominator, because a // field it must reason over is a field it will skip. assert.match(scope.summary, /3 of 5 live capacity commitment\(s\) on the book/); assert.match(scope.summary, /the total is 5/); assert.equal(scope.filters.idleThresholdPct, 0.25); }); test('a truncated read hedges the matched count as well as the total', () => { const scope = resultScope({ covers: 'are open', matched: 500, total: 500, totalLabel: 'demand deal(s) on the book', listed: 8, filters: { stages: 'open only' }, truncated: true, }); assert.match(scope.summary, /at least 500 of at least 500/); // Hedging only the total would present a capped match count as exact. assert.match(scope.summary, /lower bound/); }); // --------------------------------------------------------------------------- // The measured defect // --------------------------------------------------------------------------- test('the idle tool reports the size of the book beside the size of its filter', async () => { const reading = await read('/capacity'); const scope = reading.scope; assert.ok(scope); // Three blocks matched out of five on the book: the production numbers. assert.equal(scope.matched, IDLE_BLOCKS); assert.equal(scope.total, LIVE_COMMITMENTS); assert.equal(reading.idleBlocks, IDLE_BLOCKS); assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); // The headline is what a small model quotes, so the denominator has to be in // it. "3" alone was true of the filter and false of the book. assert.match(String(reading.headline), /3 of 5 live capacity commitment\(s\) on the book/); assert.match(String(reading.headline), /the book holds 5 live commitment\(s\) in total/); assert.match(scope.summary, /the total is 5/); }); test('the idle tool names the threshold that produced its count', async () => { const reading = await read('/capacity'); assert.equal(reading.scope?.filters.idleThresholdPct, 0.25); assert.equal(reading.scope?.filters.withinDays, 30); assert.equal(reading.thresholdPct, 0.25); // Three surfaces of this product have quoted three different idle counts for // one book. A result that does not say which threshold it used cannot be // reconciled with the screen beside it. assert.match(String(reading.headline), /at least 25% unsold/); }); test('the filtered count is never the only count in the idle result', async () => { const reading = await read('/capacity'); const listed = reading.blocks; assert.ok(Array.isArray(listed)); // Everything that counts blocks: the matched figure, the listed rows, and the // denominator. The denominator must be present and must differ from them. const counts = [reading.idleBlocks, listed.length, reading.liveCommitments]; assert.equal(counts.includes(LIVE_COMMITMENTS), true); assert.notEqual(reading.idleBlocks, reading.liveCommitments); }); // --------------------------------------------------------------------------- // The same trap in every other tool // --------------------------------------------------------------------------- test('the margin summary describes itself as the whole book, not a slice', async () => { const reading = await read('/margin'); const scope = reading.scope; assert.ok(scope); assert.equal(scope.matched, LIVE_COMMITMENTS); assert.equal(scope.total, LIVE_COMMITMENTS); assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); assert.match(String(reading.headline), /all 5 live capacity commitment\(s\) on the book/); // `largestBlocks` is still a slice, and `listed` is what says so. assert.equal(scope.listed, LIVE_COMMITMENTS); }); test('both pipelines carry the number of deals they were drawn from', async () => { const reading = await read('/demand'); const demand = reading.demand as { scope: ResultScope; openDeals: number; totalDeals: number }; const supply = reading.supply as { scope: ResultScope; openDeals: number; totalDeals: number }; assert.equal(demand.openDeals, 4); assert.equal(demand.totalDeals, 13); assert.equal(demand.scope.total, 13); assert.equal(supply.openDeals, 2); assert.equal(supply.totalDeals, 8); assert.equal(supply.scope.total, 8); assert.match(String(reading.headline), /4 of 13 demand deal\(s\) on the book are open/); assert.match(String(reading.headline), /2 of 8 supply deal\(s\) on the book are open/); }); test('the workspace summary states the threshold behind its worst-idle list', async () => { const reading = await read('/'); const worst = reading.worstIdle as { scope: ResultScope; blocks: unknown[] }; // Four of the five blocks have some idle; three are listed. Both figures are // present, so "three blocks are idle" cannot be read off the list length. assert.equal(worst.blocks.length, 3); assert.equal(worst.scope.matched, 4); assert.equal(worst.scope.total, LIVE_COMMITMENTS); assert.equal(worst.scope.listed, 3); // NOT 0.25. This list and pig_get_idle_capacity answer different questions // and return different counts; each says which threshold it applied. assert.equal(worst.scope.filters.idleThresholdPct, 0); assert.match(String(reading.headline), /the worst idle of 4 with any idle hours/); }); test('the workspace summary counts open deals against every deal on the book', async () => { const reading = await read('/'); assert.equal(reading.openDemandDeals, 4); assert.equal(reading.totalDemandDeals, 13); assert.equal(reading.openSupplyDeals, 2); assert.equal(reading.totalSupplyDeals, 8); assert.equal((reading.openDemandDealsScope as ResultScope).total, 13); assert.equal((reading.openSupplyDealsScope as ResultScope).total, 8); }); // --------------------------------------------------------------------------- // The sweep // --------------------------------------------------------------------------- test('every page tool result carries at least one scope, and every scope is complete', async () => { for (const route of ['/margin', '/capacity', '/demand', '/'] as const) { const reading = await read(route); const found = scopes(reading); assert.ok(found.length > 0, `${route} returned a result with no scope at all`); for (const scope of found) { assert.equal(typeof scope.summary, 'string', `${route}: scope has no summary`); assert.ok(scope.summary.length > 0, `${route}: empty scope summary`); assert.equal(typeof scope.matched, 'number', `${route}: scope has no matched`); assert.equal(typeof scope.total, 'number', `${route}: scope has no total`); assert.ok(scope.totalLabel.length > 0, `${route}: scope has no totalLabel`); assert.equal(typeof scope.listed, 'number', `${route}: scope has no listed`); assert.equal(typeof scope.truncated, 'boolean', `${route}: scope has no truncated`); // The denominator has to reach the sentence, because the sentence is what // gets quoted. A scope whose summary omits its own total is the defect. assert.match( scope.summary, new RegExp(`\\b${scope.total}\\b`), `${route}: a scope summary omits the total it was drawn from`, ); assert.ok(scope.matched <= scope.total, `${route}: matched exceeds its own denominator`); assert.ok(scope.listed <= scope.matched, `${route}: more rows listed than matched`); } } }); test('no page headline reports a filtered count without the total beside it', async () => { for (const route of ['/margin', '/capacity', '/demand', '/'] as const) { const reading = await read(route); const headline = String(reading.headline); for (const scope of scopes(reading)) { if (scope.matched === scope.total) continue; assert.match( headline, new RegExp(`\\b${scope.total}\\b`), `${route}: the headline quotes a filtered figure with no denominator`, ); } } }); // --------------------------------------------------------------------------- // The record read // --------------------------------------------------------------------------- test('a record read says whose figures these are, so they are not read as the book', async () => { const [record] = createInteractivePigTools(stub, { type: 'account', id: 'acct' }); assert.ok(record); const reading = (await record.execute({})) as Reading; const scope = reading.scope; assert.ok(scope); // Nothing was filtered out — this is an enumeration of one row's relations — // so the figures are exact. What the sentence must carry is the boundary: // four deals belong to this account, not to the book. assert.equal(scope.matched, scope.total); assert.match(scope.summary, /DEMO — Halcyon Research/); assert.match(scope.summary, /never book-wide totals/); assert.match(scope.summary, /4 demand deal\(s\)/); });