/** * The scope contract, pinned. * * This suite exists because of one production answer. Asked "How many capacity * commitments are on the book?" on /capacity, Piggy called * `pig_get_idle_capacity` — the only tool that page offers — and said "3". The * book held 5. The tool filters to blocks at least 25% unsold, so 3 was the * size of a filter, and the payload gave the model nothing else to read: the * length of the list it had been handed was the only count in front of it. * * The system prompt already forbade that, naming this exact tool. So the guard * cannot be a prompt and cannot be a convention; it has to be a test that fails * when a result stops carrying its own denominator. Three things are pinned * here and nothing else: * * 1. every result carrying a count or a collection carries a `scope`; * 2. a filtered count is never the only count in its own result; * 3. the threshold that produced a filtered count is named in the payload, * because three surfaces of this product have quoted three different idle * figures and the only way to reconcile them is to know which is which. * * The page tools are executed against a stub handle rather than Postgres. The * unit suite runs in CI BEFORE the migration step, so a query here would meet a * database with no tables; the stub answers the four reads these tools make and * nothing else, which is enough because what is under test is the shaping, not * the SQL. `e2e/page-tools.test.ts` covers the SQL against a real book. */ import assert from 'node:assert/strict'; import test from 'node:test'; import { accounts, allocations, capacityCommitments, contacts, contracts, demandDeals, supplyDeals, type Database, } from '@pig/db'; import { createInteractivePigTools } from '../src/chat-tools'; import { createPagePigTools, resultScope, type ResultScope } from '../src/page-tools'; // --------------------------------------------------------------------------- // The stub handle // --------------------------------------------------------------------------- interface StubBook { /** Live commitments. The stub does not evaluate where clauses. */ commitments: readonly Record[]; allocations: readonly Record[]; /** The OPEN deals, which is what the row reads in these tools select. */ demandDeals: readonly Record[]; supplyDeals: readonly Record[]; /** * What the `count()` reads select: every deal on the book, every contact row, * and the accounts the archive filter removes. */ counts: { demandDeals: number; supplyDeals: number; contacts: number; archivedAccounts: number; }; accounts?: readonly Record[]; contacts?: readonly Record[]; contracts?: readonly Record[]; /** * The one grouped count these tools make: accounts per side, archived * excluded. Fixtured rather than derived from `accounts` above, because the * stub evaluates no where clause and so cannot tell an archived row from a * live one — deriving it would quietly test the fixture against itself. */ accountsBySide?: readonly { side: string; value: number }[]; } /** * A thenable that answers one read. * * Drizzle's builder is a promise you can keep calling methods on, so the stub * is the same: every chaining method returns itself and `then` resolves the * rows. The where clauses are ignored deliberately — a stub that reimplemented * them would be testing itself. */ function stubQuery(rows: readonly unknown[]): Record { const builder: Record = {}; for (const method of [ 'where', 'limit', 'orderBy', 'groupBy', 'leftJoin', 'innerJoin', 'innerJoinLateral', ]) { builder[method] = () => builder; } builder.then = (resolve: (value: readonly unknown[]) => unknown) => resolve(rows); return builder; } function stubDatabase(book: StubBook): Database { const rowsFor = (table: unknown): readonly unknown[] => { if (table === capacityCommitments) return book.commitments; if (table === allocations) return book.allocations; if (table === demandDeals) return book.demandDeals; if (table === supplyDeals) return book.supplyDeals; if (table === accounts) return book.accounts ?? []; if (table === contacts) return book.contacts ?? []; if (table === contracts) return book.contracts ?? []; throw new Error('the stub was asked for a table this suite does not fixture'); }; const countFor = (table: unknown): number => { if (table === demandDeals) return book.counts.demandDeals; if (table === supplyDeals) return book.counts.supplyDeals; // The only ungrouped count taken against accounts is the archived one; the // live figure is summed from the grouped read below, so that the total and // its own breakdown cannot disagree. if (table === accounts) return book.counts.archivedAccounts; if (table === contacts) return book.counts.contacts; return rowsFor(table).length; }; const select = (projection?: Record) => ({ from: (table: unknown) => { // `count()` always lands in a key called `value`. Alone it is a // denominator; beside another column it is a grouped count, and accounts // per side is the only one these tools take. const counting = projection !== undefined && Object.hasOwn(projection, 'value'); if (counting && Object.keys(projection).length > 1) { if (table !== accounts) throw new Error('the stub groups counts for accounts only'); return stubQuery(book.accountsBySide ?? []); } return stubQuery(counting ? [{ value: countFor(table) }] : rowsFor(table)); }, }); return { select } as unknown as Database; } const DAY = 86_400_000; const now = Date.now(); /** One live block: `sold` of `hours` bought at `costCents` per GPU-hour. */ function block(name: string, hours: number, sold: number, costCents = 100) { return { id: name, name, gpuType: 'H100_80GB', gpuCount: 8, startsAt: new Date(now - 10 * DAY), endsAt: new Date(now + 100 * DAY), // numeric columns arrive from Postgres as strings, and so must these. totalGpuHours: `${hours}.00`, costPerGpuHourCents: costCents, sold, }; } function allocation(commitmentId: string, gpuHours: number) { return { capacityCommitmentId: commitmentId, status: 'committed', gpuHours: `${gpuHours}.00`, pricePerGpuHourCents: 120, holdExpiresAt: null, }; } /** * Five live commitments, three of them at least 25% unsold. * * The production book was five and the tool returned three. Reproducing that * ratio exactly is the point: a fixture where the filter happens to keep * everything cannot fail the way production did. */ const BOOK = [ block('idle-90', 1000, 100), block('idle-50', 1000, 500), block('idle-30', 1000, 700), block('idle-10', 1000, 900), block('idle-0', 1000, 1000), ]; const LIVE_COMMITMENTS = BOOK.length; const IDLE_BLOCKS = 3; /** * The account book, sized as production was when it was measured. * * Production held 17 accounts and 7 demand deals, and Piggy answered "The book * contains 7 demand deals (accounts) in total" to a question about accounts. So * the fixture keeps the two apart by more than an accident of arithmetic: 17 is * not the size of any deal figure, any commitment figure or any list in this * suite, and a payload that reports it can only have got it from the account * count. `both` is present because the sides must partition the book — 9 + 7 + 1 * is 17, and an account that trades on each side is counted once. */ const ACCOUNTS_BY_SIDE = [ { side: 'supply', value: 9 }, { side: 'demand', value: 7 }, { side: 'both', value: 1 }, ]; const ACCOUNTS_ON_BOOK = 17; /** Archived, so on no screen and in no total. The gap is still counted. */ const ARCHIVED_ACCOUNTS = 2; const CONTACTS = 42; const stub = stubDatabase({ commitments: BOOK.map(({ sold: _sold, ...row }) => row), allocations: BOOK.filter((row) => row.sold > 0).map((row) => allocation(row.id, row.sold)), demandDeals: Array.from({ length: 4 }, (_, i) => ({ id: `demand-${i}`, name: `Demand ${i}`, stage: 'proposal', acvCents: 1_000_000, tcvCents: 2_500_000, expectedCloseDate: null, })), supplyDeals: Array.from({ length: 2 }, (_, i) => ({ id: `supply-${i}`, name: `Supply ${i}`, stage: 'sourced', gpuType: 'H200', gpuCount: 64, targetCostPerGpuHourCents: 189, })), counts: { demandDeals: 13, supplyDeals: 8, contacts: CONTACTS, archivedAccounts: ARCHIVED_ACCOUNTS, }, accounts: [{ id: 'acct', name: 'DEMO — Halcyon Research' }], contacts: [{ id: 'contact-1', accountId: 'acct', fullName: 'A Person' }], contracts: [{ id: 'contract-1', accountId: 'acct', title: 'DEMO — MSA' }], accountsBySide: ACCOUNTS_BY_SIDE, }); type Reading = Record & { headline?: string; scope?: ResultScope }; async function read( route: '/margin' | '/capacity' | '/demand' | '/' | '/accounts' | '/team', ): Promise { const [tool] = createPagePigTools(stub, route); assert.ok(tool, `no tool for ${route}`); return (await tool.execute({})) as Reading; } /** Every `scope` object anywhere in a result, however deeply it is nested. */ function scopes(value: unknown, found: ResultScope[] = []): ResultScope[] { if (Array.isArray(value)) { for (const entry of value) scopes(entry, found); return found; } if (value === null || typeof value !== 'object') return found; for (const [key, entry] of Object.entries(value)) { if (key === 'scope' || key.endsWith('Scope')) found.push(entry as ResultScope); else scopes(entry, found); } return found; } // --------------------------------------------------------------------------- // The shape itself // --------------------------------------------------------------------------- test('an unfiltered scope says so, rather than hedging a figure that is exact', () => { const scope = resultScope({ covers: 'are live', matched: 5, total: 5, totalLabel: 'live capacity commitment(s) on the book', listed: 5, }); assert.equal(scope.summary, 'All 5 live capacity commitment(s) on the book; 5 listed here.'); assert.equal(scope.matched, scope.total); }); test('a filtered scope states both figures and names the filtered one as filtered', () => { const scope = resultScope({ covers: 'are at least 25% unsold', matched: 3, total: 5, totalLabel: 'live capacity commitment(s) on the book', listed: 3, filters: { idleThresholdPct: 0.25 }, }); // The sentence a small model quotes has to carry the denominator, because a // field it must reason over is a field it will skip. assert.match(scope.summary, /3 of 5 live capacity commitment\(s\) on the book/); assert.match(scope.summary, /the total is 5/); assert.equal(scope.filters.idleThresholdPct, 0.25); }); test('a truncated read hedges the matched count as well as the total', () => { const scope = resultScope({ covers: 'are open', matched: 500, total: 500, totalLabel: 'demand deal(s) on the book', listed: 8, filters: { stages: 'open only' }, truncated: true, }); assert.match(scope.summary, /at least 500 of at least 500/); // Hedging only the total would present a capped match count as exact. assert.match(scope.summary, /lower bound/); }); // --------------------------------------------------------------------------- // The measured defect // --------------------------------------------------------------------------- test('the idle tool reports the size of the book beside the size of its filter', async () => { const reading = await read('/capacity'); const scope = reading.scope; assert.ok(scope); // Three blocks matched out of five on the book: the production numbers. assert.equal(scope.matched, IDLE_BLOCKS); assert.equal(scope.total, LIVE_COMMITMENTS); assert.equal(reading.idleBlocks, IDLE_BLOCKS); assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); // The headline is what a small model quotes, so the denominator has to be in // it. "3" alone was true of the filter and false of the book. assert.match(String(reading.headline), /3 of 5 live capacity commitment\(s\) on the book/); assert.match(String(reading.headline), /the book holds 5 live commitment\(s\) in total/); assert.match(scope.summary, /the total is 5/); }); test('the idle tool names the threshold that produced its count', async () => { const reading = await read('/capacity'); assert.equal(reading.scope?.filters.idleThresholdPct, 0.25); assert.equal(reading.scope?.filters.withinDays, 30); assert.equal(reading.thresholdPct, 0.25); // Three surfaces of this product have quoted three different idle counts for // one book. A result that does not say which threshold it used cannot be // reconciled with the screen beside it. assert.match(String(reading.headline), /at least 25% unsold/); }); test('the filtered count is never the only count in the idle result', async () => { const reading = await read('/capacity'); const listed = reading.blocks; assert.ok(Array.isArray(listed)); // Everything that counts blocks: the matched figure, the listed rows, and the // denominator. The denominator must be present and must differ from them. const counts = [reading.idleBlocks, listed.length, reading.liveCommitments]; assert.equal(counts.includes(LIVE_COMMITMENTS), true); assert.notEqual(reading.idleBlocks, reading.liveCommitments); }); // --------------------------------------------------------------------------- // The same trap in every other tool // --------------------------------------------------------------------------- test('the margin summary describes itself as the whole book, not a slice', async () => { const reading = await read('/margin'); const scope = reading.scope; assert.ok(scope); assert.equal(scope.matched, LIVE_COMMITMENTS); assert.equal(scope.total, LIVE_COMMITMENTS); assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); assert.match(String(reading.headline), /all 5 live capacity commitment\(s\) on the book/); // `largestBlocks` is still a slice, and `listed` is what says so. assert.equal(scope.listed, LIVE_COMMITMENTS); }); test('both pipelines carry the number of deals they were drawn from', async () => { const reading = await read('/demand'); const demand = reading.demand as { scope: ResultScope; openDeals: number; totalDeals: number }; const supply = reading.supply as { scope: ResultScope; openDeals: number; totalDeals: number }; assert.equal(demand.openDeals, 4); assert.equal(demand.totalDeals, 13); assert.equal(demand.scope.total, 13); assert.equal(supply.openDeals, 2); assert.equal(supply.totalDeals, 8); assert.equal(supply.scope.total, 8); assert.match(String(reading.headline), /4 of 13 demand deal\(s\) on the book are open/); assert.match(String(reading.headline), /2 of 8 supply deal\(s\) on the book are open/); }); test('the workspace summary states the threshold behind its worst-idle list', async () => { const reading = await read('/'); const worst = reading.worstIdle as { scope: ResultScope; blocks: unknown[] }; // Four of the five blocks have some idle; three are listed. Both figures are // present, so "three blocks are idle" cannot be read off the list length. assert.equal(worst.blocks.length, 3); assert.equal(worst.scope.matched, 4); assert.equal(worst.scope.total, LIVE_COMMITMENTS); assert.equal(worst.scope.listed, 3); // NOT 0.25. This list and pig_get_idle_capacity answer different questions // and return different counts; each says which threshold it applied. assert.equal(worst.scope.filters.idleThresholdPct, 0); assert.match(String(reading.headline), /the worst idle of 4 with any idle hours/); }); test('the workspace summary counts open deals against every deal on the book', async () => { const reading = await read('/'); assert.equal(reading.openDemandDeals, 4); assert.equal(reading.totalDemandDeals, 13); assert.equal(reading.openSupplyDeals, 2); assert.equal(reading.totalSupplyDeals, 8); assert.equal((reading.openDemandDealsScope as ResultScope).total, 13); assert.equal((reading.openSupplyDealsScope as ResultScope).total, 8); }); // --------------------------------------------------------------------------- // The missing denominator // --------------------------------------------------------------------------- /** Both keys the workspace summary carries its party counts under. */ interface Parties { accounts: { scope: ResultScope; onBook: number; archived: number; bySide: Record; bySideNote: string; }; contacts: { scope: ResultScope; total: number }; } async function parties(route: '/' | '/accounts' | '/team'): Promise { return (await read(route)) as unknown as Parties; } test('the workspace summary counts the accounts and contacts on the book', async () => { const { accounts: book, contacts: people } = await parties('/'); assert.equal(book.onBook, ACCOUNTS_ON_BOOK); assert.equal(people.total, CONTACTS); // Nothing was filtered out of either, so `matched` IS the total: these are // answers to "how many are there", not counts that need a denominator. assert.equal(book.scope.matched, ACCOUNTS_ON_BOOK); assert.equal(book.scope.total, ACCOUNTS_ON_BOOK); assert.equal(people.scope.total, CONTACTS); // The label is what the grounding rule tells the model to read the figure // against, so it has to name the noun the question would use. assert.match(book.scope.totalLabel, /account\(s\)/); assert.match(people.scope.totalLabel, /contact\(s\)/); assert.match(book.scope.summary, /All 17 account\(s\) on the book/); assert.match(people.scope.summary, /All 42 contact\(s\) in the CRM/); }); test('the headline states the account count, because the headline is what gets quoted', async () => { const reading = await read('/'); const headline = String(reading.headline); // The production answer was assembled from the first countable thing in this // sentence. There is now an account figure in it, and it is first. assert.match(headline, /^17 account\(s\) on the book/); assert.match(headline, /42 contact\(s\) in the CRM/); // A count of deals is not a count of accounts, and no deal figure in this // fixture can be mistaken for one. for (const dealFigure of [13, 8, 4, 2]) { assert.notEqual(ACCOUNTS_ON_BOOK, dealFigure); } }); test('the sides partition the account book rather than overlapping it', async () => { const { accounts: book } = await parties('/'); // Every side present, at zero if need be: an absent key reads as "not known" // to a model quoting the payload. assert.deepEqual(book.bySide, { supply: 9, demand: 7, both: 1 }); const summed = Object.values(book.bySide).reduce((sum, value) => sum + value, 0); assert.equal(summed, ACCOUNTS_ON_BOOK); // A breakdown that disagrees with the /accounts side tabs, quoted beside that // screen, is the next version of this bug. The note is what reconciles them: // the tabs match `side = X or both`, so they overlap and do not sum. assert.match(book.bySideNote, /counted once, under both/); assert.match(book.bySideNote, /do not sum/); }); test('archived accounts are off the total and still counted', async () => { const reading = await read('/'); const { accounts: book } = await parties('/'); // The /accounts list excludes them, so the total that answers "how many // accounts are on the book" must exclude them too — Piggy disagreeing with // the list on screen is worse than Piggy knowing less than it does. assert.equal(book.archived, ARCHIVED_ACCOUNTS); assert.equal(book.onBook, ACCOUNTS_ON_BOOK); assert.notEqual(book.onBook, ACCOUNTS_ON_BOOK + ARCHIVED_ACCOUNTS); // Excluded, but not invisible: a figure that differs from a raw table count // has to be reconcilable from the payload alone. assert.match(String(reading.headline), /a further 2 account\(s\) archived and off the book/); }); test('/accounts is given a tool that can answer how many accounts there are', async () => { // The measured defect, at the route it was measured on. Asked "How many // accounts are on the book in total?" here, Piggy answered "The book contains // 7 demand deals (accounts) in total" — a real figure, correctly scoped as // deals by the payload, relabelled as accounts in the prose, because no // account figure existed anywhere in the result it was handed. const { accounts: book } = await parties('/accounts'); assert.equal(book.onBook, ACCOUNTS_ON_BOOK); assert.match(book.scope.summary, /account\(s\) on the book/); }); test('a page with no data tool still gets the book denominators, never nothing', async () => { // /team has no tool of its own and falls through to the summary. It must not // arrive with a payload that is silent about every noun: the guide tells the // model it can see no users, and the counts it CAN see are all labelled. const { accounts: book, contacts: people } = await parties('/team'); assert.equal(book.onBook, ACCOUNTS_ON_BOOK); assert.equal(people.total, CONTACTS); }); // --------------------------------------------------------------------------- // The sweep // --------------------------------------------------------------------------- test('every page tool result carries at least one scope, and every scope is complete', async () => { for (const route of ['/margin', '/capacity', '/demand', '/', '/accounts'] as const) { const reading = await read(route); const found = scopes(reading); assert.ok(found.length > 0, `${route} returned a result with no scope at all`); for (const scope of found) { assert.equal(typeof scope.summary, 'string', `${route}: scope has no summary`); assert.ok(scope.summary.length > 0, `${route}: empty scope summary`); assert.equal(typeof scope.matched, 'number', `${route}: scope has no matched`); assert.equal(typeof scope.total, 'number', `${route}: scope has no total`); assert.ok(scope.totalLabel.length > 0, `${route}: scope has no totalLabel`); assert.equal(typeof scope.listed, 'number', `${route}: scope has no listed`); assert.equal(typeof scope.truncated, 'boolean', `${route}: scope has no truncated`); // The denominator has to reach the sentence, because the sentence is what // gets quoted. A scope whose summary omits its own total is the defect. assert.match( scope.summary, new RegExp(`\\b${scope.total}\\b`), `${route}: a scope summary omits the total it was drawn from`, ); assert.ok(scope.matched <= scope.total, `${route}: matched exceeds its own denominator`); assert.ok(scope.listed <= scope.matched, `${route}: more rows listed than matched`); } } }); test('no page headline reports a filtered count without the total beside it', async () => { for (const route of ['/margin', '/capacity', '/demand', '/', '/accounts'] as const) { const reading = await read(route); const headline = String(reading.headline); for (const scope of scopes(reading)) { if (scope.matched === scope.total) continue; assert.match( headline, new RegExp(`\\b${scope.total}\\b`), `${route}: the headline quotes a filtered figure with no denominator`, ); } } }); // --------------------------------------------------------------------------- // The record read // --------------------------------------------------------------------------- test('a record read says whose figures these are, so they are not read as the book', async () => { const [record] = createInteractivePigTools(stub, { type: 'account', id: 'acct' }); assert.ok(record); const reading = (await record.execute({})) as Reading; const scope = reading.scope; assert.ok(scope); // Nothing was filtered out — this is an enumeration of one row's relations — // so the figures are exact. What the sentence must carry is the boundary: // four deals belong to this account, not to the book. assert.equal(scope.matched, scope.total); assert.match(scope.summary, /DEMO — Halcyon Research/); assert.match(scope.summary, /never book-wide totals/); assert.match(scope.summary, /4 demand deal\(s\)/); });