diff --git a/apps/piggy/src/agent/prompt.ts b/apps/piggy/src/agent/prompt.ts index 9dea87f..e01019e 100644 --- a/apps/piggy/src/agent/prompt.ts +++ b/apps/piggy/src/agent/prompt.ts @@ -26,7 +26,7 @@ const UNITS_RULE = `Units, before you quote any figure: - Never write a money figure in cents. "112 cents" and "112c" are both wrong; write $1.12. Every money figure you write starts with a dollar sign. - Any field whose name ends in Pct, and utilisation, is a share between 0 and 1. 0.38 is 38 per cent. - Any field whose name ends in GpuHours is a count of GPU-hours, not money. -- The headline string is the one figure already formatted in dollars. Quote it as written rather than reformatting it. +- The headline string is the one figure already formatted in dollars, and it also states what the result covers. Quote it as written rather than reformatting it. - A null money field means not applicable, not zero. Say why it is absent.`; /** @@ -68,11 +68,22 @@ const DOMAIN_BRIEFING = `How this business works, so the figures mean what you s * So: an explicit ban, the lookup tools named as the way out, and the maths * formatting forbidden outright — `\boxed{}` is the tell that the model has * stopped answering about a CRM and started solving a puzzle. + * + * The scope bullets are the second half of a fix whose first half is in the + * data. This rule already said, naming the tool, that a filtered count is not a + * total; on /capacity nemotron read `pig_get_idle_capacity`'s three blocks as + * the size of a five-commitment book anyway, because nothing in the payload + * contradicted it. Every result now carries `scope` with `matched`, `total` and + * `totalLabel`, so the instruction has a field to point at rather than a + * principle to hold — and that is the only form of this rule that has survived + * contact with a 30B model. Terse on purpose: it rides on every request. */ const GROUNDING_RULE = `Grounding, which overrides everything else: - NEVER state a number, name, date or status about this business unless it appeared in a tool result in THIS conversation. Not from memory, not from what a figure "should" be, not by inference from the page you are on. - If the tool you were given does not answer the question, do not guess and do not stop: pig_search_records finds a record by name and pig_get_record_by_id opens it. Reach for those before concluding anything. -- A tool result answers only what that tool covers. Never report a filtered count as a total: pig_get_idle_capacity returns the blocks with idle hours, not the book. If the result does not cover the question as asked, say what it does cover and what is missing. +- Every result says what it covers. Read its scope object first: matched is how many passed a filter, total is the whole set they were drawn from, totalLabel names what total counts, listed is how many rows the payload carries, filters names every threshold applied. +- Asked how many there are, quote total, never matched and never the length of a list you can see. matched answers "how many are unsold" or "how many match"; it is never the size of the book. If total does not cover the question as asked, say what the result does cover and what is missing. +- Two tools can report different counts of the same thing because they applied different thresholds. Say which threshold produced the figure you quote; it is in filters. - If no tool can answer it, say exactly that and name what you would need. "I cannot see that from here" is a correct answer. An invented figure is not, and is worse than silence — someone will act on it. - Never use LaTeX or mathematical notation. No \\boxed{}, no \\(...\\). Write plain prose and plain numbers.`; diff --git a/apps/piggy/src/chat-tools.ts b/apps/piggy/src/chat-tools.ts index 4ed4afe..6feb386 100644 --- a/apps/piggy/src/chat-tools.ts +++ b/apps/piggy/src/chat-tools.ts @@ -41,15 +41,52 @@ import { } from '@pig/db'; import { CapacityService } from '@pig/api/src/services/capacity'; import { renewalAlarm } from '@pig/api/src/services/contracts'; -import { and, asc, eq, gt, ilike, inArray, isNotNull, isNull } from 'drizzle-orm'; +import { and, asc, count, eq, gt, ilike, inArray, isNotNull, isNull } from 'drizzle-orm'; import { z } from 'zod'; import type { PiggyChatContext } from './chat'; import { createAccountLifecycleTool } from './lifecycle-tools'; -import { createPagePigTools } from './page-tools'; +import { atLeast, createPagePigTools, resultScope, type ResultScope } from './page-tools'; import { defineTool, type AgentTool } from './provider'; const noInput = z.object({}).strict(); +/** + * The per-collection cap on a record read. + * + * Named rather than repeated as a literal because the scope below reports it: + * a related list that came back exactly full is a list that was probably cut, + * and a cut list the reader cannot see is how "this account has 100 deals" + * gets said about an account with three hundred. + */ +const RELATED_LIMIT = 100; + +/** + * The scope of a record read. + * + * Unlike the page tools, a record read filters nothing — it enumerates what + * belongs to one row — so `matched` equals `total` and the sentence is not + * hedged. What it must still say is the boundary, because the failure here is + * the same shape as the /capacity one measured in production: asked how many + * deals are on the book while an account is in focus, a model with only this + * payload counts the four in front of it. `totalLabel` therefore names the + * record and says the figures stop there. + */ +function recordScope(subject: string, collections: Record): ResultScope { + const entries = Object.entries(collections); + const rows = entries.reduce((sum, [, list]) => sum + list.length, 0); + const capped = entries.some(([, list]) => list.length >= RELATED_LIMIT); + const breakdown = entries.map(([label, list]) => `${list.length} ${label}`).join(', '); + return resultScope({ + covers: `belong to ${subject}`, + matched: rows, + total: rows, + totalLabel: `record(s) belonging to ${subject} and to no other — ${breakdown}; these are that record's own figures, never book-wide totals`, + listed: rows, + filters: capped ? { rowCapPerCollection: RELATED_LIMIT } : {}, + truncated: capped, + }); +} + /** * Interactive chat gets one scoped read tool for where it is, plus the lookup * layer, and no ambient access. @@ -102,12 +139,24 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro const [account] = await db.select().from(accounts).where(eq(accounts.id, context.id)).limit(1); if (!account) throw missingRecord(context.type, context.id); const [people, demand, supply, paperwork] = await Promise.all([ - db.select().from(contacts).where(eq(contacts.accountId, context.id)).limit(100), - db.select().from(demandDeals).where(eq(demandDeals.accountId, context.id)).limit(100), - db.select().from(supplyDeals).where(eq(supplyDeals.accountId, context.id)).limit(100), - db.select().from(contracts).where(eq(contracts.accountId, context.id)).limit(100), + db.select().from(contacts).where(eq(contacts.accountId, context.id)).limit(RELATED_LIMIT), + db.select().from(demandDeals).where(eq(demandDeals.accountId, context.id)).limit(RELATED_LIMIT), + db.select().from(supplyDeals).where(eq(supplyDeals.accountId, context.id)).limit(RELATED_LIMIT), + db.select().from(contracts).where(eq(contracts.accountId, context.id)).limit(RELATED_LIMIT), ]); - return { account, contacts: people, demandDeals: demand, supplyDeals: supply, contracts: paperwork }; + return { + scope: recordScope(`the account ${account.name}`, { + 'contact(s)': people, + 'demand deal(s)': demand, + 'supply deal(s)': supply, + 'contract(s)': paperwork, + }), + account, + contacts: people, + demandDeals: demand, + supplyDeals: supply, + contracts: paperwork, + }; } if (context.type === 'contact') { @@ -116,7 +165,13 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro const [account] = contact.accountId ? await db.select().from(accounts).where(eq(accounts.id, contact.accountId)).limit(1) : []; - return { contact, account: account ?? null }; + return { + scope: recordScope(`the contact ${contact.fullName}`, { + 'account(s)': account ? [account] : [], + }), + contact, + account: account ?? null, + }; } if (context.type === 'demand_deal') { @@ -127,8 +182,16 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro .select() .from(allocations) .where(eq(allocations.demandDealId, deal.id)) - .limit(100); - return { deal, account: account ?? null, allocations: reservations }; + .limit(RELATED_LIMIT); + return { + scope: recordScope(`the demand deal ${deal.name}`, { + 'allocation(s)': reservations, + 'account(s)': account ? [account] : [], + }), + deal, + account: account ?? null, + allocations: reservations, + }; } if (context.type === 'supply_deal') { @@ -139,8 +202,16 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro .select() .from(capacityCommitments) .where(eq(capacityCommitments.supplyDealId, deal.id)) - .limit(100); - return { deal, account: account ?? null, commitments }; + .limit(RELATED_LIMIT); + return { + scope: recordScope(`the supply deal ${deal.name}`, { + 'capacity commitment(s)': commitments, + 'account(s)': account ? [account] : [], + }), + deal, + account: account ?? null, + commitments, + }; } if (context.type === 'commitment') { @@ -154,8 +225,14 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro .select() .from(allocations) .where(eq(allocations.capacityCommitmentId, commitment.id)) - .limit(100); - return { commitment, allocations: reservations }; + .limit(RELATED_LIMIT); + return { + scope: recordScope(`the capacity commitment ${commitment.name}`, { + 'allocation(s)': reservations, + }), + commitment, + allocations: reservations, + }; } const [contract] = await db.select().from(contracts).where(eq(contracts.id, context.id)).limit(1); @@ -166,16 +243,26 @@ async function readFocusedRecord(db: Database, context: PiggyRecordContext): Pro .select() .from(contractObligations) .where(eq(contractObligations.contractId, contract.id)) - .limit(100), + .limit(RELATED_LIMIT), ]); const metrics = serviceLevels[0] ? await db .select() .from(slaMetricTargets) .where(eq(slaMetricTargets.slaTermId, serviceLevels[0].id)) - .limit(100) + .limit(RELATED_LIMIT) : []; - return { contract, slaTerms: serviceLevels, slaMetricTargets: metrics, obligations }; + return { + scope: recordScope(`the contract ${contract.title}`, { + 'SLA term(s)': serviceLevels, + 'SLA metric target(s)': metrics, + 'obligation(s)': obligations, + }), + contract, + slaTerms: serviceLevels, + slaMetricTargets: metrics, + obligations, + }; } // --------------------------------------------------------------------------- @@ -416,7 +503,18 @@ async function searchRecords(db: Database, query: string): Promise { const fragment = likeFragment(query); const take = SEARCH_PER_TYPE + 1; - const [accountRows, demandRows, supplyRows, contractRows, commitmentRows] = await Promise.all([ + const [ + accountRows, + demandRows, + supplyRows, + contractRows, + commitmentRows, + accountsAll, + demandAll, + supplyAll, + contractsAll, + commitmentsAll, + ] = await Promise.all([ db .select({ id: accounts.id, @@ -482,6 +580,14 @@ async function searchRecords(db: Database, query: string): Promise { .from(capacityCommitments) .where(ilike(capacityCommitments.name, fragment)) .limit(take), + // The five denominators. A search that reports only its hits invites + // "there are 3 accounts" from a book of twenty-three, and these counts also + // make this tool able to answer how many of a thing exist at all. + db.select({ value: count() }).from(accounts).where(isNull(accounts.archivedAt)), + db.select({ value: count() }).from(demandDeals), + db.select({ value: count() }).from(supplyDeals), + db.select({ value: count() }).from(contracts), + db.select({ value: count() }).from(capacityCommitments), ]); const names = await accountNames(db, [ @@ -491,6 +597,8 @@ async function searchRecords(db: Database, query: string): Promise { ...commitmentRows.map((row) => row.accountId), ]); + const rows = (result: readonly { value: number }[]): number => result[0]?.value ?? 0; + return assembleSearchResult(query, { accounts: accountRows, demandDeals: demandRows, @@ -498,6 +606,13 @@ async function searchRecords(db: Database, query: string): Promise { contracts: contractRows, commitments: commitmentRows, accountNames: names, + totals: { + account: rows(accountsAll), + demand_deal: rows(demandAll), + supply_deal: rows(supplyAll), + contract: rows(contractsAll), + commitment: rows(commitmentsAll), + }, }); } @@ -549,8 +664,18 @@ export interface SearchRowSets { costPerGpuHourCents: number; }[]; accountNames: ReadonlyMap; + /** + * How many rows each searched table holds in total — the denominators the + * per-type match counts are drawn from. Keyed by the same names the results + * carry, so a model reading `counts.account: 1` beside `totals.account: 23` + * cannot mistake a name match for a census. + */ + totals: Record; } +/** The five types a name search covers. People are deliberately not indexed. */ +export type SearchedRecordType = Exclude; + /** * Ranking, capping and counting, with no database in sight. * @@ -664,20 +789,35 @@ export function assembleSearchResult(query: string, sets: SearchRowSets): unknow commitmentsCut.truncated; const results = ranked.slice(0, SEARCH_RESULTS); const truncated = perTypeTruncated || ranked.length > results.length; + const searched = Object.values(sets.totals).reduce((sum, rows) => sum + rows, 0); + const breakdown = Object.entries(counts) + .filter(([, matches]) => matches > 0) + .map(([type, matches]) => `${matches} of ${sets.totals[type as SearchedRecordType]} ${type}(s)`) + .join(', '); return { headline: results.length === 0 - ? `No account, deal, contract or capacity commitment has a name containing "${query}".` - : `${truncated ? 'at least ' : ''}${ranked.length} record(s) match "${query}": ` + - Object.entries(counts) - .filter(([, count]) => count > 0) - .map(([type, count]) => `${count} ${type}(s)`) - .join(', ') + - '.', + ? `None of the ${searched} account(s), deal(s), contract(s) and capacity commitment(s) ` + + `on the book has a name containing "${query}".` + : `${truncated ? 'At least ' : ''}${ranked.length} of ${searched} searchable record(s) ` + + `match "${query}": ${breakdown}. Those are name matches, not totals; the counts they ` + + 'were drawn from are beside them.', + scope: resultScope({ + covers: `have a name containing "${query}"`, + matched: ranked.length, + total: searched, + totalLabel: + 'record(s) searchable by name: accounts, demand deals, supply deals, contracts and capacity commitments', + listed: results.length, + filters: { query }, + truncated, + }), query, truncated, counts, + /** The denominator for each entry in `counts`, keyed identically. */ + totals: sets.totals, results: results.map((entry) => entry.hit), }; } @@ -702,26 +842,36 @@ export function assembleSearchResult(query: string, sets: SearchRowSets): unknow */ async function listRenewals(db: Database, side: 'demand' | 'supply' | undefined): Promise { const now = new Date(); - const rows = await db - .select({ contract: contracts, accountName: accounts.name }) - .from(contracts) - .leftJoin(accounts, eq(accounts.id, contracts.accountId)) - .where( - and( - eq(contracts.status, 'executed'), - isNull(contracts.terminatedAt), - isNotNull(contracts.expiresAt), - gt(contracts.expiresAt, now), - side ? eq(contracts.side, side) : undefined, - ), - ) - .orderBy(asc(contracts.expiresAt)) - .limit(SCAN_LIMIT + 1); + // The denominator is counted on the same side filter the list uses, so + // "4 of 20" and "4 of 11 on the demand side" are both answers to the + // question that was actually asked. + const [rows, all] = await Promise.all([ + db + .select({ contract: contracts, accountName: accounts.name }) + .from(contracts) + .leftJoin(accounts, eq(accounts.id, contracts.accountId)) + .where( + and( + eq(contracts.status, 'executed'), + isNull(contracts.terminatedAt), + isNotNull(contracts.expiresAt), + gt(contracts.expiresAt, now), + side ? eq(contracts.side, side) : undefined, + ), + ) + .orderBy(asc(contracts.expiresAt)) + .limit(SCAN_LIMIT + 1), + db + .select({ value: count() }) + .from(contracts) + .where(side ? eq(contracts.side, side) : undefined), + ]); return assembleRenewals(rows.slice(0, SCAN_LIMIT), { now, side, truncated: rows.length > SCAN_LIMIT, + totalContracts: all[0]?.value ?? 0, }); } @@ -745,9 +895,15 @@ export interface RenewalRow { */ export function assembleRenewals( rows: readonly RenewalRow[], - options: { now: Date; side?: 'demand' | 'supply'; truncated: boolean }, + options: { + now: Date; + side?: 'demand' | 'supply'; + truncated: boolean; + /** Every contract on this side, whatever its status. The denominator. */ + totalContracts: number; + }, ): unknown { - const { now, side, truncated } = options; + const { now, side, truncated, totalContracts } = options; const renewals = rows .flatMap(({ contract, accountName }) => { // The query already requires an expiry; narrowing here rather than @@ -788,26 +944,51 @@ export function assembleRenewals( const statedValueCents = noticeOpen.reduce((sum, row) => sum + (row.valueCents ?? 0), 0); const anyStatedValue = noticeOpen.some((row) => row.valueCents != null); + const sideLabel = side ? `${side}-side contract(s) on the book` : 'contract(s) on the book'; + const listed = Math.min(renewals.length, EXEMPLARS); + return { headline: (nearest - ? `${truncated ? 'At least ' : ''}${renewals.length} executed contract(s) still live` + - `${side ? ` on the ${side} side` : ''}. Nearest deadline: the ` + + ? `${truncated ? 'At least ' : ''}${renewals.length} of ${totalContracts} ${sideLabel} ` + + 'are executed and not yet expired' + + `${renewals.length > listed ? `; the nearest ${listed} are listed` : ''}. ` + + 'Nearest deadline: the ' + `${nearest.deadlineKind === 'renewal_notice' ? 'renewal notice' : 'expiry'} for ` + `${nearest.title}${nearest.accountName ? ` (${nearest.accountName})` : ''} on ` + `${nearest.deadlineAt.slice(0, 10)}` + `${nearest.daysUntilDeadline < 0 ? ', which has already passed' : ''}.` - : `No executed contract${side ? ` on the ${side} side` : ''} has an expiry date ahead of it.`) + + : `None of the ${totalContracts} ${sideLabel} is executed with an expiry date ahead of it.`) + (noticeOpen.length > 0 - ? ` ${noticeOpen.length} notice window(s) already open` + + ? ` ${noticeOpen.length} of those ${renewals.length} have a notice window already open` + (anyStatedValue ? `, covering ${formatCents(statedValueCents)} of stated contract value.` : '; none of those contracts states a value of its own.') : ''), + scope: resultScope({ + covers: 'are executed, not terminated and not yet expired', + matched: renewals.length, + total: totalContracts, + totalLabel: sideLabel, + listed, + filters: { side: side ?? 'both', status: 'executed', expired: 'excluded' }, + truncated, + }), side: side ?? 'both', truncated, count: renewals.length, + totalContracts, noticeWindowOpenCount: noticeOpen.length, + /** A filter over a filter, so it states its own denominator too. */ + noticeWindowOpenScope: resultScope({ + covers: 'have a renewal-notice window that is already open', + matched: noticeOpen.length, + total: renewals.length, + totalLabel: `executed, unexpired ${sideLabel}`, + listed: 0, + filters: { renewalState: 'due' }, + truncated, + }), renewals: renewals.slice(0, EXEMPLARS), }; } @@ -836,11 +1017,19 @@ interface InventoryQuery { * bounded read. The width never leaves this process; only EXEMPLARS rows do. */ async function listInventory(db: Database, query: InventoryQuery): Promise { - const listings = await new CapacityService(db).searchInventory({ - minGpuCount: query.minGpuCount, - requiresHighSpeedInterconnect: query.requiresFastInterconnect, - limit: SCAN_LIMIT, - }); + const capacity = new CapacityService(db); + // The unfiltered read is the denominator, and it is taken through the same + // service rather than counted here: the service decides what "purchasable" + // means (it drops Unavailable stock), and a denominator computed from a + // second definition of that word would disagree with its own numerator. + const [listings, market] = await Promise.all([ + capacity.searchInventory({ + minGpuCount: query.minGpuCount, + requiresHighSpeedInterconnect: query.requiresFastInterconnect, + limit: SCAN_LIMIT, + }), + capacity.searchInventory({ limit: SCAN_LIMIT }), + ]); const providerNames = await accountNames( db, listings.flatMap((listing) => (listing.accountId ? [listing.accountId] : [])), @@ -850,6 +1039,8 @@ async function listInventory(db: Database, query: InventoryQuery): Promise= SCAN_LIMIT, providerNames, + totalListings: market.length, + totalTruncated: market.length >= SCAN_LIMIT, }); } @@ -879,9 +1070,15 @@ export type InventoryOffer = Pick< export function assembleInventoryResult( query: InventoryQuery, listings: readonly InventoryOffer[], - options: { truncated: boolean; providerNames: ReadonlyMap }, + options: { + truncated: boolean; + providerNames: ReadonlyMap; + /** Purchasable listings on the market with no filter applied at all. */ + totalListings: number; + totalTruncated: boolean; + }, ): unknown { - const { truncated, providerNames: providers } = options; + const { truncated, providerNames: providers, totalListings, totalTruncated } = options; const needle = query.gpuType?.toLowerCase(); const matched = needle ? listings.filter((listing) => listing.gpuType.toLowerCase().includes(needle)) @@ -894,23 +1091,43 @@ export function assembleInventoryResult( ); const cheapest = ranked.find((listing) => listing.onDemandPriceCents != null); + const filters = { + gpuType: query.gpuType ?? null, + minGpuCount: query.minGpuCount ?? null, + requiresFastInterconnect: query.requiresFastInterconnect ?? false, + }; + const anyFilter = Object.values(filters).some((value) => value !== null && value !== false); + const listed = Math.min(ranked.length, EXEMPLARS); + const market = 'purchasable listing(s) on the market'; + return { headline: ranked.length === 0 - ? `No provider is currently listing capacity matching that request${query.gpuType ? ` for ${query.gpuType}` : ''}.` - : `${truncated ? 'At least ' : ''}${ranked.length} purchasable listing(s)` + - `${query.gpuType ? ` matching ${query.gpuType}` : ''}` + + ? `None of the ${atLeast(totalListings, totalTruncated)} ${market} matches that ` + + `request${query.gpuType ? ` for ${query.gpuType}` : ''}.` + : `${ranked.length} of ${atLeast(totalListings, totalTruncated)} ${market} match` + + `${anyFilter ? ' the filters given' : ' (no filter was applied)'}` + + `${query.gpuType ? `, including ${query.gpuType}` : ''}` + (cheapest ? `; cheapest on-demand is ${formatCents(cheapest.onDemandPriceCents ?? 0)} per ` + `GPU-hour for ${cheapest.gpuType}.` - : '; none of them carry a published on-demand price.'), + : '; none of them carry a published on-demand price.') + + ` ${listed} listed here.`, + scope: resultScope({ + covers: anyFilter ? 'match the filters given' : 'are purchasable', + matched: ranked.length, + total: totalListings, + totalLabel: market, + listed, + // An unasked-for filter is not a filter: passing the three nulls through + // would have an unfiltered result describe itself as a slice. + filters: anyFilter ? filters : {}, + truncated: truncated || totalTruncated, + }), truncated, count: ranked.length, - filters: { - gpuType: query.gpuType ?? null, - minGpuCount: query.minGpuCount ?? null, - requiresFastInterconnect: query.requiresFastInterconnect ?? false, - }, + totalListings, + filters, listings: ranked.slice(0, EXEMPLARS).map((listing) => shapeListing(listing, providers)), }; } diff --git a/apps/piggy/src/lifecycle-tools.ts b/apps/piggy/src/lifecycle-tools.ts index 1f002ab..b63709f 100644 --- a/apps/piggy/src/lifecycle-tools.ts +++ b/apps/piggy/src/lifecycle-tools.ts @@ -9,12 +9,16 @@ import { demandDeals, type Database, } from '@pig/db'; -import { and, desc, eq, inArray } from 'drizzle-orm'; +import { and, count, desc, eq, inArray } from 'drizzle-orm'; import { z } from 'zod'; +import { resultScope } from './page-tools'; import { defineTool, type AgentTool } from './provider'; const noInput = z.object({}).strict(); +/** The per-collection cap here, matching `RELATED_LIMIT` in chat-tools. */ +const RELATED_LIMIT = 100; + export function createAccountLifecycleTool(db: Database, accountId: string): AgentTool { return defineTool({ name: 'pig_get_account_lifecycle', @@ -23,11 +27,17 @@ export function createAccountLifecycleTool(db: Database, accountId: string): Age execute: async () => { const [account] = await db.select().from(accounts).where(eq(accounts.id, accountId)).limit(1); if (!account) throw new Error('The account in focus no longer exists.'); - const [deals, paperwork, recentActivity] = await Promise.all([ - db.select().from(demandDeals).where(eq(demandDeals.accountId, accountId)).limit(100), - db.select().from(contracts).where(and(eq(contracts.accountId, accountId), eq(contracts.side, 'demand'))).limit(100), + const [deals, paperwork, recentActivity, dealsOnBook] = await Promise.all([ + db.select().from(demandDeals).where(eq(demandDeals.accountId, accountId)).limit(RELATED_LIMIT), + db.select().from(contracts).where(and(eq(contracts.accountId, accountId), eq(contracts.side, 'demand'))).limit(RELATED_LIMIT), db.select().from(activities).where(eq(activities.accountId, accountId)).orderBy(desc(activities.occurredAt)).limit(1), + // The denominator. This result is one account's slice of the book and + // every count in it is an account count; without the book's own figure + // beside them, "4 demand deals" is the only deal number in the payload + // and becomes the answer to a question about the whole book. + db.select({ value: count() }).from(demandDeals), ]); + const demandDealsOnBook = dealsOnBook[0]?.value ?? 0; const dealIds = deals.map((deal) => deal.id); const contractIds = paperwork.map((contract) => contract.id); const [requests, reservations, obligations] = await Promise.all([ @@ -36,7 +46,18 @@ export function createAccountLifecycleTool(db: Database, accountId: string): Age contractIds.length ? db.select().from(contractObligations).where(inArray(contractObligations.contractId, contractIds)) : [], ]); return { + scope: resultScope({ + covers: `belong to the account ${account.name}`, + matched: deals.length, + total: demandDealsOnBook, + totalLabel: 'demand deal(s) on the book', + listed: 0, + filters: { accountId, side: 'demand', rowCapPerCollection: RELATED_LIMIT }, + truncated: deals.length >= RELATED_LIMIT || paperwork.length >= RELATED_LIMIT, + }), account: { id: account.id, name: account.name }, + demandDealsForThisAccount: deals.length, + demandDealsOnBook, lifecycle: evaluateCustomerLifecycle({ accountId, deals: deals.map((deal) => ({ ...deal })), @@ -47,7 +68,7 @@ export function createAccountLifecycleTool(db: Database, accountId: string): Age lastActivityAt: recentActivity[0]?.occurredAt ?? account.lastActivityAt, lastActivityId: recentActivity[0]?.id, }), - interpretation: 'Scores rank review attention. Capacity totals mean sold or reserved capacity, not customer workload utilization.', + interpretation: 'Scores rank review attention. Capacity totals mean sold or reserved capacity, not customer workload utilisation. Every figure here covers this one account, never the book.', }; }, }); diff --git a/apps/piggy/src/page-tools.ts b/apps/piggy/src/page-tools.ts index 1f3ede7..0bee6d8 100644 --- a/apps/piggy/src/page-tools.ts +++ b/apps/piggy/src/page-tools.ts @@ -48,7 +48,7 @@ import { type Database, } from '@pig/db'; import { CalendarService } from '@pig/api/src/services/calendar'; -import { and, gte, inArray, isNull } from 'drizzle-orm'; +import { and, count, gte, inArray, isNull } from 'drizzle-orm'; import { z } from 'zod'; import { piggyPageGuide, type PiggyPageToolName } from './page-routes'; import { defineTool, type AgentTool } from './provider'; @@ -84,8 +84,107 @@ const TRUNCATION_NOTE = 'book — present them as a lower bound, not as the whole.'; /** Prefixes a count the model must not read as exact. */ -function atLeast(count: number, truncated: boolean): string { - return truncated ? `at least ${count}` : `${count}`; +export function atLeast(rows: number, truncated: boolean): string { + return truncated ? `at least ${rows}` : `${rows}`; +} + +/** + * What a result covers, and what it was drawn from. + * + * Measured in production on /capacity, an hour before this was written. Asked + * "How many capacity commitments are on the book?", the model called + * `pig_get_idle_capacity` — the only tool that page offers — and answered "3". + * The book held 5. The tool filters to blocks at least 25% unsold, so 3 was the + * size of a filter; nothing in the payload said so, and reading the length of + * the list it had been handed as the size of the book was the only reading the + * data supported. + * + * The system prompt already forbade exactly that, naming this exact tool, and + * the model did it anyway. Prompting a 30B model out of a mistake its data + * invites does not work, so the data stopped inviting it: every result here + * that carries a count or a collection carries this object beside it, naming + * the filter, the denominator it was drawn from, and how much of the matched + * set is actually listed. A filtered count is therefore never the only number + * in its own result. + * + * One shape to learn rather than one per tool — a different shape per tool is + * how this happened. The result's primary subject gets a top-level `scope`; + * every other count or collection in the same payload gets its own, nested + * where the payload already groups it and named `Scope` where it does + * not. And `summary` restates the numbers as prose on purpose — it is the + * field a small model quotes, and a figure it has to assemble out of three + * other fields is a figure it will assemble wrongly. + * + * `filters` is not decoration either. This product has shipped three different + * idle figures across three surfaces because each applied its own threshold, so + * a result that does not name the threshold it used cannot be reconciled with + * the screen beside it. + */ +export interface ResultScope { + /** The whole scope in one sentence, figures included. Quote this. */ + summary: string; + /** What made a row match, as a clause: "are at least 25% unsold". */ + covers: string; + /** How many rows matched. Never the answer to "how many are there". */ + matched: number; + /** The set `matched` was drawn from. This is the total. */ + total: number; + /** What `total` counts, as a noun phrase. */ + totalLabel: string; + /** How many of `matched` this payload lists. The rest are counted only. */ + listed: number; + /** Every filter applied, named, so a threshold is never invisible. */ + filters: Record; + /** True when a read hit its row cap, so both figures are lower bounds. */ + truncated: boolean; +} + +export function resultScope(input: { + covers: string; + matched: number; + total: number; + totalLabel: string; + listed: number; + filters?: Record; + truncated?: boolean; +}): ResultScope { + const { covers, matched, total, totalLabel, listed } = input; + const filters = input.filters ?? {}; + const truncated = input.truncated ?? false; + // Nothing was filtered out, so `matched` IS the total and saying otherwise + // would teach the model to distrust a figure that is exact. + const unfiltered = matched === total && Object.keys(filters).length === 0; + const listedClause = listed > 0 ? `; ${listed} listed here` : ''; + // Both figures are hedged together when a read was cut. Hedging only the + // total would present a capped `matched` as exact, which is the same class of + // overstatement this whole object exists to stop. + const summary = unfiltered + ? `All ${atLeast(total, truncated)} ${totalLabel}${listedClause}.` + : `${atLeast(matched, truncated)} of ${atLeast(total, truncated)} ${totalLabel} ` + + `${covers}${listedClause}. That is a filtered count — the total is ` + + `${atLeast(total, truncated)} ${totalLabel}.`; + return { + summary: truncated ? `${summary} ${TRUNCATION_NOTE}` : summary, + covers, + matched, + total, + totalLabel, + listed, + filters, + truncated, + }; +} + +/** The denominator every commitment figure in this file is drawn from. */ +const COMMITMENTS_LABEL = 'live capacity commitment(s) on the book'; + +/** The denominators the two pipelines are drawn from. */ +const DEMAND_DEALS_LABEL = 'demand deal(s) on the book'; +const SUPPLY_DEALS_LABEL = 'supply deal(s) on the book'; + +/** One whole-table count, for use as a denominator. */ +function rowCount(rows: readonly { value: number }[]): number { + return rows[0]?.value ?? 0; } /** @@ -306,9 +405,24 @@ async function readMarginSummary(db: Database): Promise { headline: `Revenue ${formatCents(totals.revenueCents)} against cost ${formatCents(totals.costCents)}; ` + `gross margin ${formatCents(totals.grossMarginCents)} (${percent(totals.grossMarginPct)}) ` + - `at ${percent(totals.utilisation)} utilisation across ` + - `${atLeast(blocks.length, truncated)} live commitment(s).` + + `at ${percent(totals.utilisation)} utilisation across all ` + + `${atLeast(blocks.length, truncated)} ${COMMITMENTS_LABEL} — the whole book, unfiltered. ` + + `The ${largest.length} largest by cost are listed; the book holds ` + + `${atLeast(blocks.length, truncated)}.` + (truncated ? ` ${TRUNCATION_NOTE}` : ''), + /** + * Unfiltered, and the only tool here that is: `matched` equals `total`, so + * `liveCommitments` below is a real answer to "how many are on the book". + * `largestBlocks` is still a slice, which is what `listed` is for. + */ + scope: resultScope({ + covers: 'are live', + matched: blocks.length, + total: blocks.length, + totalLabel: COMMITMENTS_LABEL, + listed: largest.length, + truncated, + }), truncated, totals: { revenueCents: totals.revenueCents, @@ -332,13 +446,27 @@ async function readMarginSummary(db: Database): Promise { }; } +/** + * The threshold and horizon this tool filters on. + * + * The same defaults the API and MCP use — and NOT the same as the workspace + * summary's worst-idle list, which takes any block with idle hours at all. + * Both are correct for what they answer and they return different counts, so + * each states its own threshold in `filters` rather than leaving the reader to + * reconcile two figures that were never the same figure. + */ +const IDLE_THRESHOLD_PCT = 0.25; +const IDLE_WITHIN_DAYS = 30; + /** Idle blocks, on the same defaults the API and MCP use: 25% within 30 days. */ async function readIdleCapacity(db: Database): Promise { const now = new Date(); - const horizon = new Date(now.getTime() + 30 * 86_400_000); + const horizon = new Date(now.getTime() + IDLE_WITHIN_DAYS * 86_400_000); const { blocks, truncated } = await readLiveBlocks(db, now); const idle = blocks - .filter((block) => block.startsAt <= horizon && 1 - block.margin.utilisation >= 0.25) + .filter( + (block) => block.startsAt <= horizon && 1 - block.margin.utilisation >= IDLE_THRESHOLD_PCT, + ) .map((block) => ({ block, idleGpuHours: block.margin.idleGpuHours, @@ -348,19 +476,42 @@ async function readIdleCapacity(db: Database): Promise { .sort((a, b) => b.idleCostCents - a.idleCostCents); const totalIdleCostCents = idle.reduce((sum, row) => sum + row.idleCostCents, 0); + const listed = idle.slice(0, EXEMPLARS); + const filter = + `are at least ${IDLE_THRESHOLD_PCT * 100}% unsold and start within ` + + `${IDLE_WITHIN_DAYS} day(s)`; return { + /** + * The sentence the production defect was answered from, so it carries the + * denominator first and the filtered figure second. "3" alone was true of + * the filter and false of the book; "3 of 5" cannot be misread as 5. + */ headline: (idle.length === 0 - ? 'No live block is more than 25% unsold within the next 30 days.' - : `${atLeast(idle.length, truncated)} block(s) at least 25% unsold within 30 days, ` + - `${formatCents(totalIdleCostCents)} of capacity bought and not yet earning.`) + + ? `None of the ${atLeast(blocks.length, truncated)} ${COMMITMENTS_LABEL} ${filter}.` + : `${idle.length} of ${atLeast(blocks.length, truncated)} ${COMMITMENTS_LABEL} ${filter}, ` + + `${formatCents(totalIdleCostCents)} of capacity bought and not yet earning. ` + + `${idle.length} is a filtered count — the book holds ` + + `${atLeast(blocks.length, truncated)} live commitment(s) in total.`) + (truncated ? ` ${TRUNCATION_NOTE}` : ''), + scope: resultScope({ + covers: filter, + matched: idle.length, + total: blocks.length, + totalLabel: COMMITMENTS_LABEL, + listed: listed.length, + filters: { idleThresholdPct: IDLE_THRESHOLD_PCT, withinDays: IDLE_WITHIN_DAYS }, + truncated, + }), truncated, - thresholdPct: 0.25, - withinDays: 30, + /** The denominator, repeated as a bare field: this is the size of the book. */ + liveCommitments: blocks.length, + thresholdPct: IDLE_THRESHOLD_PCT, + withinDays: IDLE_WITHIN_DAYS, + idleBlocks: idle.length, totalIdleCostCents, - blocks: idle.slice(0, EXEMPLARS).map((row) => ({ + blocks: listed.map((row) => ({ name: row.block.name, gpuType: row.block.gpuType, gpuCount: row.block.gpuCount, @@ -379,7 +530,10 @@ async function readIdleCapacity(db: Database): Promise { // --------------------------------------------------------------------------- async function readPipeline(db: Database): Promise { - const [demandRead, supplyRead] = await Promise.all([ + // The two whole-table counts are the denominators. Without them "12 open + // demand deals" is a filtered count with nothing to be filtered from, and + // "how many deals do we have" is answered with the number of open ones. + const [demandRead, supplyRead, demandAll, supplyAll] = await Promise.all([ db .select() .from(demandDeals) @@ -390,10 +544,17 @@ async function readPipeline(db: Database): Promise { .from(supplyDeals) .where(inArray(supplyDeals.stage, [...SUPPLY_OPEN_STAGES])) .limit(SCAN_LIMIT + 1), + db.select({ value: count() }).from(demandDeals), + db.select({ value: count() }).from(supplyDeals), ]); const { rows: demand, truncated: demandTruncated } = bounded(demandRead); const { rows: supply, truncated: supplyTruncated } = bounded(supplyRead); const truncated = demandTruncated || supplyTruncated; + const demandTotal = rowCount(demandAll); + const supplyTotal = rowCount(supplyAll); + const demandListed = Math.min(demand.length, EXEMPLARS); + const supplyListed = Math.min(supply.length, EXEMPLARS); + const openStages = 'are at an open stage (not closed, not lost)'; // Total contract value where it is known, annual value otherwise: a deal // valued only by ACV is still worth counting, and treating it as zero would @@ -403,13 +564,35 @@ async function readPipeline(db: Database): Promise { return { headline: - `${atLeast(demand.length, demandTruncated)} open demand deal(s) worth ` + - `${formatCents(demandValueCents)} and ${atLeast(supply.length, supplyTruncated)} ` + - 'open supply deal(s).' + + `${demand.length} of ${atLeast(demandTotal, demandTruncated)} ${DEMAND_DEALS_LABEL} are open, ` + + `worth ${formatCents(demandValueCents)}, and ${supply.length} of ` + + `${atLeast(supplyTotal, supplyTruncated)} ${SUPPLY_DEALS_LABEL} are open — ` + + `${demand.length + supply.length} of ${atLeast(demandTotal + supplyTotal, truncated)} ` + + 'deal(s) on the book in all. Those are open-stage counts, not the size of either pipeline.' + (truncated ? ` ${TRUNCATION_NOTE}` : ''), + // Both sides together, so a question about "deals" has a denominator too. + scope: resultScope({ + covers: openStages, + matched: demand.length + supply.length, + total: demandTotal + supplyTotal, + totalLabel: 'deal(s) on the book, demand and supply together', + listed: demandListed + supplyListed, + filters: { stages: 'open only' }, + truncated, + }), truncated: { demandDeals: demandTruncated, supplyDeals: supplyTruncated }, demand: { + scope: resultScope({ + covers: openStages, + matched: demand.length, + total: demandTotal, + totalLabel: DEMAND_DEALS_LABEL, + listed: demandListed, + filters: { stages: [...DEMAND_OPEN_STAGES].join(', ') }, + truncated: demandTruncated, + }), openDeals: demand.length, + totalDeals: demandTotal, valueCents: demandValueCents, byStage: countByStage(demand.map((deal) => deal.stage)), largest: [...demand] @@ -423,7 +606,17 @@ async function readPipeline(db: Database): Promise { })), }, supply: { + scope: resultScope({ + covers: openStages, + matched: supply.length, + total: supplyTotal, + totalLabel: SUPPLY_DEALS_LABEL, + listed: supplyListed, + filters: { stages: [...SUPPLY_OPEN_STAGES].join(', ') }, + truncated: supplyTruncated, + }), openDeals: supply.length, + totalDeals: supplyTotal, byStage: countByStage(supply.map((deal) => deal.stage)), largest: [...supply] .sort((a, b) => (b.gpuCount ?? 0) - (a.gpuCount ?? 0)) @@ -510,10 +703,22 @@ async function readCalendarAhead(db: Database, withinDays: number): Promise event.state === 'overdue'); const truncated = ahead.truncated || behind.truncated; const upcomingByKind = countByKind(upcoming); + const upcomingListed = Math.min(upcoming.length, EXEMPLARS * 2); + const overdueListed = Math.min(overdue.length, EXEMPLARS); + /** + * Both denominators are windows, not the book, and the labels say so. Nothing + * here can answer "how many obligations are there" — only how many fall in + * these dates — so a label that read "obligations on the book" would be the + * same lie in a different tool. + */ + const windowLabel = `dated item(s) falling in the next ${withinDays} day(s), done or not`; + const lookbackLabel = `dated item(s) in the last ${OVERDUE_LOOKBACK_DAYS} day(s) that can fall late`; return { headline: - `Next ${withinDays} day(s): ${atLeast(upcoming.length, ahead.truncated)} dated item(s) ` + + `Next ${withinDays} day(s) only, not the whole book: ` + + `${atLeast(upcoming.length, ahead.truncated)} of ` + + `${atLeast(ahead.events.length, ahead.truncated)} dated item(s) ` + `across ${Object.keys(upcomingByKind).length} kind(s), of which ` + `${ahead.totals.obligationCount} obligation(s) due, ${ahead.totals.closingCount} demand ` + `deal(s) expected to close worth ${formatCents(ahead.totals.weightedPipelineCents)} ` + @@ -522,6 +727,15 @@ async function readCalendarAhead(db: Database, withinDays: number): Promise * answered from the fragment it happened to receive. */ async function readWorkspaceSummary(db: Database): Promise { - const [book, demandRead, supplyRead] = await Promise.all([ + const [book, demandRead, supplyRead, demandAll, supplyAll] = await Promise.all([ readLiveBlocks(db), db .select({ id: demandDeals.id }) @@ -609,8 +838,12 @@ async function readWorkspaceSummary(db: Database): Promise { .from(supplyDeals) .where(inArray(supplyDeals.stage, [...SUPPLY_OPEN_STAGES])) .limit(SCAN_LIMIT + 1), + db.select({ value: count() }).from(demandDeals), + db.select({ value: count() }).from(supplyDeals), ]); const { blocks } = book; + const demandTotal = rowCount(demandAll); + const supplyTotal = rowCount(supplyAll); const { rows: demand, truncated: demandTruncated } = bounded(demandRead); const { rows: supply, truncated: supplyTruncated } = bounded(supplyRead); const truncated = { @@ -620,23 +853,42 @@ async function readWorkspaceSummary(db: Database): Promise { }; const anyTruncated = Object.values(truncated).some(Boolean); const totals = bookTotals(blocks); - const worstIdle = [...blocks] + /** + * Any idle at all, which is a THIRD threshold — `pig_get_idle_capacity` uses + * 25% and the web uses its own. Three surfaces have quoted three different + * idle counts for one book because of exactly this, so the scope below names + * the threshold rather than leaving the reader to guess which one produced + * the number in front of them. + */ + const withIdle = [...blocks] .filter((block) => block.margin.idleGpuHours > 0) .sort( (a, b) => b.margin.idleGpuHours * b.costPerGpuHourCents - a.margin.idleGpuHours * a.costPerGpuHourCents, - ) - .slice(0, 3); + ); + const worstIdle = withIdle.slice(0, 3); return { headline: - `${atLeast(blocks.length, truncated.commitments)} live commitment(s) at ` + + `All ${atLeast(blocks.length, truncated.commitments)} ${COMMITMENTS_LABEL} at ` + `${percent(totals.utilisation)} utilisation; ` + `gross margin ${formatCents(totals.grossMarginCents)}; ` + - `${atLeast(demand.length, demandTruncated)} open demand and ` + - `${atLeast(supply.length, supplyTruncated)} open supply deal(s).` + + `${demand.length} of ${atLeast(demandTotal, demandTruncated)} ${DEMAND_DEALS_LABEL} ` + + `and ${supply.length} of ${atLeast(supplyTotal, supplyTruncated)} ${SUPPLY_DEALS_LABEL} ` + + `are open. The ${worstIdle.length} block(s) listed below are the worst idle of ` + + `${withIdle.length} with any idle hours, not the whole book.` + (anyTruncated ? ` ${TRUNCATION_NOTE}` : ''), + // The book, unfiltered: `matched` equals `total`, so this is the figure to + // quote when someone asks how large the book is. + scope: resultScope({ + covers: 'are live', + matched: blocks.length, + total: blocks.length, + totalLabel: COMMITMENTS_LABEL, + listed: 0, + truncated: truncated.commitments, + }), truncated, book: { liveCommitments: blocks.length, @@ -646,14 +898,50 @@ async function readWorkspaceSummary(db: Database): Promise { utilisation: totals.utilisation, idleGpuHours: Math.round(totals.idleGpuHours), }, + // Two filtered counts, each next to the denominator it came from. Without + // `totalDemandDeals` beside it, `openDemandDeals` is the only deal figure + // in the payload and becomes the answer to "how many deals do we have". openDemandDeals: demand.length, + totalDemandDeals: demandTotal, + openDemandDealsScope: resultScope({ + covers: 'are at an open stage (not closed, not lost)', + matched: demand.length, + total: demandTotal, + totalLabel: DEMAND_DEALS_LABEL, + listed: 0, + filters: { stages: [...DEMAND_OPEN_STAGES].join(', ') }, + truncated: demandTruncated, + }), openSupplyDeals: supply.length, - worstIdleBlocks: worstIdle.map((block) => ({ - name: block.name, - gpuType: block.gpuType, - idleGpuHours: Math.round(block.margin.idleGpuHours), - idleCostCents: Math.round(block.margin.idleGpuHours * block.costPerGpuHourCents), - })), + totalSupplyDeals: supplyTotal, + openSupplyDealsScope: resultScope({ + covers: 'are at an open stage (not closed, not lost)', + matched: supply.length, + total: supplyTotal, + totalLabel: SUPPLY_DEALS_LABEL, + listed: 0, + filters: { stages: [...SUPPLY_OPEN_STAGES].join(', ') }, + truncated: supplyTruncated, + }), + worstIdle: { + scope: resultScope({ + covers: 'have any unsold hours at all', + matched: withIdle.length, + total: blocks.length, + totalLabel: COMMITMENTS_LABEL, + listed: worstIdle.length, + // Not 0.25. This list and pig_get_idle_capacity answer different + // questions and will disagree; the thresholds say which is which. + filters: { idleThresholdPct: 0 }, + truncated: truncated.commitments, + }), + blocks: worstIdle.map((block) => ({ + name: block.name, + gpuType: block.gpuType, + idleGpuHours: Math.round(block.margin.idleGpuHours), + idleCostCents: Math.round(block.margin.idleGpuHours * block.costPerGpuHourCents), + })), + }, }; } diff --git a/apps/piggy/test/lookup-tools.test.ts b/apps/piggy/test/lookup-tools.test.ts index cc10091..82c827e 100644 --- a/apps/piggy/test/lookup-tools.test.ts +++ b/apps/piggy/test/lookup-tools.test.ts @@ -20,6 +20,7 @@ import test from 'node:test'; import type { Database } from '@pig/db'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { assertPigToolBoundary } from '../src/chat'; +import type { ResultScope } from '../src/page-tools'; import { assembleInventoryResult, assembleRenewals, @@ -184,6 +185,24 @@ test('every parameter description survives into the emitted schema', () => { // Search shaping // --------------------------------------------------------------------------- +/** + * The five denominators, roughly the demo book's own shape. + * + * A search reports how many rows it matched; without these it would be the only + * count in its own payload, and "3 accounts match" is one careless sentence away + * from "we have 3 accounts". + */ +const TOTALS = { + account: 23, + demand_deal: 13, + supply_deal: 8, + contract: 20, + commitment: 6, +} as const; + +/** Every row in every searched table: the denominator the headline quotes. */ +const SEARCHABLE = Object.values(TOTALS).reduce((sum, rows) => sum + rows, 0); + const emptySets: SearchRowSets = { accounts: [], demandDeals: [], @@ -191,6 +210,7 @@ const emptySets: SearchRowSets = { contracts: [], commitments: [], accountNames: new Map(), + totals: { ...TOTALS }, }; function account(name: string, id = name): SearchRowSets['accounts'][number] { @@ -201,6 +221,8 @@ interface SearchReading { headline: string; truncated: boolean; counts: Record; + totals: Record; + scope: ResultScope; results: { type: string; id: string; name: string }[]; } @@ -256,13 +278,19 @@ test('a search result is capped per type and overall, and says when it was cut', assert.equal(reading.truncated, true); // The model quotes the headline, so the hedge has to live in it rather than // in a `truncated` flag further down the payload. - assert.match(reading.headline, /at least 5 record\(s\) match "alpha"/); + assert.match(reading.headline, new RegExp(`At least 5 of ${SEARCHABLE} searchable record\\(s\\)`)); + // The denominator travels with the hedge: a capped match count next to the + // number of rows it was drawn from cannot be read as "we have five accounts". + assert.equal(reading.scope.matched, 5); + assert.equal(reading.scope.total, SEARCHABLE); + assert.equal(reading.totals.account, TOTALS.account); }); test('the overall cap holds even when no single type reached its own', () => { const three = (prefix: string) => Array.from({ length: 3 }, (_, i) => `${prefix} ${i}`); const reading = assembleSearchResult('block', { + totals: { ...TOTALS }, accounts: three('block acct').map((name) => account(name, name)), demandDeals: three('block demand').map((name) => ({ id: name, @@ -355,7 +383,10 @@ test('a search that matches nothing says so rather than returning a bare empty l const reading = assembleSearchResult('nobody', emptySets) as SearchReading; assert.equal(reading.results.length, 0); assert.equal(reading.truncated, false); - assert.match(reading.headline, /No account, deal, contract or capacity commitment/); + assert.match(reading.headline, new RegExp(`None of the ${SEARCHABLE} account\\(s\\)`)); + // Even an empty search states the size of what it looked through. + assert.equal(reading.scope.matched, 0); + assert.equal(reading.scope.total, SEARCHABLE); }); // --------------------------------------------------------------------------- @@ -378,10 +409,15 @@ function contract(overrides: Partial & { id: string }): Renewal }; } +/** Contracts of every status on the book — the renewal list's denominator. */ +const CONTRACTS_ON_BOOK = 20; + interface RenewalReading { headline: string; truncated: boolean; + scope: ResultScope; count: number; + totalContracts: number; noticeWindowOpenCount: number; renewals: { id: string; @@ -409,7 +445,7 @@ test('a lapsed notice outranks a nearer expiry, because the decision is the dead accountName: 'Halcyon', }, ], - { now: NOW, truncated: false }, + { now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK }, ) as RenewalReading; assert.deepEqual(reading.renewals.map((row) => row.id), ['missed', 'soon']); @@ -441,7 +477,7 @@ test('an open notice window on unpriced paper is not reported as worth nothing', accountName: 'Halcyon', }, ], - { now: NOW, truncated: false }, + { now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK }, ) as RenewalReading; assert.equal(reading.noticeWindowOpenCount, 1); @@ -452,7 +488,7 @@ test('an open notice window on unpriced paper is not reported as worth nothing', test('a contract that cannot auto-renew has an expiry deadline and no notice state', () => { const reading = assembleRenewals( [{ contract: contract({ id: 'plain' }), accountName: 'Verity Health AI' }], - { now: NOW, truncated: false }, + { now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK }, ) as RenewalReading; const [row] = reading.renewals; @@ -468,20 +504,25 @@ test('the renewal count covers the whole set while the list is capped', () => { contract: contract({ id: `c${i}`, expiresAt: new Date(NOW.getTime() + (i + 1) * DAY) }), accountName: null, })); - const reading = assembleRenewals(rows, { now: NOW, truncated: true }) as RenewalReading; + const reading = assembleRenewals(rows, { now: NOW, truncated: true, totalContracts: CONTRACTS_ON_BOOK }) as RenewalReading; assert.equal(reading.count, 14); assert.equal(reading.renewals.length, 8); assert.equal(reading.truncated, true); // A capped list quoted as a total is the defect this whole pattern exists to // prevent, so the hedge has to reach the headline. - assert.match(reading.headline, /At least 14 executed contract\(s\)/); + assert.match( + reading.headline, + new RegExp(`At least 14 of ${CONTRACTS_ON_BOOK} contract\\(s\\) on the book are executed`), + ); + assert.equal(reading.scope.matched, 14); + assert.equal(reading.scope.total, CONTRACTS_ON_BOOK); }); test('an empty book states the absence rather than implying nothing is due', () => { - const reading = assembleRenewals([], { now: NOW, side: 'supply', truncated: false }) as RenewalReading; + const reading = assembleRenewals([], { now: NOW, side: 'supply', truncated: false, totalContracts: 8 }) as RenewalReading; assert.equal(reading.count, 0); - assert.match(reading.headline, /No executed contract on the supply side/); + assert.match(reading.headline, /None of the 8 supply-side contract\(s\) on the book/); }); // --------------------------------------------------------------------------- @@ -508,10 +549,15 @@ function offer(overrides: Partial & { gpuType: string }): Invent const providerNames = new Map([['provider-1', 'RunPod']]); +/** Purchasable listings on the market with no filter at all: the denominator. */ +const LISTINGS_ON_MARKET = 30; + interface InventoryReading { headline: string; truncated: boolean; + scope: ResultScope; count: number; + totalListings: number; listings: { gpuType: string; providerName: string | null; @@ -527,7 +573,7 @@ test('offers are cheapest first, and an unpriced one sorts last rather than free offer({ gpuType: 'QUOTE_ONLY', onDemandPriceCents: null }), offer({ gpuType: 'H100_80GB', onDemandPriceCents: 189 }), ], - { truncated: false, providerNames }, + { truncated: false, providerNames, totalListings: LISTINGS_ON_MARKET, totalTruncated: false }, ) as InventoryReading; assert.deepEqual(reading.listings.map((row) => row.gpuType), [ @@ -546,7 +592,7 @@ test('a GPU-type fragment matches the SKU, because a model asks for H100', () => const reading = assembleInventoryResult( { gpuType: 'h100' }, [offer({ gpuType: 'H100_80GB' }), offer({ gpuType: 'H200' })], - { truncated: false, providerNames }, + { truncated: false, providerNames, totalListings: LISTINGS_ON_MARKET, totalTruncated: false }, ) as InventoryReading; assert.equal(reading.count, 1); @@ -560,20 +606,28 @@ test('the offer list is capped and the count is not', () => { const reading = assembleInventoryResult({}, many, { truncated: true, providerNames, + totalListings: 20, + totalTruncated: true, }) as InventoryReading; assert.equal(reading.count, 20); assert.equal(reading.listings.length, 8); assert.equal(reading.listings[0]?.onDemandPricePerGpuHourCents, 281); - assert.match(reading.headline, /At least 20 purchasable listing\(s\)/); + assert.match(reading.headline, /20 of at least 20 purchasable listing\(s\) on the market/); + assert.equal(reading.scope.truncated, true); }); test('no matching offer is reported as an absence, not as an empty market', () => { const reading = assembleInventoryResult({ gpuType: 'MI300X' }, [offer({ gpuType: 'H200' })], { truncated: false, providerNames, + totalListings: LISTINGS_ON_MARKET, + totalTruncated: false, }) as InventoryReading; assert.equal(reading.count, 0); - assert.match(reading.headline, /No provider is currently listing capacity matching that request for MI300X/); + assert.match( + reading.headline, + new RegExp(`None of the ${LISTINGS_ON_MARKET} purchasable listing\\(s\\) on the market matches`), + ); }); diff --git a/apps/piggy/test/result-scope.test.ts b/apps/piggy/test/result-scope.test.ts new file mode 100644 index 0000000..d303fd3 --- /dev/null +++ b/apps/piggy/test/result-scope.test.ts @@ -0,0 +1,413 @@ +/** + * The scope contract, pinned. + * + * This suite exists because of one production answer. Asked "How many capacity + * commitments are on the book?" on /capacity, Piggy called + * `pig_get_idle_capacity` — the only tool that page offers — and said "3". The + * book held 5. The tool filters to blocks at least 25% unsold, so 3 was the + * size of a filter, and the payload gave the model nothing else to read: the + * length of the list it had been handed was the only count in front of it. + * + * The system prompt already forbade that, naming this exact tool. So the guard + * cannot be a prompt and cannot be a convention; it has to be a test that fails + * when a result stops carrying its own denominator. Three things are pinned + * here and nothing else: + * + * 1. every result carrying a count or a collection carries a `scope`; + * 2. a filtered count is never the only count in its own result; + * 3. the threshold that produced a filtered count is named in the payload, + * because three surfaces of this product have quoted three different idle + * figures and the only way to reconcile them is to know which is which. + * + * The page tools are executed against a stub handle rather than Postgres. The + * unit suite runs in CI BEFORE the migration step, so a query here would meet a + * database with no tables; the stub answers the four reads these tools make and + * nothing else, which is enough because what is under test is the shaping, not + * the SQL. `e2e/page-tools.test.ts` covers the SQL against a real book. + */ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + accounts, + allocations, + capacityCommitments, + contacts, + contracts, + demandDeals, + supplyDeals, + type Database, +} from '@pig/db'; +import { createInteractivePigTools } from '../src/chat-tools'; +import { createPagePigTools, resultScope, type ResultScope } from '../src/page-tools'; + +// --------------------------------------------------------------------------- +// The stub handle +// --------------------------------------------------------------------------- + +interface StubBook { + /** Live commitments. The stub does not evaluate where clauses. */ + commitments: readonly Record[]; + allocations: readonly Record[]; + /** The OPEN deals, which is what the row reads in these tools select. */ + demandDeals: readonly Record[]; + supplyDeals: readonly Record[]; + /** Every deal on the book, which is what the `count()` reads select. */ + counts: { demandDeals: number; supplyDeals: number }; + accounts?: readonly Record[]; + contacts?: readonly Record[]; + contracts?: readonly Record[]; +} + +/** + * A thenable that answers one read. + * + * Drizzle's builder is a promise you can keep calling methods on, so the stub + * is the same: every chaining method returns itself and `then` resolves the + * rows. The where clauses are ignored deliberately — a stub that reimplemented + * them would be testing itself. + */ +function stubQuery(rows: readonly unknown[]): Record { + const builder: Record = {}; + for (const method of ['where', 'limit', 'orderBy', 'leftJoin', 'innerJoin', 'innerJoinLateral']) { + builder[method] = () => builder; + } + builder.then = (resolve: (value: readonly unknown[]) => unknown) => resolve(rows); + return builder; +} + +function stubDatabase(book: StubBook): Database { + const rowsFor = (table: unknown): readonly unknown[] => { + if (table === capacityCommitments) return book.commitments; + if (table === allocations) return book.allocations; + if (table === demandDeals) return book.demandDeals; + if (table === supplyDeals) return book.supplyDeals; + if (table === accounts) return book.accounts ?? []; + if (table === contacts) return book.contacts ?? []; + if (table === contracts) return book.contracts ?? []; + throw new Error('the stub was asked for a table this suite does not fixture'); + }; + const countFor = (table: unknown): number => { + if (table === demandDeals) return book.counts.demandDeals; + if (table === supplyDeals) return book.counts.supplyDeals; + return rowsFor(table).length; + }; + const select = (projection?: Record) => ({ + from: (table: unknown) => { + // `select({ value: count() })` is the only projection with that shape, + // and it is how every denominator in these tools is read. + const counting = + projection !== undefined && + Object.keys(projection).length === 1 && + Object.hasOwn(projection, 'value'); + return stubQuery(counting ? [{ value: countFor(table) }] : rowsFor(table)); + }, + }); + return { select } as unknown as Database; +} + +const DAY = 86_400_000; +const now = Date.now(); + +/** One live block: `sold` of `hours` bought at `costCents` per GPU-hour. */ +function block(name: string, hours: number, sold: number, costCents = 100) { + return { + id: name, + name, + gpuType: 'H100_80GB', + gpuCount: 8, + startsAt: new Date(now - 10 * DAY), + endsAt: new Date(now + 100 * DAY), + // numeric columns arrive from Postgres as strings, and so must these. + totalGpuHours: `${hours}.00`, + costPerGpuHourCents: costCents, + sold, + }; +} + +function allocation(commitmentId: string, gpuHours: number) { + return { + capacityCommitmentId: commitmentId, + status: 'committed', + gpuHours: `${gpuHours}.00`, + pricePerGpuHourCents: 120, + holdExpiresAt: null, + }; +} + +/** + * Five live commitments, three of them at least 25% unsold. + * + * The production book was five and the tool returned three. Reproducing that + * ratio exactly is the point: a fixture where the filter happens to keep + * everything cannot fail the way production did. + */ +const BOOK = [ + block('idle-90', 1000, 100), + block('idle-50', 1000, 500), + block('idle-30', 1000, 700), + block('idle-10', 1000, 900), + block('idle-0', 1000, 1000), +]; + +const LIVE_COMMITMENTS = BOOK.length; +const IDLE_BLOCKS = 3; + +const stub = stubDatabase({ + commitments: BOOK.map(({ sold: _sold, ...row }) => row), + allocations: BOOK.filter((row) => row.sold > 0).map((row) => allocation(row.id, row.sold)), + demandDeals: Array.from({ length: 4 }, (_, i) => ({ + id: `demand-${i}`, + name: `Demand ${i}`, + stage: 'proposal', + acvCents: 1_000_000, + tcvCents: 2_500_000, + expectedCloseDate: null, + })), + supplyDeals: Array.from({ length: 2 }, (_, i) => ({ + id: `supply-${i}`, + name: `Supply ${i}`, + stage: 'sourced', + gpuType: 'H200', + gpuCount: 64, + targetCostPerGpuHourCents: 189, + })), + counts: { demandDeals: 13, supplyDeals: 8 }, + accounts: [{ id: 'acct', name: 'DEMO — Halcyon Research' }], + contacts: [{ id: 'contact-1', accountId: 'acct', fullName: 'A Person' }], + contracts: [{ id: 'contract-1', accountId: 'acct', title: 'DEMO — MSA' }], +}); + +type Reading = Record & { headline?: string; scope?: ResultScope }; + +async function read(route: '/margin' | '/capacity' | '/demand' | '/'): Promise { + const [tool] = createPagePigTools(stub, route); + assert.ok(tool, `no tool for ${route}`); + return (await tool.execute({})) as Reading; +} + +/** Every `scope` object anywhere in a result, however deeply it is nested. */ +function scopes(value: unknown, found: ResultScope[] = []): ResultScope[] { + if (Array.isArray(value)) { + for (const entry of value) scopes(entry, found); + return found; + } + if (value === null || typeof value !== 'object') return found; + for (const [key, entry] of Object.entries(value)) { + if (key === 'scope' || key.endsWith('Scope')) found.push(entry as ResultScope); + else scopes(entry, found); + } + return found; +} + +// --------------------------------------------------------------------------- +// The shape itself +// --------------------------------------------------------------------------- + +test('an unfiltered scope says so, rather than hedging a figure that is exact', () => { + const scope = resultScope({ + covers: 'are live', + matched: 5, + total: 5, + totalLabel: 'live capacity commitment(s) on the book', + listed: 5, + }); + assert.equal(scope.summary, 'All 5 live capacity commitment(s) on the book; 5 listed here.'); + assert.equal(scope.matched, scope.total); +}); + +test('a filtered scope states both figures and names the filtered one as filtered', () => { + const scope = resultScope({ + covers: 'are at least 25% unsold', + matched: 3, + total: 5, + totalLabel: 'live capacity commitment(s) on the book', + listed: 3, + filters: { idleThresholdPct: 0.25 }, + }); + // The sentence a small model quotes has to carry the denominator, because a + // field it must reason over is a field it will skip. + assert.match(scope.summary, /3 of 5 live capacity commitment\(s\) on the book/); + assert.match(scope.summary, /the total is 5/); + assert.equal(scope.filters.idleThresholdPct, 0.25); +}); + +test('a truncated read hedges the matched count as well as the total', () => { + const scope = resultScope({ + covers: 'are open', + matched: 500, + total: 500, + totalLabel: 'demand deal(s) on the book', + listed: 8, + filters: { stages: 'open only' }, + truncated: true, + }); + assert.match(scope.summary, /at least 500 of at least 500/); + // Hedging only the total would present a capped match count as exact. + assert.match(scope.summary, /lower bound/); +}); + +// --------------------------------------------------------------------------- +// The measured defect +// --------------------------------------------------------------------------- + +test('the idle tool reports the size of the book beside the size of its filter', async () => { + const reading = await read('/capacity'); + const scope = reading.scope; + assert.ok(scope); + + // Three blocks matched out of five on the book: the production numbers. + assert.equal(scope.matched, IDLE_BLOCKS); + assert.equal(scope.total, LIVE_COMMITMENTS); + assert.equal(reading.idleBlocks, IDLE_BLOCKS); + assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); + + // The headline is what a small model quotes, so the denominator has to be in + // it. "3" alone was true of the filter and false of the book. + assert.match(String(reading.headline), /3 of 5 live capacity commitment\(s\) on the book/); + assert.match(String(reading.headline), /the book holds 5 live commitment\(s\) in total/); + assert.match(scope.summary, /the total is 5/); +}); + +test('the idle tool names the threshold that produced its count', async () => { + const reading = await read('/capacity'); + assert.equal(reading.scope?.filters.idleThresholdPct, 0.25); + assert.equal(reading.scope?.filters.withinDays, 30); + assert.equal(reading.thresholdPct, 0.25); + // Three surfaces of this product have quoted three different idle counts for + // one book. A result that does not say which threshold it used cannot be + // reconciled with the screen beside it. + assert.match(String(reading.headline), /at least 25% unsold/); +}); + +test('the filtered count is never the only count in the idle result', async () => { + const reading = await read('/capacity'); + const listed = reading.blocks; + assert.ok(Array.isArray(listed)); + // Everything that counts blocks: the matched figure, the listed rows, and the + // denominator. The denominator must be present and must differ from them. + const counts = [reading.idleBlocks, listed.length, reading.liveCommitments]; + assert.equal(counts.includes(LIVE_COMMITMENTS), true); + assert.notEqual(reading.idleBlocks, reading.liveCommitments); +}); + +// --------------------------------------------------------------------------- +// The same trap in every other tool +// --------------------------------------------------------------------------- + +test('the margin summary describes itself as the whole book, not a slice', async () => { + const reading = await read('/margin'); + const scope = reading.scope; + assert.ok(scope); + assert.equal(scope.matched, LIVE_COMMITMENTS); + assert.equal(scope.total, LIVE_COMMITMENTS); + assert.equal(reading.liveCommitments, LIVE_COMMITMENTS); + assert.match(String(reading.headline), /all 5 live capacity commitment\(s\) on the book/); + // `largestBlocks` is still a slice, and `listed` is what says so. + assert.equal(scope.listed, LIVE_COMMITMENTS); +}); + +test('both pipelines carry the number of deals they were drawn from', async () => { + const reading = await read('/demand'); + const demand = reading.demand as { scope: ResultScope; openDeals: number; totalDeals: number }; + const supply = reading.supply as { scope: ResultScope; openDeals: number; totalDeals: number }; + + assert.equal(demand.openDeals, 4); + assert.equal(demand.totalDeals, 13); + assert.equal(demand.scope.total, 13); + assert.equal(supply.openDeals, 2); + assert.equal(supply.totalDeals, 8); + assert.equal(supply.scope.total, 8); + assert.match(String(reading.headline), /4 of 13 demand deal\(s\) on the book are open/); + assert.match(String(reading.headline), /2 of 8 supply deal\(s\) on the book are open/); +}); + +test('the workspace summary states the threshold behind its worst-idle list', async () => { + const reading = await read('/'); + const worst = reading.worstIdle as { scope: ResultScope; blocks: unknown[] }; + + // Four of the five blocks have some idle; three are listed. Both figures are + // present, so "three blocks are idle" cannot be read off the list length. + assert.equal(worst.blocks.length, 3); + assert.equal(worst.scope.matched, 4); + assert.equal(worst.scope.total, LIVE_COMMITMENTS); + assert.equal(worst.scope.listed, 3); + // NOT 0.25. This list and pig_get_idle_capacity answer different questions + // and return different counts; each says which threshold it applied. + assert.equal(worst.scope.filters.idleThresholdPct, 0); + assert.match(String(reading.headline), /the worst idle of 4 with any idle hours/); +}); + +test('the workspace summary counts open deals against every deal on the book', async () => { + const reading = await read('/'); + assert.equal(reading.openDemandDeals, 4); + assert.equal(reading.totalDemandDeals, 13); + assert.equal(reading.openSupplyDeals, 2); + assert.equal(reading.totalSupplyDeals, 8); + assert.equal((reading.openDemandDealsScope as ResultScope).total, 13); + assert.equal((reading.openSupplyDealsScope as ResultScope).total, 8); +}); + +// --------------------------------------------------------------------------- +// The sweep +// --------------------------------------------------------------------------- + +test('every page tool result carries at least one scope, and every scope is complete', async () => { + for (const route of ['/margin', '/capacity', '/demand', '/'] as const) { + const reading = await read(route); + const found = scopes(reading); + assert.ok(found.length > 0, `${route} returned a result with no scope at all`); + for (const scope of found) { + assert.equal(typeof scope.summary, 'string', `${route}: scope has no summary`); + assert.ok(scope.summary.length > 0, `${route}: empty scope summary`); + assert.equal(typeof scope.matched, 'number', `${route}: scope has no matched`); + assert.equal(typeof scope.total, 'number', `${route}: scope has no total`); + assert.ok(scope.totalLabel.length > 0, `${route}: scope has no totalLabel`); + assert.equal(typeof scope.listed, 'number', `${route}: scope has no listed`); + assert.equal(typeof scope.truncated, 'boolean', `${route}: scope has no truncated`); + // The denominator has to reach the sentence, because the sentence is what + // gets quoted. A scope whose summary omits its own total is the defect. + assert.match( + scope.summary, + new RegExp(`\\b${scope.total}\\b`), + `${route}: a scope summary omits the total it was drawn from`, + ); + assert.ok(scope.matched <= scope.total, `${route}: matched exceeds its own denominator`); + assert.ok(scope.listed <= scope.matched, `${route}: more rows listed than matched`); + } + } +}); + +test('no page headline reports a filtered count without the total beside it', async () => { + for (const route of ['/margin', '/capacity', '/demand', '/'] as const) { + const reading = await read(route); + const headline = String(reading.headline); + for (const scope of scopes(reading)) { + if (scope.matched === scope.total) continue; + assert.match( + headline, + new RegExp(`\\b${scope.total}\\b`), + `${route}: the headline quotes a filtered figure with no denominator`, + ); + } + } +}); + +// --------------------------------------------------------------------------- +// The record read +// --------------------------------------------------------------------------- + +test('a record read says whose figures these are, so they are not read as the book', async () => { + const [record] = createInteractivePigTools(stub, { type: 'account', id: 'acct' }); + assert.ok(record); + const reading = (await record.execute({})) as Reading; + const scope = reading.scope; + assert.ok(scope); + + // Nothing was filtered out — this is an enumeration of one row's relations — + // so the figures are exact. What the sentence must carry is the boundary: + // four deals belong to this account, not to the book. + assert.equal(scope.matched, scope.total); + assert.match(scope.summary, /DEMO — Halcyon Research/); + assert.match(scope.summary, /never book-wide totals/); + assert.match(scope.summary, /4 demand deal\(s\)/); +});