pig/apps/piggy/test/result-scope.test.ts

/**
 * The scope contract, pinned.
 *
 * This suite exists because of one production answer. Asked "How many capacity
 * commitments are on the book?" on /capacity, Piggy called
 * `pig_get_idle_capacity` — the only tool that page offers — and said "3". The
 * book held 5. The tool filters to blocks at least 25% unsold, so 3 was the
 * size of a filter, and the payload gave the model nothing else to read: the
 * length of the list it had been handed was the only count in front of it.
 *
 * The system prompt already forbade that, naming this exact tool. So the guard
 * cannot be a prompt and cannot be a convention; it has to be a test that fails
 * when a result stops carrying its own denominator. Three things are pinned
 * here and nothing else:
 *
 *   1. every result carrying a count or a collection carries a `scope`;
 *   2. a filtered count is never the only count in its own result;
 *   3. the threshold that produced a filtered count is named in the payload,
 *      because three surfaces of this product have quoted three different idle
 *      figures and the only way to reconcile them is to know which is which.
 *
 * The page tools are executed against a stub handle rather than Postgres. The
 * unit suite runs in CI BEFORE the migration step, so a query here would meet a
 * database with no tables; the stub answers the four reads these tools make and
 * nothing else, which is enough because what is under test is the shaping, not
 * the SQL. `e2e/page-tools.test.ts` covers the SQL against a real book.
 */
import assert from 'node:assert/strict';
import test from 'node:test';
import {
  accounts,
  allocations,
  capacityCommitments,
  contacts,
  contracts,
  demandDeals,
  supplyDeals,
  type Database,
} from '@pig/db';
import { createInteractivePigTools } from '../src/chat-tools';
import { createPagePigTools, resultScope, type ResultScope } from '../src/page-tools';

// ---------------------------------------------------------------------------
// The stub handle
// ---------------------------------------------------------------------------

interface StubBook {
  /** Live commitments. The stub does not evaluate where clauses. */
  commitments: readonly Record<string, unknown>[];
  allocations: readonly Record<string, unknown>[];
  /** The OPEN deals, which is what the row reads in these tools select. */
  demandDeals: readonly Record<string, unknown>[];
  supplyDeals: readonly Record<string, unknown>[];
  /**
   * What the `count()` reads select: every deal on the book, every contact row,
   * and the accounts the archive filter removes.
   */
  counts: {
    demandDeals: number;
    supplyDeals: number;
    contacts: number;
    archivedAccounts: number;
  };
  accounts?: readonly Record<string, unknown>[];
  contacts?: readonly Record<string, unknown>[];
  contracts?: readonly Record<string, unknown>[];
  /**
   * The one grouped count these tools make: accounts per side, archived
   * excluded. Fixtured rather than derived from `accounts` above, because the
   * stub evaluates no where clause and so cannot tell an archived row from a
   * live one — deriving it would quietly test the fixture against itself.
   */
  accountsBySide?: readonly { side: string; value: number }[];
}

/**
 * A thenable that answers one read.
 *
 * Drizzle's builder is a promise you can keep calling methods on, so the stub
 * is the same: every chaining method returns itself and `then` resolves the
 * rows. The where clauses are ignored deliberately — a stub that reimplemented
 * them would be testing itself.
 */
function stubQuery(rows: readonly unknown[]): Record<string, unknown> {
  const builder: Record<string, unknown> = {};
  for (const method of [
    'where',
    'limit',
    'orderBy',
    'groupBy',
    'leftJoin',
    'innerJoin',
    'innerJoinLateral',
  ]) {
    builder[method] = () => builder;
  }
  builder.then = (resolve: (value: readonly unknown[]) => unknown) => resolve(rows);
  return builder;
}

function stubDatabase(book: StubBook): Database {
  const rowsFor = (table: unknown): readonly unknown[] => {
    if (table === capacityCommitments) return book.commitments;
    if (table === allocations) return book.allocations;
    if (table === demandDeals) return book.demandDeals;
    if (table === supplyDeals) return book.supplyDeals;
    if (table === accounts) return book.accounts ?? [];
    if (table === contacts) return book.contacts ?? [];
    if (table === contracts) return book.contracts ?? [];
    throw new Error('the stub was asked for a table this suite does not fixture');
  };
  const countFor = (table: unknown): number => {
    if (table === demandDeals) return book.counts.demandDeals;
    if (table === supplyDeals) return book.counts.supplyDeals;
    // The only ungrouped count taken against accounts is the archived one; the
    // live figure is summed from the grouped read below, so that the total and
    // its own breakdown cannot disagree.
    if (table === accounts) return book.counts.archivedAccounts;
    if (table === contacts) return book.counts.contacts;
    return rowsFor(table).length;
  };
  const select = (projection?: Record<string, unknown>) => ({
    from: (table: unknown) => {
      // `count()` always lands in a key called `value`. Alone it is a
      // denominator; beside another column it is a grouped count, and accounts
      // per side is the only one these tools take.
      const counting = projection !== undefined && Object.hasOwn(projection, 'value');
      if (counting && Object.keys(projection).length > 1) {
        if (table !== accounts) throw new Error('the stub groups counts for accounts only');
        return stubQuery(book.accountsBySide ?? []);
      }
      return stubQuery(counting ? [{ value: countFor(table) }] : rowsFor(table));
    },
  });
  return { select } as unknown as Database;
}

const DAY = 86_400_000;
const now = Date.now();

/** One live block: `sold` of `hours` bought at `costCents` per GPU-hour. */
function block(name: string, hours: number, sold: number, costCents = 100) {
  return {
    id: name,
    name,
    gpuType: 'H100_80GB',
    gpuCount: 8,
    startsAt: new Date(now - 10 * DAY),
    endsAt: new Date(now + 100 * DAY),
    // numeric columns arrive from Postgres as strings, and so must these.
    totalGpuHours: `${hours}.00`,
    costPerGpuHourCents: costCents,
    sold,
  };
}

function allocation(commitmentId: string, gpuHours: number) {
  return {
    capacityCommitmentId: commitmentId,
    status: 'committed',
    gpuHours: `${gpuHours}.00`,
    pricePerGpuHourCents: 120,
    holdExpiresAt: null,
  };
}

/**
 * Five live commitments, three of them at least 25% unsold.
 *
 * The production book was five and the tool returned three. Reproducing that
 * ratio exactly is the point: a fixture where the filter happens to keep
 * everything cannot fail the way production did.
 */
const BOOK = [
  block('idle-90', 1000, 100),
  block('idle-50', 1000, 500),
  block('idle-30', 1000, 700),
  block('idle-10', 1000, 900),
  block('idle-0', 1000, 1000),
];

const LIVE_COMMITMENTS = BOOK.length;
const IDLE_BLOCKS = 3;

/**
 * The account book, sized as production was when it was measured.
 *
 * Production held 17 accounts and 7 demand deals, and Piggy answered "The book
 * contains 7 demand deals (accounts) in total" to a question about accounts. So
 * the fixture keeps the two apart by more than an accident of arithmetic: 17 is
 * not the size of any deal figure, any commitment figure or any list in this
 * suite, and a payload that reports it can only have got it from the account
 * count. `both` is present because the sides must partition the book — 9 + 7 + 1
 * is 17, and an account that trades on each side is counted once.
 */
const ACCOUNTS_BY_SIDE = [
  { side: 'supply', value: 9 },
  { side: 'demand', value: 7 },
  { side: 'both', value: 1 },
];
const ACCOUNTS_ON_BOOK = 17;
/** Archived, so on no screen and in no total. The gap is still counted. */
const ARCHIVED_ACCOUNTS = 2;
const CONTACTS = 42;

const stub = stubDatabase({
  commitments: BOOK.map(({ sold: _sold, ...row }) => row),
  allocations: BOOK.filter((row) => row.sold > 0).map((row) => allocation(row.id, row.sold)),
  demandDeals: Array.from({ length: 4 }, (_, i) => ({
    id: `demand-${i}`,
    name: `Demand ${i}`,
    stage: 'proposal',
    acvCents: 1_000_000,
    tcvCents: 2_500_000,
    expectedCloseDate: null,
  })),
  supplyDeals: Array.from({ length: 2 }, (_, i) => ({
    id: `supply-${i}`,
    name: `Supply ${i}`,
    stage: 'sourced',
    gpuType: 'H200',
    gpuCount: 64,
    targetCostPerGpuHourCents: 189,
  })),
  counts: {
    demandDeals: 13,
    supplyDeals: 8,
    contacts: CONTACTS,
    archivedAccounts: ARCHIVED_ACCOUNTS,
  },
  accounts: [{ id: 'acct', name: 'DEMO — Halcyon Research' }],
  contacts: [{ id: 'contact-1', accountId: 'acct', fullName: 'A Person' }],
  contracts: [{ id: 'contract-1', accountId: 'acct', title: 'DEMO — MSA' }],
  accountsBySide: ACCOUNTS_BY_SIDE,
});

type Reading = Record<string, unknown> & { headline?: string; scope?: ResultScope };

async function read(
  route: '/margin' | '/capacity' | '/demand' | '/' | '/accounts' | '/team',
): Promise<Reading> {
  const [tool] = createPagePigTools(stub, route);
  assert.ok(tool, `no tool for ${route}`);
  return (await tool.execute({})) as Reading;
}

/** Every `scope` object anywhere in a result, however deeply it is nested. */
function scopes(value: unknown, found: ResultScope[] = []): ResultScope[] {
  if (Array.isArray(value)) {
    for (const entry of value) scopes(entry, found);
    return found;
  }
  if (value === null || typeof value !== 'object') return found;
  for (const [key, entry] of Object.entries(value)) {
    if (key === 'scope' || key.endsWith('Scope')) found.push(entry as ResultScope);
    else scopes(entry, found);
  }
  return found;
}

// ---------------------------------------------------------------------------
// The shape itself
// ---------------------------------------------------------------------------

test('an unfiltered scope says so, rather than hedging a figure that is exact', () => {
  const scope = resultScope({
    covers: 'are live',
    matched: 5,
    total: 5,
    totalLabel: 'live capacity commitment(s) on the book',
    listed: 5,
  });
  assert.equal(scope.summary, 'All 5 live capacity commitment(s) on the book; 5 listed here.');
  assert.equal(scope.matched, scope.total);
});

test('a filtered scope states both figures and names the filtered one as filtered', () => {
  const scope = resultScope({
    covers: 'are at least 25% unsold',
    matched: 3,
    total: 5,
    totalLabel: 'live capacity commitment(s) on the book',
    listed: 3,
    filters: { idleThresholdPct: 0.25 },
  });
  // The sentence a small model quotes has to carry the denominator, because a
  // field it must reason over is a field it will skip.
  assert.match(scope.summary, /3 of 5 live capacity commitment\(s\) on the book/);
  assert.match(scope.summary, /the total is 5/);
  assert.equal(scope.filters.idleThresholdPct, 0.25);
});

test('a truncated read hedges the matched count as well as the total', () => {
  const scope = resultScope({
    covers: 'are open',
    matched: 500,
    total: 500,
    totalLabel: 'demand deal(s) on the book',
    listed: 8,
    filters: { stages: 'open only' },
    truncated: true,
  });
  assert.match(scope.summary, /at least 500 of at least 500/);
  // Hedging only the total would present a capped match count as exact.
  assert.match(scope.summary, /lower bound/);
});

// ---------------------------------------------------------------------------
// The measured defect
// ---------------------------------------------------------------------------

test('the idle tool reports the size of the book beside the size of its filter', async () => {
  const reading = await read('/capacity');
  const scope = reading.scope;
  assert.ok(scope);

  // Three blocks matched out of five on the book: the production numbers.
  assert.equal(scope.matched, IDLE_BLOCKS);
  assert.equal(scope.total, LIVE_COMMITMENTS);
  assert.equal(reading.idleBlocks, IDLE_BLOCKS);
  assert.equal(reading.liveCommitments, LIVE_COMMITMENTS);

  // The headline is what a small model quotes, so the denominator has to be in
  // it. "3" alone was true of the filter and false of the book.
  assert.match(String(reading.headline), /3 of 5 live capacity commitment\(s\) on the book/);
  assert.match(String(reading.headline), /the book holds 5 live commitment\(s\) in total/);
  assert.match(scope.summary, /the total is 5/);
});

test('the idle tool names the threshold that produced its count', async () => {
  const reading = await read('/capacity');
  assert.equal(reading.scope?.filters.idleThresholdPct, 0.25);
  assert.equal(reading.scope?.filters.withinDays, 30);
  assert.equal(reading.thresholdPct, 0.25);
  // Three surfaces of this product have quoted three different idle counts for
  // one book. A result that does not say which threshold it used cannot be
  // reconciled with the screen beside it.
  assert.match(String(reading.headline), /at least 25% unsold/);
});

test('the filtered count is never the only count in the idle result', async () => {
  const reading = await read('/capacity');
  const listed = reading.blocks;
  assert.ok(Array.isArray(listed));
  // Everything that counts blocks: the matched figure, the listed rows, and the
  // denominator. The denominator must be present and must differ from them.
  const counts = [reading.idleBlocks, listed.length, reading.liveCommitments];
  assert.equal(counts.includes(LIVE_COMMITMENTS), true);
  assert.notEqual(reading.idleBlocks, reading.liveCommitments);
});

// ---------------------------------------------------------------------------
// The same trap in every other tool
// ---------------------------------------------------------------------------

test('the margin summary describes itself as the whole book, not a slice', async () => {
  const reading = await read('/margin');
  const scope = reading.scope;
  assert.ok(scope);
  assert.equal(scope.matched, LIVE_COMMITMENTS);
  assert.equal(scope.total, LIVE_COMMITMENTS);
  assert.equal(reading.liveCommitments, LIVE_COMMITMENTS);
  assert.match(String(reading.headline), /all 5 live capacity commitment\(s\) on the book/);
  // `largestBlocks` is still a slice, and `listed` is what says so.
  assert.equal(scope.listed, LIVE_COMMITMENTS);
});

test('both pipelines carry the number of deals they were drawn from', async () => {
  const reading = await read('/demand');
  const demand = reading.demand as { scope: ResultScope; openDeals: number; totalDeals: number };
  const supply = reading.supply as { scope: ResultScope; openDeals: number; totalDeals: number };

  assert.equal(demand.openDeals, 4);
  assert.equal(demand.totalDeals, 13);
  assert.equal(demand.scope.total, 13);
  assert.equal(supply.openDeals, 2);
  assert.equal(supply.totalDeals, 8);
  assert.equal(supply.scope.total, 8);
  assert.match(String(reading.headline), /4 of 13 demand deal\(s\) on the book are open/);
  assert.match(String(reading.headline), /2 of 8 supply deal\(s\) on the book are open/);
});

test('the workspace summary states the threshold behind its worst-idle list', async () => {
  const reading = await read('/');
  const worst = reading.worstIdle as { scope: ResultScope; blocks: unknown[] };

  // Four of the five blocks have some idle; three are listed. Both figures are
  // present, so "three blocks are idle" cannot be read off the list length.
  assert.equal(worst.blocks.length, 3);
  assert.equal(worst.scope.matched, 4);
  assert.equal(worst.scope.total, LIVE_COMMITMENTS);
  assert.equal(worst.scope.listed, 3);
  // NOT 0.25. This list and pig_get_idle_capacity answer different questions
  // and return different counts; each says which threshold it applied.
  assert.equal(worst.scope.filters.idleThresholdPct, 0);
  assert.match(String(reading.headline), /the worst idle of 4 with any idle hours/);
});

test('the workspace summary counts open deals against every deal on the book', async () => {
  const reading = await read('/');
  assert.equal(reading.openDemandDeals, 4);
  assert.equal(reading.totalDemandDeals, 13);
  assert.equal(reading.openSupplyDeals, 2);
  assert.equal(reading.totalSupplyDeals, 8);
  assert.equal((reading.openDemandDealsScope as ResultScope).total, 13);
  assert.equal((reading.openSupplyDealsScope as ResultScope).total, 8);
});

// ---------------------------------------------------------------------------
// The missing denominator
// ---------------------------------------------------------------------------

/** Both keys the workspace summary carries its party counts under. */
interface Parties {
  accounts: {
    scope: ResultScope;
    onBook: number;
    archived: number;
    bySide: Record<string, number>;
    bySideNote: string;
  };
  contacts: { scope: ResultScope; total: number };
}

async function parties(route: '/' | '/accounts' | '/team'): Promise<Parties> {
  return (await read(route)) as unknown as Parties;
}

test('the workspace summary counts the accounts and contacts on the book', async () => {
  const { accounts: book, contacts: people } = await parties('/');

  assert.equal(book.onBook, ACCOUNTS_ON_BOOK);
  assert.equal(people.total, CONTACTS);
  // Nothing was filtered out of either, so `matched` IS the total: these are
  // answers to "how many are there", not counts that need a denominator.
  assert.equal(book.scope.matched, ACCOUNTS_ON_BOOK);
  assert.equal(book.scope.total, ACCOUNTS_ON_BOOK);
  assert.equal(people.scope.total, CONTACTS);
  // The label is what the grounding rule tells the model to read the figure
  // against, so it has to name the noun the question would use.
  assert.match(book.scope.totalLabel, /account\(s\)/);
  assert.match(people.scope.totalLabel, /contact\(s\)/);
  assert.match(book.scope.summary, /All 17 account\(s\) on the book/);
  assert.match(people.scope.summary, /All 42 contact\(s\) in the CRM/);
});

test('the headline states the account count, because the headline is what gets quoted', async () => {
  const reading = await read('/');
  const headline = String(reading.headline);

  // The production answer was assembled from the first countable thing in this
  // sentence. There is now an account figure in it, and it is first.
  assert.match(headline, /^17 account\(s\) on the book/);
  assert.match(headline, /42 contact\(s\) in the CRM/);
  // A count of deals is not a count of accounts, and no deal figure in this
  // fixture can be mistaken for one.
  for (const dealFigure of [13, 8, 4, 2]) {
    assert.notEqual(ACCOUNTS_ON_BOOK, dealFigure);
  }
});

test('the sides partition the account book rather than overlapping it', async () => {
  const { accounts: book } = await parties('/');

  // Every side present, at zero if need be: an absent key reads as "not known"
  // to a model quoting the payload.
  assert.deepEqual(book.bySide, { supply: 9, demand: 7, both: 1 });
  const summed = Object.values(book.bySide).reduce((sum, value) => sum + value, 0);
  assert.equal(summed, ACCOUNTS_ON_BOOK);
  // A breakdown that disagrees with the /accounts side tabs, quoted beside that
  // screen, is the next version of this bug. The note is what reconciles them:
  // the tabs match `side = X or both`, so they overlap and do not sum.
  assert.match(book.bySideNote, /counted once, under both/);
  assert.match(book.bySideNote, /do not sum/);
});

test('archived accounts are off the total and still counted', async () => {
  const reading = await read('/');
  const { accounts: book } = await parties('/');

  // The /accounts list excludes them, so the total that answers "how many
  // accounts are on the book" must exclude them too — Piggy disagreeing with
  // the list on screen is worse than Piggy knowing less than it does.
  assert.equal(book.archived, ARCHIVED_ACCOUNTS);
  assert.equal(book.onBook, ACCOUNTS_ON_BOOK);
  assert.notEqual(book.onBook, ACCOUNTS_ON_BOOK + ARCHIVED_ACCOUNTS);
  // Excluded, but not invisible: a figure that differs from a raw table count
  // has to be reconcilable from the payload alone.
  assert.match(String(reading.headline), /a further 2 account\(s\) archived and off the book/);
});

test('/accounts is given a tool that can answer how many accounts there are', async () => {
  // The measured defect, at the route it was measured on. Asked "How many
  // accounts are on the book in total?" here, Piggy answered "The book contains
  // 7 demand deals (accounts) in total" — a real figure, correctly scoped as
  // deals by the payload, relabelled as accounts in the prose, because no
  // account figure existed anywhere in the result it was handed.
  const { accounts: book } = await parties('/accounts');
  assert.equal(book.onBook, ACCOUNTS_ON_BOOK);
  assert.match(book.scope.summary, /account\(s\) on the book/);
});

test('a page with no data tool still gets the book denominators, never nothing', async () => {
  // /team has no tool of its own and falls through to the summary. It must not
  // arrive with a payload that is silent about every noun: the guide tells the
  // model it can see no users, and the counts it CAN see are all labelled.
  const { accounts: book, contacts: people } = await parties('/team');
  assert.equal(book.onBook, ACCOUNTS_ON_BOOK);
  assert.equal(people.total, CONTACTS);
});

// ---------------------------------------------------------------------------
// The sweep
// ---------------------------------------------------------------------------

test('every page tool result carries at least one scope, and every scope is complete', async () => {
  for (const route of ['/margin', '/capacity', '/demand', '/', '/accounts'] as const) {
    const reading = await read(route);
    const found = scopes(reading);
    assert.ok(found.length > 0, `${route} returned a result with no scope at all`);
    for (const scope of found) {
      assert.equal(typeof scope.summary, 'string', `${route}: scope has no summary`);
      assert.ok(scope.summary.length > 0, `${route}: empty scope summary`);
      assert.equal(typeof scope.matched, 'number', `${route}: scope has no matched`);
      assert.equal(typeof scope.total, 'number', `${route}: scope has no total`);
      assert.ok(scope.totalLabel.length > 0, `${route}: scope has no totalLabel`);
      assert.equal(typeof scope.listed, 'number', `${route}: scope has no listed`);
      assert.equal(typeof scope.truncated, 'boolean', `${route}: scope has no truncated`);
      // The denominator has to reach the sentence, because the sentence is what
      // gets quoted. A scope whose summary omits its own total is the defect.
      assert.match(
        scope.summary,
        new RegExp(`\\b${scope.total}\\b`),
        `${route}: a scope summary omits the total it was drawn from`,
      );
      assert.ok(scope.matched <= scope.total, `${route}: matched exceeds its own denominator`);
      assert.ok(scope.listed <= scope.matched, `${route}: more rows listed than matched`);
    }
  }
});

test('no page headline reports a filtered count without the total beside it', async () => {
  for (const route of ['/margin', '/capacity', '/demand', '/', '/accounts'] as const) {
    const reading = await read(route);
    const headline = String(reading.headline);
    for (const scope of scopes(reading)) {
      if (scope.matched === scope.total) continue;
      assert.match(
        headline,
        new RegExp(`\\b${scope.total}\\b`),
        `${route}: the headline quotes a filtered figure with no denominator`,
      );
    }
  }
});

// ---------------------------------------------------------------------------
// The record read
// ---------------------------------------------------------------------------

test('a record read says whose figures these are, so they are not read as the book', async () => {
  const [record] = createInteractivePigTools(stub, { type: 'account', id: 'acct' });
  assert.ok(record);
  const reading = (await record.execute({})) as Reading;
  const scope = reading.scope;
  assert.ok(scope);

  // Nothing was filtered out — this is an enumeration of one row's relations —
  // so the figures are exact. What the sentence must carry is the boundary:
  // four deals belong to this account, not to the book.
  assert.equal(scope.matched, scope.total);
  assert.match(scope.summary, /DEMO — Halcyon Research/);
  assert.match(scope.summary, /never book-wide totals/);
  assert.match(scope.summary, /4 demand deal\(s\)/);
});