claude
f2ef403ee9
Asked how many capacity commitments were on the book, Piggy answered "3". Production holds 5. It had called the idle-capacity tool, which filters to blocks above an idle threshold, and read the length of that list as the size of the book. The system prompt already forbade this in terms — "never report a filtered count as a total; pig_get_idle_capacity returns the blocks with idle hours, not the book" — and the model did it anyway. That is the second time this argument has been lost in the prompt, so it is settled in the payload instead: a result that cannot describe its own scope will be misread eventually, however firmly the prompt objects. Every tool that returns a count or a collection now carries one shape: what it covers, how many matched, out of how many, under which filters, and whether the list was truncated. The denominators are read from the database rather than inferred. The pre-formatted headline states the scope too, since that is the sentence a small model quotes most readily — the idle tool now opens "3 of 5 live capacity commitments on the book", which is the sentence that makes the original mistake impossible to phrase. Two details worth keeping. Record reads enumerate rather than filter, so their scope states a boundary instead of a ratio: these are that record's own figures, never book-wide totals. And the workspace summary's idle threshold is deliberately recorded as 0, distinct from the idle tool's 0.25 — that mismatch is why three different idle figures appeared across the UI, and naming it in the data is how it stops being invisible. Verified against the live model: the failing question now answers 5, demand deals 13 and contracts 20 — each drawn from a payload whose filtered figure was smaller — while "which blocks are sitting idle" still names exactly the blocks that are. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
634 lines
23 KiB
TypeScript
634 lines
23 KiB
TypeScript
/**
|
|
* The lookup layer: search, read-by-id, renewals and provider inventory.
|
|
*
|
|
* Two things are covered here and nothing else. The first is the contract with
|
|
* the model — every name inside the PIG boundary, every input schema strict and
|
|
* bounded — because those are the failures that reach a user as a tool call
|
|
* that never runs. The second is the shaping, which is pure by design so that
|
|
* this suite can reach it: the unit suite runs in CI BEFORE the migration step,
|
|
* against a database with no tables, so anything needing a row belongs in
|
|
* `e2e/`.
|
|
*
|
|
* Shaping is where the expensive mistakes live. A count taken off a capped list
|
|
* is asserted to the user as a total; an exact name match sorted below a
|
|
* coincidental substring sends the model to the wrong account; a lapsed renewal
|
|
* notice sorted below a distant expiry hides the only row anyone was looking
|
|
* for. All three typecheck.
|
|
*/
|
|
import assert from 'node:assert/strict';
|
|
import test from 'node:test';
|
|
import type { Database } from '@pig/db';
|
|
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
import { assertPigToolBoundary } from '../src/chat';
|
|
import type { ResultScope } from '../src/page-tools';
|
|
import {
|
|
assembleInventoryResult,
|
|
assembleRenewals,
|
|
assembleSearchResult,
|
|
createLookupPigTools,
|
|
likeFragment,
|
|
type InventoryOffer,
|
|
type RenewalContract,
|
|
type SearchRowSets,
|
|
} from '../src/chat-tools';
|
|
|
|
/** Schema and naming checks run before any query, so identity is enough. */
|
|
const db = {} as Database;
|
|
|
|
const tools = createLookupPigTools(db);
|
|
|
|
function tool(name: string) {
|
|
const found = tools.find((candidate) => candidate.name === name);
|
|
assert.ok(found, `${name} is registered`);
|
|
return found;
|
|
}
|
|
|
|
function accepts(name: string, input: unknown): boolean {
|
|
return tool(name).inputSchema.safeParse(input).success;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The boundary
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('every lookup tool sits inside the PIG tool boundary', () => {
|
|
assert.deepEqual(tools.map((entry) => entry.name), [
|
|
'pig_search_records',
|
|
'pig_get_record_by_id',
|
|
'pig_list_renewals',
|
|
'pig_list_inventory',
|
|
]);
|
|
// The assertion the chat provider runs on every request. A name that fails it
|
|
// takes the whole conversation down rather than one tool.
|
|
assert.doesNotThrow(() => assertPigToolBoundary(tools));
|
|
for (const entry of tools) {
|
|
assert.ok(entry.name.startsWith('pig_'), entry.name);
|
|
// Nothing here may read as a shell, filesystem or code-execution tool: the
|
|
// system prompt tells the model it has none, and a name that suggests
|
|
// otherwise is an invitation to try.
|
|
assert.doesNotMatch(entry.name, /bash|shell|filesystem|file_read|file_write|exec|eval/i);
|
|
assert.ok(entry.description.length > 40, `${entry.name} has a usable description`);
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The input bounds
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('the search query is bounded at both ends because it comes from a model', () => {
|
|
assert.equal(accepts('pig_search_records', { query: 'Halcyon' }), true);
|
|
// Trimmed before the length check, so trailing whitespace cannot smuggle a
|
|
// one-character query past the floor.
|
|
assert.equal(accepts('pig_search_records', { query: ' H ' }), false);
|
|
assert.equal(accepts('pig_search_records', { query: '' }), false);
|
|
assert.equal(accepts('pig_search_records', { query: 'a' }), false);
|
|
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(64) }), true);
|
|
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(65) }), false);
|
|
// A model that pastes an entire user turn into the query would otherwise put
|
|
// arbitrary text into a LIKE pattern and get the whole book back.
|
|
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(4000) }), false);
|
|
assert.equal(accepts('pig_search_records', {}), false);
|
|
assert.equal(accepts('pig_search_records', { query: 'Halcyon', limit: 500 }), false);
|
|
});
|
|
|
|
test('LIKE wildcards in a model-supplied query are escaped, not honoured', () => {
|
|
// `%` unescaped matches every row in every searched table, and the model is
|
|
// handed the first five of each as though they answered the question.
|
|
assert.equal(likeFragment('%'), '%\\%%');
|
|
assert.equal(likeFragment('_'), '%\\_%');
|
|
assert.equal(likeFragment('a\\b'), '%a\\\\b%');
|
|
assert.equal(likeFragment('Halcyon'), '%Halcyon%');
|
|
});
|
|
|
|
test('read-by-id takes a known record type and a real uuid', () => {
|
|
const id = '20000000-0000-4000-8000-000000000002';
|
|
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id }), true);
|
|
assert.equal(accepts('pig_get_record_by_id', { type: 'commitment', id }), true);
|
|
// An id the model invented is far more likely than one it mistyped, and a
|
|
// free-text id would reach the database as a cast error rather than a miss.
|
|
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id: 'halcyon' }), false);
|
|
assert.equal(accepts('pig_get_record_by_id', { type: 'invoice', id }), false);
|
|
assert.equal(accepts('pig_get_record_by_id', { id }), false);
|
|
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id, expand: true }), false);
|
|
});
|
|
|
|
test('the renewal and inventory filters reject everything they do not name', () => {
|
|
assert.equal(accepts('pig_list_renewals', {}), true);
|
|
assert.equal(accepts('pig_list_renewals', { side: 'demand' }), true);
|
|
assert.equal(accepts('pig_list_renewals', { side: 'supply' }), true);
|
|
assert.equal(accepts('pig_list_renewals', { side: 'both' }), false);
|
|
assert.equal(accepts('pig_list_renewals', { withinDays: 30 }), false);
|
|
|
|
assert.equal(accepts('pig_list_inventory', {}), true);
|
|
assert.equal(accepts('pig_list_inventory', { gpuType: 'H100' }), true);
|
|
assert.equal(accepts('pig_list_inventory', { gpuType: 'x'.repeat(25) }), false);
|
|
assert.equal(accepts('pig_list_inventory', { minGpuCount: 8 }), true);
|
|
assert.equal(accepts('pig_list_inventory', { minGpuCount: 0 }), false);
|
|
assert.equal(accepts('pig_list_inventory', { minGpuCount: 8.5 }), false);
|
|
assert.equal(accepts('pig_list_inventory', { minGpuCount: 1_000_000 }), false);
|
|
assert.equal(accepts('pig_list_inventory', { requiresFastInterconnect: true }), true);
|
|
assert.equal(accepts('pig_list_inventory', { maxPriceCents: 200 }), false);
|
|
});
|
|
|
|
/**
|
|
* What the model is actually sent, rather than what the zod reads like.
|
|
*
|
|
* `zodToJsonSchema(..., { target: 'openAi' })` — the exact call both inference
|
|
* paths make — emits an optional field as REQUIRED and nullable. Two failures
|
|
* follow from that and neither is visible in TypeScript: a schema-abiding model
|
|
* sends `null` and `.optional()` rejects it, and a `.describe()` applied after
|
|
* the wrapper is dropped from the emitted schema, so the sentence explaining
|
|
* the parameter never reaches the prompt.
|
|
*/
|
|
function emittedSchema(name: string): {
|
|
properties?: Record<string, { description?: string }>;
|
|
required?: string[];
|
|
} {
|
|
return zodToJsonSchema(tool(name).inputSchema, { $refStrategy: 'none', target: 'openAi' }) as {
|
|
properties?: Record<string, { description?: string }>;
|
|
required?: string[];
|
|
};
|
|
}
|
|
|
|
test('an optional parameter accepts the null the emitted schema asks for', () => {
|
|
assert.equal(accepts('pig_list_renewals', { side: null }), true);
|
|
assert.equal(
|
|
accepts('pig_list_inventory', {
|
|
gpuType: null,
|
|
minGpuCount: null,
|
|
requiresFastInterconnect: null,
|
|
}),
|
|
true,
|
|
);
|
|
// The schema tells the model these are required, so a model that obeys it
|
|
// sends all three every time — including when it wants no filter at all.
|
|
assert.deepEqual(emittedSchema('pig_list_inventory').required, [
|
|
'gpuType',
|
|
'minGpuCount',
|
|
'requiresFastInterconnect',
|
|
]);
|
|
});
|
|
|
|
test('every parameter description survives into the emitted schema', () => {
|
|
for (const entry of tools) {
|
|
const properties = emittedSchema(entry.name).properties ?? {};
|
|
for (const [parameter, shape] of Object.entries(properties)) {
|
|
assert.ok(
|
|
shape.description && shape.description.length > 10,
|
|
`${entry.name}.${parameter} reaches the model with no description`,
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Search shaping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* The five denominators, roughly the demo book's own shape.
|
|
*
|
|
* A search reports how many rows it matched; without these it would be the only
|
|
* count in its own payload, and "3 accounts match" is one careless sentence away
|
|
* from "we have 3 accounts".
|
|
*/
|
|
const TOTALS = {
|
|
account: 23,
|
|
demand_deal: 13,
|
|
supply_deal: 8,
|
|
contract: 20,
|
|
commitment: 6,
|
|
} as const;
|
|
|
|
/** Every row in every searched table: the denominator the headline quotes. */
|
|
const SEARCHABLE = Object.values(TOTALS).reduce((sum, rows) => sum + rows, 0);
|
|
|
|
const emptySets: SearchRowSets = {
|
|
accounts: [],
|
|
demandDeals: [],
|
|
supplyDeals: [],
|
|
contracts: [],
|
|
commitments: [],
|
|
accountNames: new Map(),
|
|
totals: { ...TOTALS },
|
|
};
|
|
|
|
function account(name: string, id = name): SearchRowSets['accounts'][number] {
|
|
return { id, name, side: 'demand', customerSegment: 'enterprise', country: 'US' };
|
|
}
|
|
|
|
interface SearchReading {
|
|
headline: string;
|
|
truncated: boolean;
|
|
counts: Record<string, number>;
|
|
totals: Record<string, number>;
|
|
scope: ResultScope;
|
|
results: { type: string; id: string; name: string }[];
|
|
}
|
|
|
|
test('an exact name outranks a prefix, and a prefix outranks a substring', () => {
|
|
const reading = assembleSearchResult('meridian', {
|
|
...emptySets,
|
|
accounts: [
|
|
account('Old Meridian Holdings'),
|
|
account('Meridian Sovereign Cloud'),
|
|
account('Meridian'),
|
|
],
|
|
}) as SearchReading;
|
|
|
|
assert.deepEqual(reading.results.map((row) => row.name), [
|
|
'Meridian',
|
|
'Meridian Sovereign Cloud',
|
|
'Old Meridian Holdings',
|
|
]);
|
|
});
|
|
|
|
test('at equal match quality the account comes first, because it reaches the rest', () => {
|
|
const reading = assembleSearchResult('halcyon', {
|
|
...emptySets,
|
|
accounts: [account('DEMO — Halcyon Research', 'acct')],
|
|
contracts: [
|
|
{
|
|
id: 'dpa',
|
|
title: 'DEMO — DPA — Halcyon Research',
|
|
accountId: 'acct',
|
|
contractType: 'dpa',
|
|
status: 'executed',
|
|
side: 'demand',
|
|
expiresAt: null,
|
|
valueCents: null,
|
|
},
|
|
],
|
|
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
|
}) as SearchReading;
|
|
|
|
// Alphabetically the addendum wins, and that is the wrong answer to
|
|
// "tell me about Halcyon".
|
|
assert.deepEqual(reading.results.map((row) => row.type), ['account', 'contract']);
|
|
});
|
|
|
|
test('a search result is capped per type and overall, and says when it was cut', () => {
|
|
const six = Array.from({ length: 6 }, (_, i) => account(`Alpha ${i}`, `a${i}`));
|
|
const reading = assembleSearchResult('alpha', { ...emptySets, accounts: six }) as SearchReading;
|
|
|
|
// Six rows come back from a five-row budget precisely so the cut is visible;
|
|
// the sixth is evidence, never a result.
|
|
assert.equal(reading.results.length, 5);
|
|
assert.equal(reading.counts.account, 5);
|
|
assert.equal(reading.truncated, true);
|
|
// The model quotes the headline, so the hedge has to live in it rather than
|
|
// in a `truncated` flag further down the payload.
|
|
assert.match(reading.headline, new RegExp(`At least 5 of ${SEARCHABLE} searchable record\\(s\\)`));
|
|
// The denominator travels with the hedge: a capped match count next to the
|
|
// number of rows it was drawn from cannot be read as "we have five accounts".
|
|
assert.equal(reading.scope.matched, 5);
|
|
assert.equal(reading.scope.total, SEARCHABLE);
|
|
assert.equal(reading.totals.account, TOTALS.account);
|
|
});
|
|
|
|
test('the overall cap holds even when no single type reached its own', () => {
|
|
const three = (prefix: string) =>
|
|
Array.from({ length: 3 }, (_, i) => `${prefix} ${i}`);
|
|
const reading = assembleSearchResult('block', {
|
|
totals: { ...TOTALS },
|
|
accounts: three('block acct').map((name) => account(name, name)),
|
|
demandDeals: three('block demand').map((name) => ({
|
|
id: name,
|
|
name,
|
|
accountId: 'acct',
|
|
stage: 'proposal',
|
|
acvCents: 1_000_000,
|
|
tcvCents: 2_500_000,
|
|
expectedCloseDate: new Date('2026-09-01T00:00:00.000Z'),
|
|
})),
|
|
supplyDeals: three('block supply').map((name) => ({
|
|
id: name,
|
|
name,
|
|
accountId: 'acct',
|
|
stage: 'sourced',
|
|
gpuType: 'H100_80GB',
|
|
gpuCount: 64,
|
|
targetCostPerGpuHourCents: 189,
|
|
})),
|
|
contracts: three('block msa').map((name) => ({
|
|
id: name,
|
|
title: name,
|
|
accountId: 'acct',
|
|
contractType: 'msa',
|
|
status: 'executed',
|
|
side: 'demand',
|
|
expiresAt: new Date('2027-01-01T00:00:00.000Z'),
|
|
valueCents: 125_722_500,
|
|
})),
|
|
commitments: three('block cap').map((name) => ({
|
|
id: name,
|
|
name,
|
|
accountId: 'acct',
|
|
gpuType: 'H200',
|
|
gpuCount: 128,
|
|
startsAt: new Date('2026-01-01T00:00:00.000Z'),
|
|
endsAt: new Date('2027-01-01T00:00:00.000Z'),
|
|
costPerGpuHourCents: 210,
|
|
})),
|
|
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
|
}) as SearchReading;
|
|
|
|
// Fifteen matches across five types, twelve slots. Without the overall cap a
|
|
// search is an unbounded read wearing a bounded one's clothes.
|
|
assert.equal(reading.results.length, 12);
|
|
assert.equal(reading.truncated, true);
|
|
assert.deepEqual(reading.counts, {
|
|
account: 3,
|
|
demand_deal: 3,
|
|
supply_deal: 3,
|
|
contract: 3,
|
|
commitment: 3,
|
|
});
|
|
});
|
|
|
|
test('every hit carries the type and id read-by-id needs, and a name to choose on', () => {
|
|
const reading = assembleSearchResult('halcyon', {
|
|
...emptySets,
|
|
contracts: [
|
|
{
|
|
id: 'contract-1',
|
|
title: 'DEMO — MSA — Halcyon Research',
|
|
accountId: 'acct',
|
|
contractType: 'msa',
|
|
status: 'executed',
|
|
side: 'demand',
|
|
expiresAt: new Date('2026-10-05T00:00:00.000Z'),
|
|
valueCents: null,
|
|
},
|
|
],
|
|
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
|
}) as SearchReading & { results: Record<string, unknown>[] };
|
|
|
|
assert.deepEqual(reading.results[0], {
|
|
type: 'contract',
|
|
id: 'contract-1',
|
|
name: 'DEMO — MSA — Halcyon Research',
|
|
accountName: 'DEMO — Halcyon Research',
|
|
contractType: 'msa',
|
|
status: 'executed',
|
|
side: 'demand',
|
|
expiresAt: '2026-10-05T00:00:00.000Z',
|
|
// Null money is "not stated", never zero — the units rule in the system
|
|
// prompt turns on exactly this distinction.
|
|
valueCents: null,
|
|
});
|
|
});
|
|
|
|
test('a search that matches nothing says so rather than returning a bare empty list', () => {
|
|
const reading = assembleSearchResult('nobody', emptySets) as SearchReading;
|
|
assert.equal(reading.results.length, 0);
|
|
assert.equal(reading.truncated, false);
|
|
assert.match(reading.headline, new RegExp(`None of the ${SEARCHABLE} account\\(s\\)`));
|
|
// Even an empty search states the size of what it looked through.
|
|
assert.equal(reading.scope.matched, 0);
|
|
assert.equal(reading.scope.total, SEARCHABLE);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Renewals
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const NOW = new Date('2026-08-13T12:00:00.000Z');
|
|
const DAY = 86_400_000;
|
|
|
|
function contract(overrides: Partial<RenewalContract> & { id: string }): RenewalContract {
|
|
return {
|
|
title: `Contract ${overrides.id}`,
|
|
side: 'demand',
|
|
type: 'msa',
|
|
isAutoRenew: false,
|
|
noticeDays: null,
|
|
expiresAt: new Date(NOW.getTime() + 365 * DAY),
|
|
valueCents: null,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
/** Contracts of every status on the book — the renewal list's denominator. */
|
|
const CONTRACTS_ON_BOOK = 20;
|
|
|
|
interface RenewalReading {
|
|
headline: string;
|
|
truncated: boolean;
|
|
scope: ResultScope;
|
|
count: number;
|
|
totalContracts: number;
|
|
noticeWindowOpenCount: number;
|
|
renewals: {
|
|
id: string;
|
|
renewalState: string;
|
|
deadlineKind: string;
|
|
daysUntilDeadline: number;
|
|
deadlineAt: string;
|
|
}[];
|
|
}
|
|
|
|
test('a lapsed notice outranks a nearer expiry, because the decision is the deadline', () => {
|
|
const reading = assembleRenewals(
|
|
[
|
|
// Expires in 20 days with no notice term: the expiry is the deadline.
|
|
{ contract: contract({ id: 'soon', expiresAt: new Date(NOW.getTime() + 20 * DAY) }), accountName: 'Northwind' },
|
|
// Expires in 53 days, but the 60-day notice window opened a week ago.
|
|
{
|
|
contract: contract({
|
|
id: 'missed',
|
|
expiresAt: new Date(NOW.getTime() + 53 * DAY),
|
|
isAutoRenew: true,
|
|
noticeDays: 60,
|
|
valueCents: 876_635_509,
|
|
}),
|
|
accountName: 'Halcyon',
|
|
},
|
|
],
|
|
{ now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK },
|
|
) as RenewalReading;
|
|
|
|
assert.deepEqual(reading.renewals.map((row) => row.id), ['missed', 'soon']);
|
|
const missed = reading.renewals[0];
|
|
assert.ok(missed);
|
|
assert.equal(missed.renewalState, 'due');
|
|
assert.equal(missed.deadlineKind, 'renewal_notice');
|
|
// Negative days are the honest reading of a window that opened a week ago.
|
|
assert.equal(missed.daysUntilDeadline, -7);
|
|
assert.equal(reading.noticeWindowOpenCount, 1);
|
|
assert.match(reading.headline, /which has already passed/);
|
|
// Money is stated in dollars only in the headline; the row keeps raw cents.
|
|
assert.match(reading.headline, /\$8,766,355\.09/);
|
|
});
|
|
|
|
test('an open notice window on unpriced paper is not reported as worth nothing', () => {
|
|
const reading = assembleRenewals(
|
|
[
|
|
{
|
|
// A master agreement carries the notice term; the money sits on the
|
|
// order forms beneath it. Summing nulls to zero says "$0.00".
|
|
contract: contract({
|
|
id: 'msa',
|
|
expiresAt: new Date(NOW.getTime() + 53 * DAY),
|
|
isAutoRenew: true,
|
|
noticeDays: 60,
|
|
valueCents: null,
|
|
}),
|
|
accountName: 'Halcyon',
|
|
},
|
|
],
|
|
{ now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK },
|
|
) as RenewalReading;
|
|
|
|
assert.equal(reading.noticeWindowOpenCount, 1);
|
|
assert.doesNotMatch(reading.headline, /\$0\.00/);
|
|
assert.match(reading.headline, /none of those contracts states a value of its own/);
|
|
});
|
|
|
|
test('a contract that cannot auto-renew has an expiry deadline and no notice state', () => {
|
|
const reading = assembleRenewals(
|
|
[{ contract: contract({ id: 'plain' }), accountName: 'Verity Health AI' }],
|
|
{ now: NOW, truncated: false, totalContracts: CONTRACTS_ON_BOOK },
|
|
) as RenewalReading;
|
|
|
|
const [row] = reading.renewals;
|
|
assert.ok(row);
|
|
assert.equal(row.deadlineKind, 'expiry');
|
|
assert.equal(row.renewalState, 'not_applicable');
|
|
assert.equal(reading.noticeWindowOpenCount, 0);
|
|
assert.doesNotMatch(reading.headline, /already passed/);
|
|
});
|
|
|
|
test('the renewal count covers the whole set while the list is capped', () => {
|
|
const rows = Array.from({ length: 14 }, (_, i) => ({
|
|
contract: contract({ id: `c${i}`, expiresAt: new Date(NOW.getTime() + (i + 1) * DAY) }),
|
|
accountName: null,
|
|
}));
|
|
const reading = assembleRenewals(rows, { now: NOW, truncated: true, totalContracts: CONTRACTS_ON_BOOK }) as RenewalReading;
|
|
|
|
assert.equal(reading.count, 14);
|
|
assert.equal(reading.renewals.length, 8);
|
|
assert.equal(reading.truncated, true);
|
|
// A capped list quoted as a total is the defect this whole pattern exists to
|
|
// prevent, so the hedge has to reach the headline.
|
|
assert.match(
|
|
reading.headline,
|
|
new RegExp(`At least 14 of ${CONTRACTS_ON_BOOK} contract\\(s\\) on the book are executed`),
|
|
);
|
|
assert.equal(reading.scope.matched, 14);
|
|
assert.equal(reading.scope.total, CONTRACTS_ON_BOOK);
|
|
});
|
|
|
|
test('an empty book states the absence rather than implying nothing is due', () => {
|
|
const reading = assembleRenewals([], { now: NOW, side: 'supply', truncated: false, totalContracts: 8 }) as RenewalReading;
|
|
assert.equal(reading.count, 0);
|
|
assert.match(reading.headline, /None of the 8 supply-side contract\(s\) on the book/);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Provider inventory
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function offer(overrides: Partial<InventoryOffer> & { gpuType: string }): InventoryOffer {
|
|
return {
|
|
accountId: 'provider-1',
|
|
providerSlug: 'runpod',
|
|
gpuCount: 8,
|
|
interconnectType: 'Infiniband',
|
|
region: 'us-east',
|
|
country: 'US',
|
|
securityTier: 'secure_cloud',
|
|
stockStatus: 'Available',
|
|
isSpot: false,
|
|
onDemandPriceCents: 200,
|
|
priceIsVariable: false,
|
|
observedAt: NOW,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
const providerNames = new Map([['provider-1', 'RunPod']]);
|
|
|
|
/** Purchasable listings on the market with no filter at all: the denominator. */
|
|
const LISTINGS_ON_MARKET = 30;
|
|
|
|
interface InventoryReading {
|
|
headline: string;
|
|
truncated: boolean;
|
|
scope: ResultScope;
|
|
count: number;
|
|
totalListings: number;
|
|
listings: {
|
|
gpuType: string;
|
|
providerName: string | null;
|
|
onDemandPricePerGpuHourCents: number | null;
|
|
}[];
|
|
}
|
|
|
|
test('offers are cheapest first, and an unpriced one sorts last rather than free', () => {
|
|
const reading = assembleInventoryResult(
|
|
{},
|
|
[
|
|
offer({ gpuType: 'B200', onDemandPriceCents: 489 }),
|
|
offer({ gpuType: 'QUOTE_ONLY', onDemandPriceCents: null }),
|
|
offer({ gpuType: 'H100_80GB', onDemandPriceCents: 189 }),
|
|
],
|
|
{ truncated: false, providerNames, totalListings: LISTINGS_ON_MARKET, totalTruncated: false },
|
|
) as InventoryReading;
|
|
|
|
assert.deepEqual(reading.listings.map((row) => row.gpuType), [
|
|
'H100_80GB',
|
|
'B200',
|
|
'QUOTE_ONLY',
|
|
]);
|
|
assert.equal(reading.listings[0]?.providerName, 'RunPod');
|
|
// 189 cents is $1.89 per GPU-hour. Formatting it once in the headline is the
|
|
// whole defence against a 30B model reporting "$189 per GPU-hour".
|
|
assert.match(reading.headline, /cheapest on-demand is \$1\.89 per GPU-hour for H100_80GB/);
|
|
assert.equal(reading.listings[0]?.onDemandPricePerGpuHourCents, 189);
|
|
});
|
|
|
|
test('a GPU-type fragment matches the SKU, because a model asks for H100', () => {
|
|
const reading = assembleInventoryResult(
|
|
{ gpuType: 'h100' },
|
|
[offer({ gpuType: 'H100_80GB' }), offer({ gpuType: 'H200' })],
|
|
{ truncated: false, providerNames, totalListings: LISTINGS_ON_MARKET, totalTruncated: false },
|
|
) as InventoryReading;
|
|
|
|
assert.equal(reading.count, 1);
|
|
assert.equal(reading.listings[0]?.gpuType, 'H100_80GB');
|
|
});
|
|
|
|
test('the offer list is capped and the count is not', () => {
|
|
const many = Array.from({ length: 20 }, (_, i) =>
|
|
offer({ gpuType: 'H200', onDemandPriceCents: 300 - i }),
|
|
);
|
|
const reading = assembleInventoryResult({}, many, {
|
|
truncated: true,
|
|
providerNames,
|
|
totalListings: 20,
|
|
totalTruncated: true,
|
|
}) as InventoryReading;
|
|
|
|
assert.equal(reading.count, 20);
|
|
assert.equal(reading.listings.length, 8);
|
|
assert.equal(reading.listings[0]?.onDemandPricePerGpuHourCents, 281);
|
|
assert.match(reading.headline, /20 of at least 20 purchasable listing\(s\) on the market/);
|
|
assert.equal(reading.scope.truncated, true);
|
|
});
|
|
|
|
test('no matching offer is reported as an absence, not as an empty market', () => {
|
|
const reading = assembleInventoryResult({ gpuType: 'MI300X' }, [offer({ gpuType: 'H200' })], {
|
|
truncated: false,
|
|
providerNames,
|
|
totalListings: LISTINGS_ON_MARKET,
|
|
totalTruncated: false,
|
|
}) as InventoryReading;
|
|
|
|
assert.equal(reading.count, 0);
|
|
assert.match(
|
|
reading.headline,
|
|
new RegExp(`None of the ${LISTINGS_ON_MARKET} purchasable listing\\(s\\) on the market matches`),
|
|
);
|
|
});
|