Rebuild Piggy's interface, and give the demo book a business to describe
Piggy answered in raw markdown, threw away every tool result it streamed, and fought the reader's scroll on every token. The three surfaces that made it worth having — what it read, how it reasoned, what it cost — were all on the wire and none of them reached the screen. The transcript is now composed of five parts under components/piggy: answers render through streamdown, the container sticks to the bottom without pinning the reader there, tool steps say what they read and link to the record, and each turn carries its model and token count. Three lifecycle bugs went with them: Stop left a permanent spinner, a truncated stream was indistinguishable from thinking, and a failed send destroyed the message it failed to send. Underneath, the inference path grew timeouts, jittered retries on 429 and 5xx, tolerance of the malformed frames a 30B model emits, and an agent_runs row per turn so chat spend is observable. The system prompt now states that a field ending in Cents is cents — without it nemotron renders costPerGpuHourCents: 189 as "$189 per GPU-hour", which is a 100x error on the most scrutinised number in the room. The demo book was arithmetically incoherent: every deal's value contradicted its own allocation revenue by up to 3.6x, nothing had ever closed, no customer had any paper, and the marketplace was empty. Deal value is now derived from the allocation, the book clears 5.3% across five blocks with one deliberately underwater, and the renewal, compliance and agent-provenance machinery finally has rows to act on. A --clear that deleted every obligation, SLA term and capacity request in the database regardless of origin is scoped to the demo's own ids. Around that: accounts have a detail page, ⌘K searches the book, Settings can mint the API keys it always claimed to, and deploy.sh actually ships the agent instead of silently skipping its compose profile. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,225 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
import test from 'node:test';
|
||||
import { z } from 'zod';
|
||||
import type { Database } from '@pig/db';
|
||||
import type { PiggyChatEvent, PiggyChatRequest } from '../src/chat';
|
||||
import { startPiggyChatServer, type PiggyChatServerOptions } from '../src/chat-server';
|
||||
|
||||
const TOKEN = 'test-internal-token-for-piggy-000000';
|
||||
|
||||
/**
|
||||
* The chat server writes exactly two statements per turn — one insert, one
|
||||
* update — so a fake that records them is enough to assert the whole ledger.
|
||||
* The tools are built against this handle too, but tool construction never
|
||||
* touches it and the provider here is a fake, so nothing else is reached.
|
||||
*/
|
||||
interface RecordedRun {
|
||||
values: Record<string, unknown>;
|
||||
closed?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
function fakeDatabase(runs: RecordedRun[]): Database {
|
||||
return {
|
||||
insert: () => ({
|
||||
values: (values: Record<string, unknown>) => ({
|
||||
returning: async () => {
|
||||
runs.push({ values });
|
||||
return [{ id: `run-${runs.length}` }];
|
||||
},
|
||||
}),
|
||||
}),
|
||||
update: () => ({
|
||||
set: (closed: Record<string, unknown>) => ({
|
||||
where: async () => {
|
||||
const run = runs.at(-1);
|
||||
if (run) run.closed = closed;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
} as unknown as Database;
|
||||
}
|
||||
|
||||
function providerYielding(events: PiggyChatEvent[], thrown?: Error): PiggyChatServerOptions['provider'] {
|
||||
return {
|
||||
model: 'nvidia/nemotron-3-nano-30b-a3b',
|
||||
run: async function* (_request: PiggyChatRequest) {
|
||||
for (const event of events) yield event;
|
||||
if (thrown) throw thrown;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function startForTest(
|
||||
t: { after: (fn: () => void) => void },
|
||||
provider: PiggyChatServerOptions['provider'],
|
||||
runs: RecordedRun[],
|
||||
): Promise<string> {
|
||||
const server = startPiggyChatServer(fakeDatabase(runs), {
|
||||
port: 0,
|
||||
internalToken: TOKEN,
|
||||
provider,
|
||||
tokenPricing: { inputCentsPerMillionTokens: 5, outputCentsPerMillionTokens: 20 },
|
||||
});
|
||||
t.after(() => server.close());
|
||||
// Port 0 is only resolved once the socket is bound.
|
||||
await new Promise((resolve) => server.once('listening', resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
return `http://127.0.0.1:${port}`;
|
||||
}
|
||||
|
||||
function chatBody(message = 'What is idle costing us?') {
|
||||
return JSON.stringify({
|
||||
principalUserId: '20000000-0000-4000-8000-000000000001',
|
||||
message,
|
||||
context: { type: 'page', route: '/capacity' },
|
||||
});
|
||||
}
|
||||
|
||||
const authorised = { authorization: `Bearer ${TOKEN}`, 'content-type': 'application/json' };
|
||||
|
||||
test('health answers without a token, and nothing else does', async (t) => {
|
||||
const runs: RecordedRun[] = [];
|
||||
const base = await startForTest(t, providerYielding([]), runs);
|
||||
|
||||
const health = await fetch(`${base}/internal/health`);
|
||||
assert.equal(health.status, 200);
|
||||
assert.deepEqual(await health.json(), {
|
||||
ok: true,
|
||||
service: 'piggy-chat',
|
||||
model: 'nvidia/nemotron-3-nano-30b-a3b',
|
||||
});
|
||||
|
||||
assert.equal((await fetch(`${base}/internal/anything`)).status, 404);
|
||||
assert.equal(
|
||||
(await fetch(`${base}/internal/chat`, { method: 'POST', body: chatBody() })).status,
|
||||
401,
|
||||
);
|
||||
});
|
||||
|
||||
test('a chat turn is recorded in agent_runs with its tokens and cost', async (t) => {
|
||||
const runs: RecordedRun[] = [];
|
||||
const base = await startForTest(
|
||||
t,
|
||||
providerYielding([
|
||||
{ type: 'meta', model: 'nvidia/nemotron-3-nano-30b-a3b' },
|
||||
{ type: 'tool_call', id: 'call_1', name: 'pig_get_idle_capacity', arguments: {} },
|
||||
{ type: 'tool_result', id: 'call_1', name: 'pig_get_idle_capacity', ok: true, result: {} },
|
||||
{ type: 'content_delta', delta: 'Idle is $12,000.' },
|
||||
{ type: 'done', inputTokens: 1_240, outputTokens: 180 },
|
||||
]),
|
||||
runs,
|
||||
);
|
||||
|
||||
const response = await fetch(`${base}/internal/chat`, {
|
||||
method: 'POST',
|
||||
headers: authorised,
|
||||
body: chatBody(),
|
||||
});
|
||||
assert.equal(response.status, 200);
|
||||
const frames = (await response.text()).trim().split('\n').map((line) => JSON.parse(line));
|
||||
assert.equal(frames.length, 5);
|
||||
|
||||
const run = runs[0];
|
||||
assert.equal(run?.values.model, 'nvidia/nemotron-3-nano-30b-a3b');
|
||||
assert.equal(run?.values.principalUserId, '20000000-0000-4000-8000-000000000001');
|
||||
assert.equal(run?.closed?.status, 'succeeded');
|
||||
assert.equal(run?.closed?.summary, 'Idle is $12,000.');
|
||||
assert.equal(run?.closed?.inputTokens, 1_240);
|
||||
assert.equal(run?.closed?.outputTokens, 180);
|
||||
// 1240 x 5 + 180 x 20 micro-cents, at $0.05/$0.20 per million tokens.
|
||||
assert.equal(run?.closed?.costMicroCents, 9_800);
|
||||
assert.ok(run?.closed?.finishedAt instanceof Date);
|
||||
});
|
||||
|
||||
test('a malformed request is the only thing called an invalid request', async (t) => {
|
||||
const runs: RecordedRun[] = [];
|
||||
const base = await startForTest(t, providerYielding([]), runs);
|
||||
|
||||
const response = await fetch(`${base}/internal/chat`, {
|
||||
method: 'POST',
|
||||
headers: authorised,
|
||||
body: JSON.stringify({ principalUserId: 'not-a-uuid', message: '' }),
|
||||
});
|
||||
|
||||
assert.equal(response.status, 400);
|
||||
assert.deepEqual(await response.json(), { error: 'Invalid Piggy chat request.' });
|
||||
// No inference was attempted, so no run should have been opened for it.
|
||||
assert.equal(runs.length, 0);
|
||||
});
|
||||
|
||||
test('a fault raised mid-stream is not blamed on the user, and closes its run', async (t) => {
|
||||
const runs: RecordedRun[] = [];
|
||||
// A ZodError, because that is the one the old code mistook for bad input:
|
||||
// a schema failure inside the turn reported "Invalid Piggy chat request" to
|
||||
// someone whose request was perfectly valid.
|
||||
const upstreamFault = new z.ZodError([]);
|
||||
const base = await startForTest(
|
||||
t,
|
||||
providerYielding(
|
||||
[
|
||||
{ type: 'meta', model: 'nvidia/nemotron-3-nano-30b-a3b' },
|
||||
{ type: 'content_delta', delta: 'Idle is ' },
|
||||
],
|
||||
upstreamFault,
|
||||
),
|
||||
runs,
|
||||
);
|
||||
|
||||
const response = await fetch(`${base}/internal/chat`, {
|
||||
method: 'POST',
|
||||
headers: authorised,
|
||||
body: chatBody(),
|
||||
});
|
||||
|
||||
// The stream had already begun, so the turn ends as an error frame on a 200.
|
||||
assert.equal(response.status, 200);
|
||||
const frames = (await response.text()).trim().split('\n').map((line) => JSON.parse(line));
|
||||
assert.deepEqual(frames.at(-1), { type: 'error', message: 'Piggy chat failed.' });
|
||||
assert.equal(runs[0]?.closed?.status, 'failed');
|
||||
assert.equal(runs[0]?.closed?.summary, 'Idle is');
|
||||
});
|
||||
|
||||
test('a reader who leaves mid-answer closes the run as abandoned, not as running', async (t) => {
|
||||
const runs: RecordedRun[] = [];
|
||||
const base = await startForTest(
|
||||
t,
|
||||
{
|
||||
model: 'nvidia/nemotron-3-nano-30b-a3b',
|
||||
// A real provider notices the abort at its next await; this one at its
|
||||
// next yield, which is the same thing at this scale.
|
||||
run: async function* (request: PiggyChatRequest) {
|
||||
for (let index = 0; index < 20; index += 1) {
|
||||
if (request.signal?.aborted) throw request.signal.reason;
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
yield { type: 'content_delta', delta: `chunk ${index} ` } as PiggyChatEvent;
|
||||
}
|
||||
},
|
||||
},
|
||||
runs,
|
||||
);
|
||||
|
||||
const abort = new AbortController();
|
||||
setTimeout(() => abort.abort(), 80);
|
||||
await assert.rejects(
|
||||
fetch(`${base}/internal/chat`, {
|
||||
method: 'POST',
|
||||
headers: authorised,
|
||||
body: chatBody(),
|
||||
signal: abort.signal,
|
||||
}).then((response) => response.text()),
|
||||
);
|
||||
|
||||
await waitFor(() => runs[0]?.closed !== undefined);
|
||||
// Without the finally this row stayed `running` for ever, and no later query
|
||||
// could tell it from a turn still in flight.
|
||||
assert.equal(runs[0]?.closed?.status, 'aborted');
|
||||
});
|
||||
|
||||
async function waitFor(condition: () => boolean): Promise<void> {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
if (condition()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
assert.fail('the run was never closed');
|
||||
}
|
||||
@@ -16,10 +16,29 @@ import { piggyChatRequestSchema } from '../src/chat-server';
|
||||
// answering over two sources where the page shows thirteen — all typecheck.
|
||||
const db = {} as Database;
|
||||
|
||||
/**
|
||||
* The lookup layer is on every message by design, so asserting it in each case
|
||||
* below would say nothing about selection. It is stripped here and covered on
|
||||
* its own in `lookup-tools.test.ts`; what these cases still pin is the FOCUSED
|
||||
* tool, which is the one that changes with where the user is standing.
|
||||
*/
|
||||
const LOOKUP_TOOLS = [
|
||||
'pig_search_records',
|
||||
'pig_get_record_by_id',
|
||||
'pig_list_renewals',
|
||||
'pig_list_inventory',
|
||||
];
|
||||
|
||||
function toolNames(context: Parameters<typeof createInteractivePigTools>[1]): string[] {
|
||||
const tools = createInteractivePigTools(db, context);
|
||||
assertPigToolBoundary(tools);
|
||||
return tools.map((tool) => tool.name);
|
||||
const names = tools.map((tool) => tool.name);
|
||||
assert.deepEqual(
|
||||
names.slice(-LOOKUP_TOOLS.length),
|
||||
LOOKUP_TOOLS,
|
||||
'the lookup layer is offered in every context, after the focused tool',
|
||||
);
|
||||
return names.slice(0, -LOOKUP_TOOLS.length);
|
||||
}
|
||||
|
||||
test('a page context selects the tool for that page and never pig_get_record', () => {
|
||||
@@ -62,6 +81,19 @@ test('no context reads the workspace, not six hundred rows of it', () => {
|
||||
assert.deepEqual(toolNames(undefined), ['pig_get_workspace_summary']);
|
||||
});
|
||||
|
||||
test('the calendar horizon accepts the null its emitted schema asks for', () => {
|
||||
const [calendar] = createInteractivePigTools(db, { type: 'page', route: '/calendar' });
|
||||
assert.ok(calendar);
|
||||
// `zodToJsonSchema(..., { target: 'openAi' })` emits an optional parameter as
|
||||
// required-and-nullable, so a model that follows the schema sends null and an
|
||||
// `.optional()` field would reject it — spending one of four turns on a tool
|
||||
// result that reads as a failure.
|
||||
assert.equal(calendar.inputSchema.safeParse({ withinDays: null }).success, true);
|
||||
assert.equal(calendar.inputSchema.safeParse({}).success, true);
|
||||
assert.equal(calendar.inputSchema.safeParse({ withinDays: 90 }).success, true);
|
||||
assert.equal(calendar.inputSchema.safeParse({ withinDays: 0 }).success, false);
|
||||
});
|
||||
|
||||
const validRequest = {
|
||||
principalUserId: '10000000-0000-4000-8000-000000000001',
|
||||
message: 'Where are we?',
|
||||
|
||||
@@ -26,6 +26,84 @@ function eventStream(events: unknown[]): Response {
|
||||
);
|
||||
}
|
||||
|
||||
/** Frames verbatim, so a test can send something no `JSON.stringify` would. */
|
||||
function rawEventStream(frames: string[]): Response {
|
||||
const encoder = new TextEncoder();
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
for (const frame of frames) controller.enqueue(encoder.encode(`${frame}\n\n`));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
{ headers: { 'content-type': 'text/event-stream' } },
|
||||
);
|
||||
}
|
||||
|
||||
/** One frame, then silence: the shape of an upstream that has stopped talking. */
|
||||
function stallingEventStream(frame: string): Response {
|
||||
const encoder = new TextEncoder();
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(`${frame}\n\n`));
|
||||
// Never closed, and no pull, so the next read waits for ever.
|
||||
},
|
||||
}),
|
||||
{ headers: { 'content-type': 'text/event-stream' } },
|
||||
);
|
||||
}
|
||||
|
||||
/** Frames spaced in time, to prove a long answer is not a stalled one. */
|
||||
function pacedEventStream(frames: string[], gapMs: number): Response {
|
||||
const encoder = new TextEncoder();
|
||||
const remaining = [...frames];
|
||||
return new Response(
|
||||
new ReadableStream({
|
||||
async pull(controller) {
|
||||
const frame = remaining.shift();
|
||||
if (frame === undefined) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, gapMs));
|
||||
controller.enqueue(encoder.encode(`${frame}\n\n`));
|
||||
},
|
||||
}),
|
||||
{ headers: { 'content-type': 'text/event-stream' } },
|
||||
);
|
||||
}
|
||||
|
||||
function jsonResponse(status: number, headers: Record<string, string> = {}): Response {
|
||||
return new Response(JSON.stringify({ error: { message: `upstream said ${status}` } }), {
|
||||
status,
|
||||
headers: { 'content-type': 'application/json', ...headers },
|
||||
});
|
||||
}
|
||||
|
||||
const finalAnswer = { choices: [{ delta: { content: 'Idle is $12,000.' }, finish_reason: 'stop' }] };
|
||||
|
||||
function contentOf(events: PiggyChatEvent[]): string {
|
||||
return events
|
||||
.filter((event): event is Extract<PiggyChatEvent, { type: 'content_delta' }> =>
|
||||
event.type === 'content_delta',
|
||||
)
|
||||
.map((event) => event.delta)
|
||||
.join('');
|
||||
}
|
||||
|
||||
function readTool(onCall?: () => void) {
|
||||
return defineTool({
|
||||
name: 'pig_get_idle_capacity',
|
||||
description: 'Read idle capacity.',
|
||||
inputSchema: z.object({}).strict(),
|
||||
execute: async () => {
|
||||
onCall?.();
|
||||
return { totalIdleCostCents: 1_200_000 };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test('interactive streaming keeps reasoning, tools and final content as separate events', async () => {
|
||||
const bodies: Record<string, unknown>[] = [];
|
||||
let call = 0;
|
||||
@@ -179,3 +257,270 @@ test('ambient coding tools are rejected before inference', async () => {
|
||||
);
|
||||
assert.equal(fetched, false);
|
||||
});
|
||||
|
||||
test('the system prompt states the units rule and the margin definitions', async () => {
|
||||
let systemPrompt = '';
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
fetchImpl: async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body)) as { messages: { role: string; content: string }[] };
|
||||
systemPrompt = body.messages.find((message) => message.role === 'system')?.content ?? '';
|
||||
return eventStream([finalAnswer]);
|
||||
},
|
||||
});
|
||||
|
||||
await collect(provider.run({ message: 'What is idle costing us?', tools: [readTool()] }));
|
||||
|
||||
// The whole point: 189 spoken as "$189 per GPU-hour" is a hundredfold error
|
||||
// on the number everyone in the room is watching.
|
||||
assert.match(systemPrompt, /ends in Cents is an integer number of US cents/i);
|
||||
assert.match(systemPrompt, /costPerGpuHourCents: 189 is \$1\.89 per GPU-hour/);
|
||||
assert.match(systemPrompt, /ends in Pct, and utilisation, is a share between 0 and 1/);
|
||||
// Margin against sold hours only would report a losing block as healthy.
|
||||
assert.match(systemPrompt, /revenue minus the FULL cost of the commitment/);
|
||||
assert.match(systemPrompt, /REMAINING unsold hours must fetch/);
|
||||
assert.match(systemPrompt, /null break-even means the block is fully allocated/);
|
||||
});
|
||||
|
||||
test('an unparseable frame is discarded rather than ending the turn', async () => {
|
||||
const warnings: string[] = [];
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
onWarning: (message) => warnings.push(message),
|
||||
fetchImpl: async () =>
|
||||
rawEventStream([
|
||||
'data: {"choices":[{"delta":{"content":"Idle is "}}]}',
|
||||
// Truncated mid-object, and then a frame that is JSON but not a chunk.
|
||||
'data: {"choices":[{"delta":',
|
||||
'data: {"choices":"not an array"}',
|
||||
'data: {"choices":[{"delta":{"content":"$12,000."},"finish_reason":"stop"}]}',
|
||||
'data: [DONE]',
|
||||
]),
|
||||
});
|
||||
|
||||
const events = await collect(provider.run({ message: 'What is idle?', tools: [readTool()] }));
|
||||
|
||||
assert.deepEqual(events.map((event) => event.type), [
|
||||
'meta',
|
||||
'content_delta',
|
||||
'content_delta',
|
||||
'done',
|
||||
]);
|
||||
assert.equal(contentOf(events), 'Idle is $12,000.');
|
||||
assert.equal(warnings.length, 2);
|
||||
});
|
||||
|
||||
test('a tool call that arrived without an id is handed back to the model, not thrown', async () => {
|
||||
const bodies: Record<string, unknown>[] = [];
|
||||
let executed = false;
|
||||
let call = 0;
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
onWarning: () => {},
|
||||
fetchImpl: async (_input, init) => {
|
||||
bodies.push(JSON.parse(String(init?.body)) as Record<string, unknown>);
|
||||
call += 1;
|
||||
return call === 1
|
||||
? eventStream([
|
||||
{
|
||||
choices: [{
|
||||
delta: {
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
function: { name: 'pig_get_idle_capacity', arguments: '{}' },
|
||||
}],
|
||||
},
|
||||
finish_reason: 'tool_calls',
|
||||
}],
|
||||
},
|
||||
])
|
||||
: eventStream([finalAnswer]);
|
||||
},
|
||||
});
|
||||
|
||||
const events = await collect(
|
||||
provider.run({ message: 'What is idle?', tools: [readTool(() => { executed = true; })] }),
|
||||
);
|
||||
|
||||
assert.deepEqual(events.map((event) => event.type), [
|
||||
'meta',
|
||||
'tool_call',
|
||||
'tool_result',
|
||||
'content_delta',
|
||||
'done',
|
||||
]);
|
||||
const result = events[2];
|
||||
assert.equal(result?.type === 'tool_result' && result.ok, false);
|
||||
assert.match(
|
||||
(result?.type === 'tool_result' && result.error) || '',
|
||||
/arrived without its id/,
|
||||
);
|
||||
// A call with no id must not run: the model never asked for a specific
|
||||
// invocation, and the reply would have nothing to attach to.
|
||||
assert.equal(executed, false);
|
||||
|
||||
// The correction only reaches the model if the tool reply matches the
|
||||
// synthesised id on the assistant message that preceded it.
|
||||
const messages = bodies[1]?.messages as {
|
||||
role: string;
|
||||
tool_calls?: { id: string }[];
|
||||
tool_call_id?: string;
|
||||
content?: string;
|
||||
}[];
|
||||
const assistant = messages.find((message) => message.role === 'assistant');
|
||||
const toolReply = messages.find((message) => message.role === 'tool');
|
||||
assert.equal(toolReply?.tool_call_id, assistant?.tool_calls?.[0]?.id);
|
||||
assert.match(toolReply?.content ?? '', /arrived without its id/);
|
||||
});
|
||||
|
||||
test('tool arguments that are not valid JSON come back as a tool result the model can fix', async () => {
|
||||
let executed = false;
|
||||
let call = 0;
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
onWarning: () => {},
|
||||
fetchImpl: async () => {
|
||||
call += 1;
|
||||
return call === 1
|
||||
? eventStream([
|
||||
{
|
||||
choices: [{
|
||||
delta: {
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
id: 'call_1',
|
||||
function: { name: 'pig_get_idle_capacity', arguments: '{"unclosed": ' },
|
||||
}],
|
||||
},
|
||||
finish_reason: 'tool_calls',
|
||||
}],
|
||||
},
|
||||
])
|
||||
: eventStream([finalAnswer]);
|
||||
},
|
||||
});
|
||||
|
||||
const events = await collect(
|
||||
provider.run({ message: 'What is idle?', tools: [readTool(() => { executed = true; })] }),
|
||||
);
|
||||
|
||||
const result = events[2];
|
||||
assert.equal(result?.type, 'tool_result');
|
||||
assert.match(
|
||||
(result?.type === 'tool_result' && result.error) || '',
|
||||
/were not valid JSON/,
|
||||
);
|
||||
assert.equal(executed, false);
|
||||
// The turn continued, which is the difference between a tool that failed
|
||||
// once and a conversation that stopped.
|
||||
assert.equal(events.at(-1)?.type, 'done');
|
||||
assert.equal(call, 2);
|
||||
});
|
||||
|
||||
test('a rate-limited turn is retried, honouring the Retry-After it was given', async () => {
|
||||
const retries: { attempt: number; delayMs: number; reason: string }[] = [];
|
||||
let calls = 0;
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
maxBackoffMs: 5,
|
||||
onRetry: (info) => retries.push(info),
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return calls === 1 ? jsonResponse(429, { 'retry-after': '0' }) : eventStream([finalAnswer]);
|
||||
},
|
||||
});
|
||||
|
||||
const events = await collect(provider.run({ message: 'What is idle?', tools: [readTool()] }));
|
||||
|
||||
assert.equal(calls, 2);
|
||||
assert.deepEqual(retries.map((retry) => retry.delayMs), [0]);
|
||||
assert.match(retries[0]?.reason ?? '', /429/);
|
||||
assert.deepEqual(events.map((event) => event.type), ['meta', 'content_delta', 'done']);
|
||||
});
|
||||
|
||||
test('a 5xx exhausts the attempt budget; a 4xx spends exactly one attempt', async () => {
|
||||
let serverErrors = 0;
|
||||
const failing = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
maxAttempts: 3,
|
||||
maxBackoffMs: 1,
|
||||
fetchImpl: async () => {
|
||||
serverErrors += 1;
|
||||
return jsonResponse(500);
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
collect(failing.run({ message: 'What is idle?', tools: [readTool()] })),
|
||||
/Piggy inference 500/,
|
||||
);
|
||||
assert.equal(serverErrors, 3);
|
||||
|
||||
let badRequests = 0;
|
||||
const rejected = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
maxAttempts: 3,
|
||||
maxBackoffMs: 1,
|
||||
fetchImpl: async () => {
|
||||
badRequests += 1;
|
||||
return jsonResponse(400);
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
collect(rejected.run({ message: 'What is idle?', tools: [readTool()] })),
|
||||
/Piggy inference 400/,
|
||||
);
|
||||
// A malformed request fails identically however often it is sent, and every
|
||||
// repeat spends credit to learn nothing.
|
||||
assert.equal(badRequests, 1);
|
||||
});
|
||||
|
||||
test('an upstream that never sends headers is abandoned on the attempt deadline', async () => {
|
||||
const provider = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
maxAttempts: 1,
|
||||
timeoutMs: 25,
|
||||
fetchImpl: (_input, init) =>
|
||||
new Promise((_resolve, reject) => {
|
||||
// Only the deadline can end this, which is also the proof that the
|
||||
// deadline reaches the request at all.
|
||||
init?.signal?.addEventListener('abort', () => reject(init.signal?.reason));
|
||||
}),
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
collect(provider.run({ message: 'What is idle?', tools: [readTool()] })),
|
||||
/did not respond within 25ms/,
|
||||
);
|
||||
});
|
||||
|
||||
test('a stream that goes quiet is abandoned, a slow one is not', async () => {
|
||||
const stalled = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
streamIdleTimeoutMs: 25,
|
||||
fetchImpl: async () => stallingEventStream('data: {"choices":[{"delta":{"content":"Idle "}}]}'),
|
||||
});
|
||||
await assert.rejects(
|
||||
collect(stalled.run({ message: 'What is idle?', tools: [readTool()] })),
|
||||
/stalled for 25ms/,
|
||||
);
|
||||
|
||||
// Six times the gap in total, and never a gap longer than the deadline: a
|
||||
// flat deadline would have killed this answer for being long.
|
||||
const slow = new PrimeOpenAIChatProvider({
|
||||
apiKey: 'test',
|
||||
streamIdleTimeoutMs: 60,
|
||||
fetchImpl: async () =>
|
||||
pacedEventStream(
|
||||
[
|
||||
...['Idle ', 'is ', '$12,000 ', 'across ', 'four ', 'blocks.'].map(
|
||||
(word) => `data: ${JSON.stringify({ choices: [{ delta: { content: word } }] })}`,
|
||||
),
|
||||
'data: [DONE]',
|
||||
],
|
||||
15,
|
||||
),
|
||||
});
|
||||
const events = await collect(slow.run({ message: 'What is idle?', tools: [readTool()] }));
|
||||
assert.equal(contentOf(events), 'Idle is $12,000 across four blocks.');
|
||||
assert.equal(events.at(-1)?.type, 'done');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { loadPiggyConfig } from '../src/config';
|
||||
|
||||
const minimum = {
|
||||
DATABASE_URL: 'postgres://pig:pig@localhost:54330/pig',
|
||||
PIGGY_INFERENCE_API_KEY: 'test-key',
|
||||
PIGGY_INTERNAL_TOKEN: 'test-internal-token-for-piggy-000000',
|
||||
};
|
||||
|
||||
test('the chat budget is separate from the worker budget, and larger', () => {
|
||||
const config = loadPiggyConfig(minimum);
|
||||
|
||||
// The worker extracts; the chat has to quote aggregates back. Sharing one
|
||||
// budget meant tuning either one moved both.
|
||||
assert.equal(config.PIGGY_MAX_TOKENS, 1_024);
|
||||
assert.equal(config.PIGGY_CHAT_MAX_TOKENS, 2_048);
|
||||
assert.equal(config.PIGGY_MAX_TURNS, 4);
|
||||
});
|
||||
|
||||
test('reasoning stays off by default', () => {
|
||||
// Reasoning tokens are billed like any other and nemotron-nano's are
|
||||
// verbose. The knob exists for debugging, not for the default deployment.
|
||||
assert.equal(loadPiggyConfig(minimum).PIGGY_REASONING_EFFORT, 'none');
|
||||
assert.equal(
|
||||
loadPiggyConfig({ ...minimum, PIGGY_REASONING_EFFORT: 'low' }).PIGGY_REASONING_EFFORT,
|
||||
'low',
|
||||
);
|
||||
assert.throws(
|
||||
() => loadPiggyConfig({ ...minimum, PIGGY_REASONING_EFFORT: 'maximum' }),
|
||||
/PIGGY_REASONING_EFFORT/,
|
||||
);
|
||||
});
|
||||
|
||||
test('the default token prices are the published price of the default model', () => {
|
||||
const config = loadPiggyConfig(minimum);
|
||||
// $0.05/$0.20 per million tokens, carried as cents per million so that
|
||||
// tokens x price is already micro-cents.
|
||||
assert.equal(config.PIGGY_PRICE_INPUT_CENTS_PER_MTOK, 5);
|
||||
assert.equal(config.PIGGY_PRICE_OUTPUT_CENTS_PER_MTOK, 20);
|
||||
assert.equal(config.PIGGY_MODEL, 'nvidia/nemotron-3-nano-30b-a3b');
|
||||
});
|
||||
@@ -14,7 +14,9 @@ describe('interactive lifecycle tool boundary', () => {
|
||||
id: '20000000-0000-4000-8000-000000000001',
|
||||
});
|
||||
|
||||
assert.deepEqual(accountTools.map((tool) => tool.name), ['pig_get_record', 'pig_get_account_lifecycle']);
|
||||
// Sliced to the focused tools: the lookup layer that follows them is on
|
||||
// every context and is covered in `lookup-tools.test.ts`.
|
||||
assert.deepEqual(accountTools.slice(0, 2).map((tool) => tool.name), ['pig_get_record', 'pig_get_account_lifecycle']);
|
||||
assert.equal(contractTools.some((tool) => tool.name === 'pig_get_account_lifecycle'), false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
/**
|
||||
* The lookup layer: search, read-by-id, renewals and provider inventory.
|
||||
*
|
||||
* Two things are covered here and nothing else. The first is the contract with
|
||||
* the model — every name inside the PIG boundary, every input schema strict and
|
||||
* bounded — because those are the failures that reach a user as a tool call
|
||||
* that never runs. The second is the shaping, which is pure by design so that
|
||||
* this suite can reach it: the unit suite runs in CI BEFORE the migration step,
|
||||
* against a database with no tables, so anything needing a row belongs in
|
||||
* `e2e/`.
|
||||
*
|
||||
* Shaping is where the expensive mistakes live. A count taken off a capped list
|
||||
* is asserted to the user as a total; an exact name match sorted below a
|
||||
* coincidental substring sends the model to the wrong account; a lapsed renewal
|
||||
* notice sorted below a distant expiry hides the only row anyone was looking
|
||||
* for. All three typecheck.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import type { Database } from '@pig/db';
|
||||
import { zodToJsonSchema } from 'zod-to-json-schema';
|
||||
import { assertPigToolBoundary } from '../src/chat';
|
||||
import {
|
||||
assembleInventoryResult,
|
||||
assembleRenewals,
|
||||
assembleSearchResult,
|
||||
createLookupPigTools,
|
||||
likeFragment,
|
||||
type InventoryOffer,
|
||||
type RenewalContract,
|
||||
type SearchRowSets,
|
||||
} from '../src/chat-tools';
|
||||
|
||||
/** Schema and naming checks run before any query, so identity is enough. */
|
||||
const db = {} as Database;
|
||||
|
||||
const tools = createLookupPigTools(db);
|
||||
|
||||
function tool(name: string) {
|
||||
const found = tools.find((candidate) => candidate.name === name);
|
||||
assert.ok(found, `${name} is registered`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function accepts(name: string, input: unknown): boolean {
|
||||
return tool(name).inputSchema.safeParse(input).success;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The boundary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('every lookup tool sits inside the PIG tool boundary', () => {
|
||||
assert.deepEqual(tools.map((entry) => entry.name), [
|
||||
'pig_search_records',
|
||||
'pig_get_record_by_id',
|
||||
'pig_list_renewals',
|
||||
'pig_list_inventory',
|
||||
]);
|
||||
// The assertion the chat provider runs on every request. A name that fails it
|
||||
// takes the whole conversation down rather than one tool.
|
||||
assert.doesNotThrow(() => assertPigToolBoundary(tools));
|
||||
for (const entry of tools) {
|
||||
assert.ok(entry.name.startsWith('pig_'), entry.name);
|
||||
// Nothing here may read as a shell, filesystem or code-execution tool: the
|
||||
// system prompt tells the model it has none, and a name that suggests
|
||||
// otherwise is an invitation to try.
|
||||
assert.doesNotMatch(entry.name, /bash|shell|filesystem|file_read|file_write|exec|eval/i);
|
||||
assert.ok(entry.description.length > 40, `${entry.name} has a usable description`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The input bounds
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('the search query is bounded at both ends because it comes from a model', () => {
|
||||
assert.equal(accepts('pig_search_records', { query: 'Halcyon' }), true);
|
||||
// Trimmed before the length check, so trailing whitespace cannot smuggle a
|
||||
// one-character query past the floor.
|
||||
assert.equal(accepts('pig_search_records', { query: ' H ' }), false);
|
||||
assert.equal(accepts('pig_search_records', { query: '' }), false);
|
||||
assert.equal(accepts('pig_search_records', { query: 'a' }), false);
|
||||
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(64) }), true);
|
||||
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(65) }), false);
|
||||
// A model that pastes an entire user turn into the query would otherwise put
|
||||
// arbitrary text into a LIKE pattern and get the whole book back.
|
||||
assert.equal(accepts('pig_search_records', { query: 'x'.repeat(4000) }), false);
|
||||
assert.equal(accepts('pig_search_records', {}), false);
|
||||
assert.equal(accepts('pig_search_records', { query: 'Halcyon', limit: 500 }), false);
|
||||
});
|
||||
|
||||
test('LIKE wildcards in a model-supplied query are escaped, not honoured', () => {
|
||||
// `%` unescaped matches every row in every searched table, and the model is
|
||||
// handed the first five of each as though they answered the question.
|
||||
assert.equal(likeFragment('%'), '%\\%%');
|
||||
assert.equal(likeFragment('_'), '%\\_%');
|
||||
assert.equal(likeFragment('a\\b'), '%a\\\\b%');
|
||||
assert.equal(likeFragment('Halcyon'), '%Halcyon%');
|
||||
});
|
||||
|
||||
test('read-by-id takes a known record type and a real uuid', () => {
|
||||
const id = '20000000-0000-4000-8000-000000000002';
|
||||
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id }), true);
|
||||
assert.equal(accepts('pig_get_record_by_id', { type: 'commitment', id }), true);
|
||||
// An id the model invented is far more likely than one it mistyped, and a
|
||||
// free-text id would reach the database as a cast error rather than a miss.
|
||||
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id: 'halcyon' }), false);
|
||||
assert.equal(accepts('pig_get_record_by_id', { type: 'invoice', id }), false);
|
||||
assert.equal(accepts('pig_get_record_by_id', { id }), false);
|
||||
assert.equal(accepts('pig_get_record_by_id', { type: 'account', id, expand: true }), false);
|
||||
});
|
||||
|
||||
test('the renewal and inventory filters reject everything they do not name', () => {
|
||||
assert.equal(accepts('pig_list_renewals', {}), true);
|
||||
assert.equal(accepts('pig_list_renewals', { side: 'demand' }), true);
|
||||
assert.equal(accepts('pig_list_renewals', { side: 'supply' }), true);
|
||||
assert.equal(accepts('pig_list_renewals', { side: 'both' }), false);
|
||||
assert.equal(accepts('pig_list_renewals', { withinDays: 30 }), false);
|
||||
|
||||
assert.equal(accepts('pig_list_inventory', {}), true);
|
||||
assert.equal(accepts('pig_list_inventory', { gpuType: 'H100' }), true);
|
||||
assert.equal(accepts('pig_list_inventory', { gpuType: 'x'.repeat(25) }), false);
|
||||
assert.equal(accepts('pig_list_inventory', { minGpuCount: 8 }), true);
|
||||
assert.equal(accepts('pig_list_inventory', { minGpuCount: 0 }), false);
|
||||
assert.equal(accepts('pig_list_inventory', { minGpuCount: 8.5 }), false);
|
||||
assert.equal(accepts('pig_list_inventory', { minGpuCount: 1_000_000 }), false);
|
||||
assert.equal(accepts('pig_list_inventory', { requiresFastInterconnect: true }), true);
|
||||
assert.equal(accepts('pig_list_inventory', { maxPriceCents: 200 }), false);
|
||||
});
|
||||
|
||||
/**
|
||||
* What the model is actually sent, rather than what the zod reads like.
|
||||
*
|
||||
* `zodToJsonSchema(..., { target: 'openAi' })` — the exact call both inference
|
||||
* paths make — emits an optional field as REQUIRED and nullable. Two failures
|
||||
* follow from that and neither is visible in TypeScript: a schema-abiding model
|
||||
* sends `null` and `.optional()` rejects it, and a `.describe()` applied after
|
||||
* the wrapper is dropped from the emitted schema, so the sentence explaining
|
||||
* the parameter never reaches the prompt.
|
||||
*/
|
||||
function emittedSchema(name: string): {
|
||||
properties?: Record<string, { description?: string }>;
|
||||
required?: string[];
|
||||
} {
|
||||
return zodToJsonSchema(tool(name).inputSchema, { $refStrategy: 'none', target: 'openAi' }) as {
|
||||
properties?: Record<string, { description?: string }>;
|
||||
required?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
test('an optional parameter accepts the null the emitted schema asks for', () => {
|
||||
assert.equal(accepts('pig_list_renewals', { side: null }), true);
|
||||
assert.equal(
|
||||
accepts('pig_list_inventory', {
|
||||
gpuType: null,
|
||||
minGpuCount: null,
|
||||
requiresFastInterconnect: null,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
// The schema tells the model these are required, so a model that obeys it
|
||||
// sends all three every time — including when it wants no filter at all.
|
||||
assert.deepEqual(emittedSchema('pig_list_inventory').required, [
|
||||
'gpuType',
|
||||
'minGpuCount',
|
||||
'requiresFastInterconnect',
|
||||
]);
|
||||
});
|
||||
|
||||
test('every parameter description survives into the emitted schema', () => {
|
||||
for (const entry of tools) {
|
||||
const properties = emittedSchema(entry.name).properties ?? {};
|
||||
for (const [parameter, shape] of Object.entries(properties)) {
|
||||
assert.ok(
|
||||
shape.description && shape.description.length > 10,
|
||||
`${entry.name}.${parameter} reaches the model with no description`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Search shaping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const emptySets: SearchRowSets = {
|
||||
accounts: [],
|
||||
demandDeals: [],
|
||||
supplyDeals: [],
|
||||
contracts: [],
|
||||
commitments: [],
|
||||
accountNames: new Map(),
|
||||
};
|
||||
|
||||
function account(name: string, id = name): SearchRowSets['accounts'][number] {
|
||||
return { id, name, side: 'demand', customerSegment: 'enterprise', country: 'US' };
|
||||
}
|
||||
|
||||
interface SearchReading {
|
||||
headline: string;
|
||||
truncated: boolean;
|
||||
counts: Record<string, number>;
|
||||
results: { type: string; id: string; name: string }[];
|
||||
}
|
||||
|
||||
test('an exact name outranks a prefix, and a prefix outranks a substring', () => {
|
||||
const reading = assembleSearchResult('meridian', {
|
||||
...emptySets,
|
||||
accounts: [
|
||||
account('Old Meridian Holdings'),
|
||||
account('Meridian Sovereign Cloud'),
|
||||
account('Meridian'),
|
||||
],
|
||||
}) as SearchReading;
|
||||
|
||||
assert.deepEqual(reading.results.map((row) => row.name), [
|
||||
'Meridian',
|
||||
'Meridian Sovereign Cloud',
|
||||
'Old Meridian Holdings',
|
||||
]);
|
||||
});
|
||||
|
||||
test('at equal match quality the account comes first, because it reaches the rest', () => {
|
||||
const reading = assembleSearchResult('halcyon', {
|
||||
...emptySets,
|
||||
accounts: [account('DEMO — Halcyon Research', 'acct')],
|
||||
contracts: [
|
||||
{
|
||||
id: 'dpa',
|
||||
title: 'DEMO — DPA — Halcyon Research',
|
||||
accountId: 'acct',
|
||||
contractType: 'dpa',
|
||||
status: 'executed',
|
||||
side: 'demand',
|
||||
expiresAt: null,
|
||||
valueCents: null,
|
||||
},
|
||||
],
|
||||
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
||||
}) as SearchReading;
|
||||
|
||||
// Alphabetically the addendum wins, and that is the wrong answer to
|
||||
// "tell me about Halcyon".
|
||||
assert.deepEqual(reading.results.map((row) => row.type), ['account', 'contract']);
|
||||
});
|
||||
|
||||
test('a search result is capped per type and overall, and says when it was cut', () => {
|
||||
const six = Array.from({ length: 6 }, (_, i) => account(`Alpha ${i}`, `a${i}`));
|
||||
const reading = assembleSearchResult('alpha', { ...emptySets, accounts: six }) as SearchReading;
|
||||
|
||||
// Six rows come back from a five-row budget precisely so the cut is visible;
|
||||
// the sixth is evidence, never a result.
|
||||
assert.equal(reading.results.length, 5);
|
||||
assert.equal(reading.counts.account, 5);
|
||||
assert.equal(reading.truncated, true);
|
||||
// The model quotes the headline, so the hedge has to live in it rather than
|
||||
// in a `truncated` flag further down the payload.
|
||||
assert.match(reading.headline, /at least 5 record\(s\) match "alpha"/);
|
||||
});
|
||||
|
||||
test('the overall cap holds even when no single type reached its own', () => {
|
||||
const three = (prefix: string) =>
|
||||
Array.from({ length: 3 }, (_, i) => `${prefix} ${i}`);
|
||||
const reading = assembleSearchResult('block', {
|
||||
accounts: three('block acct').map((name) => account(name, name)),
|
||||
demandDeals: three('block demand').map((name) => ({
|
||||
id: name,
|
||||
name,
|
||||
accountId: 'acct',
|
||||
stage: 'proposal',
|
||||
acvCents: 1_000_000,
|
||||
tcvCents: 2_500_000,
|
||||
expectedCloseDate: new Date('2026-09-01T00:00:00.000Z'),
|
||||
})),
|
||||
supplyDeals: three('block supply').map((name) => ({
|
||||
id: name,
|
||||
name,
|
||||
accountId: 'acct',
|
||||
stage: 'sourced',
|
||||
gpuType: 'H100_80GB',
|
||||
gpuCount: 64,
|
||||
targetCostPerGpuHourCents: 189,
|
||||
})),
|
||||
contracts: three('block msa').map((name) => ({
|
||||
id: name,
|
||||
title: name,
|
||||
accountId: 'acct',
|
||||
contractType: 'msa',
|
||||
status: 'executed',
|
||||
side: 'demand',
|
||||
expiresAt: new Date('2027-01-01T00:00:00.000Z'),
|
||||
valueCents: 125_722_500,
|
||||
})),
|
||||
commitments: three('block cap').map((name) => ({
|
||||
id: name,
|
||||
name,
|
||||
accountId: 'acct',
|
||||
gpuType: 'H200',
|
||||
gpuCount: 128,
|
||||
startsAt: new Date('2026-01-01T00:00:00.000Z'),
|
||||
endsAt: new Date('2027-01-01T00:00:00.000Z'),
|
||||
costPerGpuHourCents: 210,
|
||||
})),
|
||||
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
||||
}) as SearchReading;
|
||||
|
||||
// Fifteen matches across five types, twelve slots. Without the overall cap a
|
||||
// search is an unbounded read wearing a bounded one's clothes.
|
||||
assert.equal(reading.results.length, 12);
|
||||
assert.equal(reading.truncated, true);
|
||||
assert.deepEqual(reading.counts, {
|
||||
account: 3,
|
||||
demand_deal: 3,
|
||||
supply_deal: 3,
|
||||
contract: 3,
|
||||
commitment: 3,
|
||||
});
|
||||
});
|
||||
|
||||
test('every hit carries the type and id read-by-id needs, and a name to choose on', () => {
|
||||
const reading = assembleSearchResult('halcyon', {
|
||||
...emptySets,
|
||||
contracts: [
|
||||
{
|
||||
id: 'contract-1',
|
||||
title: 'DEMO — MSA — Halcyon Research',
|
||||
accountId: 'acct',
|
||||
contractType: 'msa',
|
||||
status: 'executed',
|
||||
side: 'demand',
|
||||
expiresAt: new Date('2026-10-05T00:00:00.000Z'),
|
||||
valueCents: null,
|
||||
},
|
||||
],
|
||||
accountNames: new Map([['acct', 'DEMO — Halcyon Research']]),
|
||||
}) as SearchReading & { results: Record<string, unknown>[] };
|
||||
|
||||
assert.deepEqual(reading.results[0], {
|
||||
type: 'contract',
|
||||
id: 'contract-1',
|
||||
name: 'DEMO — MSA — Halcyon Research',
|
||||
accountName: 'DEMO — Halcyon Research',
|
||||
contractType: 'msa',
|
||||
status: 'executed',
|
||||
side: 'demand',
|
||||
expiresAt: '2026-10-05T00:00:00.000Z',
|
||||
// Null money is "not stated", never zero — the units rule in the system
|
||||
// prompt turns on exactly this distinction.
|
||||
valueCents: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('a search that matches nothing says so rather than returning a bare empty list', () => {
|
||||
const reading = assembleSearchResult('nobody', emptySets) as SearchReading;
|
||||
assert.equal(reading.results.length, 0);
|
||||
assert.equal(reading.truncated, false);
|
||||
assert.match(reading.headline, /No account, deal, contract or capacity commitment/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Renewals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const NOW = new Date('2026-08-13T12:00:00.000Z');
|
||||
const DAY = 86_400_000;
|
||||
|
||||
function contract(overrides: Partial<RenewalContract> & { id: string }): RenewalContract {
|
||||
return {
|
||||
title: `Contract ${overrides.id}`,
|
||||
side: 'demand',
|
||||
type: 'msa',
|
||||
isAutoRenew: false,
|
||||
noticeDays: null,
|
||||
expiresAt: new Date(NOW.getTime() + 365 * DAY),
|
||||
valueCents: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
interface RenewalReading {
|
||||
headline: string;
|
||||
truncated: boolean;
|
||||
count: number;
|
||||
noticeWindowOpenCount: number;
|
||||
renewals: {
|
||||
id: string;
|
||||
renewalState: string;
|
||||
deadlineKind: string;
|
||||
daysUntilDeadline: number;
|
||||
deadlineAt: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
test('a lapsed notice outranks a nearer expiry, because the decision is the deadline', () => {
|
||||
const reading = assembleRenewals(
|
||||
[
|
||||
// Expires in 20 days with no notice term: the expiry is the deadline.
|
||||
{ contract: contract({ id: 'soon', expiresAt: new Date(NOW.getTime() + 20 * DAY) }), accountName: 'Northwind' },
|
||||
// Expires in 53 days, but the 60-day notice window opened a week ago.
|
||||
{
|
||||
contract: contract({
|
||||
id: 'missed',
|
||||
expiresAt: new Date(NOW.getTime() + 53 * DAY),
|
||||
isAutoRenew: true,
|
||||
noticeDays: 60,
|
||||
valueCents: 876_635_509,
|
||||
}),
|
||||
accountName: 'Halcyon',
|
||||
},
|
||||
],
|
||||
{ now: NOW, truncated: false },
|
||||
) as RenewalReading;
|
||||
|
||||
assert.deepEqual(reading.renewals.map((row) => row.id), ['missed', 'soon']);
|
||||
const missed = reading.renewals[0];
|
||||
assert.ok(missed);
|
||||
assert.equal(missed.renewalState, 'due');
|
||||
assert.equal(missed.deadlineKind, 'renewal_notice');
|
||||
// Negative days are the honest reading of a window that opened a week ago.
|
||||
assert.equal(missed.daysUntilDeadline, -7);
|
||||
assert.equal(reading.noticeWindowOpenCount, 1);
|
||||
assert.match(reading.headline, /which has already passed/);
|
||||
// Money is stated in dollars only in the headline; the row keeps raw cents.
|
||||
assert.match(reading.headline, /\$8,766,355\.09/);
|
||||
});
|
||||
|
||||
test('an open notice window on unpriced paper is not reported as worth nothing', () => {
|
||||
const reading = assembleRenewals(
|
||||
[
|
||||
{
|
||||
// A master agreement carries the notice term; the money sits on the
|
||||
// order forms beneath it. Summing nulls to zero says "$0.00".
|
||||
contract: contract({
|
||||
id: 'msa',
|
||||
expiresAt: new Date(NOW.getTime() + 53 * DAY),
|
||||
isAutoRenew: true,
|
||||
noticeDays: 60,
|
||||
valueCents: null,
|
||||
}),
|
||||
accountName: 'Halcyon',
|
||||
},
|
||||
],
|
||||
{ now: NOW, truncated: false },
|
||||
) as RenewalReading;
|
||||
|
||||
assert.equal(reading.noticeWindowOpenCount, 1);
|
||||
assert.doesNotMatch(reading.headline, /\$0\.00/);
|
||||
assert.match(reading.headline, /none of those contracts states a value of its own/);
|
||||
});
|
||||
|
||||
test('a contract that cannot auto-renew has an expiry deadline and no notice state', () => {
|
||||
const reading = assembleRenewals(
|
||||
[{ contract: contract({ id: 'plain' }), accountName: 'Verity Health AI' }],
|
||||
{ now: NOW, truncated: false },
|
||||
) as RenewalReading;
|
||||
|
||||
const [row] = reading.renewals;
|
||||
assert.ok(row);
|
||||
assert.equal(row.deadlineKind, 'expiry');
|
||||
assert.equal(row.renewalState, 'not_applicable');
|
||||
assert.equal(reading.noticeWindowOpenCount, 0);
|
||||
assert.doesNotMatch(reading.headline, /already passed/);
|
||||
});
|
||||
|
||||
test('the renewal count covers the whole set while the list is capped', () => {
|
||||
const rows = Array.from({ length: 14 }, (_, i) => ({
|
||||
contract: contract({ id: `c${i}`, expiresAt: new Date(NOW.getTime() + (i + 1) * DAY) }),
|
||||
accountName: null,
|
||||
}));
|
||||
const reading = assembleRenewals(rows, { now: NOW, truncated: true }) as RenewalReading;
|
||||
|
||||
assert.equal(reading.count, 14);
|
||||
assert.equal(reading.renewals.length, 8);
|
||||
assert.equal(reading.truncated, true);
|
||||
// A capped list quoted as a total is the defect this whole pattern exists to
|
||||
// prevent, so the hedge has to reach the headline.
|
||||
assert.match(reading.headline, /At least 14 executed contract\(s\)/);
|
||||
});
|
||||
|
||||
test('an empty book states the absence rather than implying nothing is due', () => {
|
||||
const reading = assembleRenewals([], { now: NOW, side: 'supply', truncated: false }) as RenewalReading;
|
||||
assert.equal(reading.count, 0);
|
||||
assert.match(reading.headline, /No executed contract on the supply side/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provider inventory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function offer(overrides: Partial<InventoryOffer> & { gpuType: string }): InventoryOffer {
|
||||
return {
|
||||
accountId: 'provider-1',
|
||||
providerSlug: 'runpod',
|
||||
gpuCount: 8,
|
||||
interconnectType: 'Infiniband',
|
||||
region: 'us-east',
|
||||
country: 'US',
|
||||
securityTier: 'secure_cloud',
|
||||
stockStatus: 'Available',
|
||||
isSpot: false,
|
||||
onDemandPriceCents: 200,
|
||||
priceIsVariable: false,
|
||||
observedAt: NOW,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const providerNames = new Map([['provider-1', 'RunPod']]);
|
||||
|
||||
interface InventoryReading {
|
||||
headline: string;
|
||||
truncated: boolean;
|
||||
count: number;
|
||||
listings: {
|
||||
gpuType: string;
|
||||
providerName: string | null;
|
||||
onDemandPricePerGpuHourCents: number | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
test('offers are cheapest first, and an unpriced one sorts last rather than free', () => {
|
||||
const reading = assembleInventoryResult(
|
||||
{},
|
||||
[
|
||||
offer({ gpuType: 'B200', onDemandPriceCents: 489 }),
|
||||
offer({ gpuType: 'QUOTE_ONLY', onDemandPriceCents: null }),
|
||||
offer({ gpuType: 'H100_80GB', onDemandPriceCents: 189 }),
|
||||
],
|
||||
{ truncated: false, providerNames },
|
||||
) as InventoryReading;
|
||||
|
||||
assert.deepEqual(reading.listings.map((row) => row.gpuType), [
|
||||
'H100_80GB',
|
||||
'B200',
|
||||
'QUOTE_ONLY',
|
||||
]);
|
||||
assert.equal(reading.listings[0]?.providerName, 'RunPod');
|
||||
// 189 cents is $1.89 per GPU-hour. Formatting it once in the headline is the
|
||||
// whole defence against a 30B model reporting "$189 per GPU-hour".
|
||||
assert.match(reading.headline, /cheapest on-demand is \$1\.89 per GPU-hour for H100_80GB/);
|
||||
assert.equal(reading.listings[0]?.onDemandPricePerGpuHourCents, 189);
|
||||
});
|
||||
|
||||
test('a GPU-type fragment matches the SKU, because a model asks for H100', () => {
|
||||
const reading = assembleInventoryResult(
|
||||
{ gpuType: 'h100' },
|
||||
[offer({ gpuType: 'H100_80GB' }), offer({ gpuType: 'H200' })],
|
||||
{ truncated: false, providerNames },
|
||||
) as InventoryReading;
|
||||
|
||||
assert.equal(reading.count, 1);
|
||||
assert.equal(reading.listings[0]?.gpuType, 'H100_80GB');
|
||||
});
|
||||
|
||||
test('the offer list is capped and the count is not', () => {
|
||||
const many = Array.from({ length: 20 }, (_, i) =>
|
||||
offer({ gpuType: 'H200', onDemandPriceCents: 300 - i }),
|
||||
);
|
||||
const reading = assembleInventoryResult({}, many, {
|
||||
truncated: true,
|
||||
providerNames,
|
||||
}) as InventoryReading;
|
||||
|
||||
assert.equal(reading.count, 20);
|
||||
assert.equal(reading.listings.length, 8);
|
||||
assert.equal(reading.listings[0]?.onDemandPricePerGpuHourCents, 281);
|
||||
assert.match(reading.headline, /At least 20 purchasable listing\(s\)/);
|
||||
});
|
||||
|
||||
test('no matching offer is reported as an absence, not as an empty market', () => {
|
||||
const reading = assembleInventoryResult({ gpuType: 'MI300X' }, [offer({ gpuType: 'H200' })], {
|
||||
truncated: false,
|
||||
providerNames,
|
||||
}) as InventoryReading;
|
||||
|
||||
assert.equal(reading.count, 0);
|
||||
assert.match(reading.headline, /No provider is currently listing capacity matching that request for MI300X/);
|
||||
});
|
||||
Reference in New Issue
Block a user