import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; import { isPiggyModelId, piggyDefaultModelId, piggyInferenceBaseUrl, piggyModelCatalogue, } from '../src/agent/models'; const modelsJson = JSON.parse( readFileSync(fileURLToPath(new URL('../src/agent/models.json', import.meta.url)), 'utf8'), ) as { providers: Record; }; test('every id in the picker is one the provider actually registers', () => { // The whole point of a curated shortlist is that nothing in it 404s. The // catalogue and models.json are the same five models by construction, and // this is what keeps them that way when someone adds a sixth to one file. const registered = (modelsJson.providers['prime-inference']?.models ?? []).map( (model) => model.id, ); const offered = piggyModelCatalogue().map((option) => option.id); assert.deepEqual(offered, registered); assert.ok(offered.length >= 4, 'the picker should offer a real choice, not just the default'); for (const id of offered) { // Prime Inference ids are always provider-qualified. A bare model name is // the classic copy-and-paste error and it fails as a 404 at the endpoint. assert.match(id, /^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/, `${id} is not provider-qualified`); assert.ok(isPiggyModelId(id)); } }); test('the default is in the catalogue and there is exactly one of it', () => { const catalogue = piggyModelCatalogue(); const defaults = catalogue.filter((option) => option.isDefault); assert.equal(defaults.length, 1); assert.equal(defaults[0]?.id, piggyDefaultModelId()); // The default is the SUPER, not the nano, and the reason is availability // rather than quality. On 2026-08-14 `nvidia/nemotron-3-nano-30b-a3b` stopped // answering on Prime Inference — the connection was accepted and no response // headers ever arrived, three attempts at 45s each — while every other model // in this catalogue answered in under two seconds on the same key in the same // minute. The nano stays in the picker for anyone who wants it back. assert.equal(piggyDefaultModelId(), 'nvidia/nemotron-3-super-120b-a12b'); assert.equal(isPiggyModelId('nvidia/nemotron-3-super-120b-a12b'), true); assert.equal(isPiggyModelId('nvidia/nemotron-3-nano-30b-a3b'), true); assert.equal(isPiggyModelId('nvidia/nemotron-9000'), false); }); test('the picker can price and size every choice', () => { for (const option of piggyModelCatalogue()) { // Dollars per million tokens, NOT cents: the field names say so, and this // is the one money field in PIG that is not an integer of cents. A price // of 0 here would render as "free" in the picker, which no model is. assert.ok(option.costPerMTokIn > 0, `${option.id} has no input price`); assert.ok(option.costPerMTokOut > 0, `${option.id} has no output price`); assert.ok(option.costPerMTokOut >= option.costPerMTokIn, `${option.id} prices output too low`); assert.ok(option.contextWindow >= 100_000, `${option.id} is too small for a CRM transcript`); assert.ok(option.label.length > 0); assert.ok((option.hint ?? '').length > 0, `${option.id} would render as a blank picker row`); } }); /** * This used to assert that the default was the cheapest thing on offer, and it * was a good rule until the cheapest thing stopped answering. What actually * protects the choice is not the ranking but the ceiling: the panel is docked on * every page, so the default is the price of a typo, and the failure worth * catching is somebody making a frontier model the default by accident. A * deliberate move up the price list should pass; a slip to Opus should not. */ test('the default is a cheap model, even though it is no longer the cheapest', () => { const catalogue = piggyModelCatalogue(); const cheapest = [...catalogue].sort((a, b) => a.costPerMTokIn - b.costPerMTokIn)[0]; const chosen = catalogue.find((option) => option.id === piggyDefaultModelId()); assert.ok(chosen && cheapest); assert.notEqual(chosen.id, cheapest.id, 'the cheapest model answers again; revisit the default'); // Six times the price of the nano is still about $0.0017 a turn, or roughly // 117,000 turns on a $200 credit. A dollar per million input tokens is an // order of magnitude above that and two below every frontier model here. assert.ok(chosen.costPerMTokIn <= 1, `${chosen.id} is too dear to be the default`); const frontier = catalogue.filter((option) => option.costPerMTokIn >= 5); assert.ok(frontier.length >= 2, 'the picker no longer offers a frontier option to contrast with'); for (const option of frontier) { assert.notEqual(option.id, chosen.id, 'a frontier model became the default by accident'); } }); /** * The half of the reasoning trap that nobody would guess, pinned to whichever * model is the default rather than to a name. * * `@earendil-works/pi-ai@0.84.1` turns a thinking level of `off` into no * `reasoning_effort` field at all unless the model entry maps it, and the * endpoint's own default then wins — 6,195 output tokens of reasoning and an * empty answer. `agent-thinking.test.ts` pins the behaviour end to end; this * pins the datum it depends on, which is the thing a new default would silently * arrive without. */ test('the default carries a thinking map for the level Piggy is configured to run at', async () => { process.env.DATABASE_URL ??= 'postgres://pig:pig@localhost:54330/pig'; process.env.PIGGY_INTERNAL_TOKEN ??= 'test-internal-token-for-piggy-000000'; process.env.PRIME_API_KEY ??= 'test-key-not-used-offline'; const { loadPiggyConfig } = await import('../src/config'); const level = loadPiggyConfig().PIGGY_AGENT_THINKING; const registered = ( modelsJson.providers['prime-inference']?.models ?? [] ) as { id: string; thinkingLevelMap?: Record }[]; const chosen = registered.find((model) => model.id === piggyDefaultModelId()); assert.ok(chosen, 'the default is not registered with the provider'); assert.ok( chosen.thinkingLevelMap, `${chosen.id} is the default and has no thinkingLevelMap, so its reasoning is whatever the endpoint feels like`, ); assert.equal( typeof chosen.thinkingLevelMap[level], 'string', `${chosen.id} does not map the configured thinking level '${level}'`, ); }); test('the catalogue cannot be reordered by a caller', () => { // It is serialised to the browser on every session; one sort() at a call // site would reorder the picker for every other session in the process. The // order is models.json's, which is no longer the same thing as "the default // first" — asserting that conflated the two and broke when the default moved. const registered = (modelsJson.providers['prime-inference']?.models ?? []).map( (model) => model.id, ); const first = piggyModelCatalogue(); first.reverse(); assert.deepEqual( piggyModelCatalogue().map((option) => option.id), registered, ); }); test('the provider points at Prime Inference', () => { assert.equal(piggyInferenceBaseUrl(), 'https://api.pinference.ai/api/v1'); });