pig/apps/piggy/test/agent-models.test.ts

import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import test from 'node:test';
import {
  isPiggyModelId,
  piggyDefaultModelId,
  piggyInferenceBaseUrl,
  piggyModelCatalogue,
} from '../src/agent/models';

const modelsJson = JSON.parse(
  readFileSync(fileURLToPath(new URL('../src/agent/models.json', import.meta.url)), 'utf8'),
) as {
  providers: Record<string, { models: { id: string }[] }>;
};

test('every id in the picker is one the provider actually registers', () => {
  // The whole point of a curated shortlist is that nothing in it 404s. The
  // catalogue and models.json are the same five models by construction, and
  // this is what keeps them that way when someone adds a sixth to one file.
  const registered = (modelsJson.providers['prime-inference']?.models ?? []).map(
    (model) => model.id,
  );
  const offered = piggyModelCatalogue().map((option) => option.id);

  assert.deepEqual(offered, registered);
  assert.ok(offered.length >= 4, 'the picker should offer a real choice, not just the default');
  for (const id of offered) {
    // Prime Inference ids are always provider-qualified. A bare model name is
    // the classic copy-and-paste error and it fails as a 404 at the endpoint.
    assert.match(id, /^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/, `${id} is not provider-qualified`);
    assert.ok(isPiggyModelId(id));
  }
});

test('the default is in the catalogue and there is exactly one of it', () => {
  const catalogue = piggyModelCatalogue();
  const defaults = catalogue.filter((option) => option.isDefault);

  assert.equal(defaults.length, 1);
  assert.equal(defaults[0]?.id, piggyDefaultModelId());
  // The default is the SUPER, not the nano, and the reason is availability
  // rather than quality. On 2026-08-14 `nvidia/nemotron-3-nano-30b-a3b` stopped
  // answering on Prime Inference — the connection was accepted and no response
  // headers ever arrived, three attempts at 45s each — while every other model
  // in this catalogue answered in under two seconds on the same key in the same
  // minute. The nano stays in the picker for anyone who wants it back.
  assert.equal(piggyDefaultModelId(), 'nvidia/nemotron-3-super-120b-a12b');
  assert.equal(isPiggyModelId('nvidia/nemotron-3-super-120b-a12b'), true);
  assert.equal(isPiggyModelId('nvidia/nemotron-3-nano-30b-a3b'), true);
  assert.equal(isPiggyModelId('nvidia/nemotron-9000'), false);
});

test('the picker can price and size every choice', () => {
  for (const option of piggyModelCatalogue()) {
    // Dollars per million tokens, NOT cents: the field names say so, and this
    // is the one money field in PIG that is not an integer of cents. A price
    // of 0 here would render as "free" in the picker, which no model is.
    assert.ok(option.costPerMTokIn > 0, `${option.id} has no input price`);
    assert.ok(option.costPerMTokOut > 0, `${option.id} has no output price`);
    assert.ok(option.costPerMTokOut >= option.costPerMTokIn, `${option.id} prices output too low`);
    assert.ok(option.contextWindow >= 100_000, `${option.id} is too small for a CRM transcript`);
    assert.ok(option.label.length > 0);
    assert.ok((option.hint ?? '').length > 0, `${option.id} would render as a blank picker row`);
  }
});

/**
 * This used to assert that the default was the cheapest thing on offer, and it
 * was a good rule until the cheapest thing stopped answering. What actually
 * protects the choice is not the ranking but the ceiling: the panel is docked on
 * every page, so the default is the price of a typo, and the failure worth
 * catching is somebody making a frontier model the default by accident. A
 * deliberate move up the price list should pass; a slip to Opus should not.
 */
test('the default is a cheap model, even though it is no longer the cheapest', () => {
  const catalogue = piggyModelCatalogue();
  const cheapest = [...catalogue].sort((a, b) => a.costPerMTokIn - b.costPerMTokIn)[0];
  const chosen = catalogue.find((option) => option.id === piggyDefaultModelId());
  assert.ok(chosen && cheapest);

  assert.notEqual(chosen.id, cheapest.id, 'the cheapest model answers again; revisit the default');
  // Six times the price of the nano is still about $0.0017 a turn, or roughly
  // 117,000 turns on a $200 credit. A dollar per million input tokens is an
  // order of magnitude above that and two below every frontier model here.
  assert.ok(chosen.costPerMTokIn <= 1, `${chosen.id} is too dear to be the default`);
  const frontier = catalogue.filter((option) => option.costPerMTokIn >= 5);
  assert.ok(frontier.length >= 2, 'the picker no longer offers a frontier option to contrast with');
  for (const option of frontier) {
    assert.notEqual(option.id, chosen.id, 'a frontier model became the default by accident');
  }
});

/**
 * The half of the reasoning trap that nobody would guess, pinned to whichever
 * model is the default rather than to a name.
 *
 * `@earendil-works/pi-ai@0.84.1` turns a thinking level of `off` into no
 * `reasoning_effort` field at all unless the model entry maps it, and the
 * endpoint's own default then wins — 6,195 output tokens of reasoning and an
 * empty answer. `agent-thinking.test.ts` pins the behaviour end to end; this
 * pins the datum it depends on, which is the thing a new default would silently
 * arrive without.
 */
test('the default carries a thinking map for the level Piggy is configured to run at', async () => {
  process.env.DATABASE_URL ??= 'postgres://pig:pig@localhost:54330/pig';
  process.env.PIGGY_INTERNAL_TOKEN ??= 'test-internal-token-for-piggy-000000';
  process.env.PRIME_API_KEY ??= 'test-key-not-used-offline';
  const { loadPiggyConfig } = await import('../src/config');
  const level = loadPiggyConfig().PIGGY_AGENT_THINKING;

  const registered = (
    modelsJson.providers['prime-inference']?.models ?? []
  ) as { id: string; thinkingLevelMap?: Record<string, string> }[];
  const chosen = registered.find((model) => model.id === piggyDefaultModelId());
  assert.ok(chosen, 'the default is not registered with the provider');
  assert.ok(
    chosen.thinkingLevelMap,
    `${chosen.id} is the default and has no thinkingLevelMap, so its reasoning is whatever the endpoint feels like`,
  );
  assert.equal(
    typeof chosen.thinkingLevelMap[level],
    'string',
    `${chosen.id} does not map the configured thinking level '${level}'`,
  );
});

test('the catalogue cannot be reordered by a caller', () => {
  // It is serialised to the browser on every session; one sort() at a call
  // site would reorder the picker for every other session in the process. The
  // order is models.json's, which is no longer the same thing as "the default
  // first" — asserting that conflated the two and broke when the default moved.
  const registered = (modelsJson.providers['prime-inference']?.models ?? []).map(
    (model) => model.id,
  );
  const first = piggyModelCatalogue();
  first.reverse();

  assert.deepEqual(
    piggyModelCatalogue().map((option) => option.id),
    registered,
  );
});

test('the provider points at Prime Inference', () => {
  assert.equal(piggyInferenceBaseUrl(), 'https://api.pinference.ai/api/v1');
});