PIG-Demo/src/content/evidence.ts

/**
 * Every external claim this site makes, with the link that backs it, plus the
 * handful of numbers about our own word list.
 *
 * Home and Honesty both quote these. They are here, once, so the two pages
 * cannot drift into citing the same fact two different ways — which is the
 * usual way an honest site becomes a dishonest one.
 *
 * The word-list counts are literals rather than imports on purpose: the answer
 * list is 4,603 strings sitting outside the Vite root, and pulling it in to
 * render one number would put the whole dictionary in the entry chunk. They are
 * reproducible with the command in `wordListRebuild` and are checked by CI.
 */

export interface Citation {
  /** What the source proves, in one line. */
  claim: string;
  label: string;
  href: string;
}

/**
 * The credential. Wordle is not our choice of demo — it is Prime Intellect's
 * own hello-world, in three separate places in their stack.
 */
export const helloWorldCitations: readonly Citation[] = [
  {
    claim: 'One of the five basic end-to-end examples in prime-rl, their RL trainer.',
    label: 'prime-rl / examples / basic / wordle',
    href: 'https://github.com/PrimeIntellect-ai/prime-rl/tree/main/examples/basic/wordle',
  },
  {
    claim: 'A shipped environment in verifiers, the library the whole ecosystem builds on.',
    label: 'verifiers / environments / wordle',
    href: 'https://github.com/PrimeIntellect-ai/verifiers/tree/main/environments/wordle',
  },
  {
    claim: 'The environment used in the official lab-cookbook prompt-optimisation tutorial.',
    label: 'lab-cookbook / guides / prompt optimization',
    href: 'https://github.com/PrimeIntellect-ai/lab-cookbook/tree/main/guides/04-prompt-optimization',
  },
];

/**
 * The one measured, citable training result on this site. It is theirs, not
 * ours, and it is a WIN RATE.
 *
 * Their README also publishes average-reward figures for the same runs. We do
 * not quote those anywhere and neither should you: the reward function has
 * changed across versions of the environment, the two numbers were produced
 * under different versions, and nobody re-measured them. A win rate survives a
 * reward change. An average reward does not.
 */
export const trainingResult = {
  model: 'Qwen3-1.7B',
  before: '0%',
  after: '~60%',
  metric: 'win rate',
  method: 'SFT warm-up, then multi-turn RL with group-relative advantages (GRPO)',
  /** From the same README: 20 held-out words, 3 rollouts each. */
  evalDescription: '20 held-out words the model never trained on, played 3 times each',
  /**
   * Rendered wherever the number is, in these words. Home and Honesty both
   * print it, and a caveat that appears on one page and not the other is the
   * kind of thing a careful reader notices first.
   */
  caveat:
    'We quote the win rate only. The reward numbers in that write-up span versions of the environment and were never re-measured together.',
  source: {
    label: 'prime-rl / examples / basic / wordle',
    href: 'https://github.com/PrimeIntellect-ai/prime-rl/tree/main/examples/basic/wordle',
  },
  checkpoints: [
    {
      label: 'PrimeIntellect/Qwen3-1.7B-Wordle-SFT',
      href: 'https://huggingface.co/PrimeIntellect/Qwen3-1.7B-Wordle-SFT',
    },
    {
      label: 'PrimeIntellect/Qwen3-1.7B-Wordle-RL',
      href: 'https://huggingface.co/PrimeIntellect/Qwen3-1.7B-Wordle-RL',
    },
  ],
} as const;

/**
 * Our word list, by the numbers. Answers are the intersection of Wordnik's
 * headwords and SCOWL's common-American tier, minus a short hand-written
 * blocklist; guesses are all of Wordnik's five-letter words.
 */
/**
 * Same greedy max-entropy solver, run over both answer pools, 250 sampled
 * targets each. Measured on amd-server, 2026-08-28. This is the only honest way
 * to compare the two lists: the published 3.4212 optimum for the original list
 * comes from an exhaustive search, and setting our greedy number against it
 * would be comparing two different algorithms and calling it a property of the
 * words.
 */
export const poolComparison = {
  original: { size: '2,315', opener: 'RAISE', mean: '3.552', worst: '5' },
  ours: { size: '4,603', opener: 'TARES', mean: '3.700', worst: '6' },
  sampled: '250',
} as const;

export const wordList = {
  answers: 4603,
  guesses: 11846,
  /** Share of answers ending in a plain plural S. */
  endsInS: '32.9%',
  /** Share of answers ending in -ED. */
  endsInEd: '6.1%',
  /** Both together. */
  endsInSorEd: '39.0%',
  /** The original game's hand-curated answer list, for comparison. */
  originalAnswers: 2315,
  rebuild: 'uv run python envs/wordle_five/words/build_words.py',
  sources: [
    { label: 'Wordnik word list (2021-07-29)', href: 'https://github.com/wordnik/wordlist' },
    { label: 'SCOWL / wamerican (2020.12.07)', href: 'http://wordlist.aspell.net/' },
  ],
} as const;

/**
 * The famous 3.42. It belongs to the original game's 2,315-word answer list and
 * to Alex Selby's exact solver, and it is quoted on this site only to say that
 * it is not ours.
 */
export const optimalPlay = {
  average: '3.42 guesses',
  label: 'Selby’s exact optimal solution for the original 2,315-word list',
  href: 'https://sonorouschocolate.com/notes/index.php/The_best_strategies_for_Wordle',
} as const;

/** Commands anyone can run against a clone of this repository. */
export const reproduce = {
  clone: 'git clone https://git.karti.ai/PIG/PIG-Demo',
  install: 'cd PIG-Demo && uv sync --all-packages',
  evaluate: 'uv run vf-eval wordle-five -n 8',
  conformance: 'pnpm conformance',
} as const;