bench/site/app/method/page.tsx

export const dynamic = "force-static";

export const metadata = {
  title: "Method — Lumbridge Bench",
  description:
    "How these numbers are produced, and what they do and do not mean.",
};

function Section({
  n,
  title,
  children,
}: {
  n: string;
  title: string;
  children: React.ReactNode;
}) {
  return (
    <section className="border-t border-rule py-10">
      <div className="grid gap-6 md:grid-cols-[auto_1fr] md:gap-12">
        <div className="label pt-1 md:w-16">{n}</div>
        <div>
          <h2 className="font-display mb-4 text-2xl">{title}</h2>
          <div className="max-w-2xl space-y-4 text-xs leading-relaxed text-dim">
            {children}
          </div>
        </div>
      </div>
    </section>
  );
}

export default function MethodPage() {
  return (
    <main className="mx-auto max-w-5xl px-6 py-14">
      <div className="rise mb-10 max-w-2xl">
        <h1 className="font-display text-4xl leading-tight sm:text-5xl">
          Method
        </h1>
        <p className="mt-5 text-sm leading-relaxed text-dim">
          What these numbers are, how they are produced, and — the part most
          leaderboards skip — what they cannot tell you.
        </p>
      </div>

      <Section n="01" title="A target is not a model">
        <p>
          Every card measures a{" "}
          <span className="text-ink">
            model × host × quantization × serving config × checkpoint
          </span>
          , not a model name. On unified-memory hardware the serving flags move
          throughput more than swapping the model does — disabling CUDA graphs,
          or capping memory utilization at 55%, changes the answer by more than
          the difference between two model families.
        </p>
        <p>
          So the flags are recorded from the live server on every run and shown
          on the card. Two rows are only comparable if their configurations
          match.
        </p>
      </Section>

      <Section n="02" title="Two tiers, and only one is a ranking">
        <p>
          <span className="text-signal">Signal</span> tasks are private, built
          from work we actually do. They are the measurement.
        </p>
        <p>
          <span className="text-reference">Reference</span> tasks are public
          benchmarks run unmodified. They exist to calibrate the harness, not to
          rank models: if our number lands far from the published value, our
          harness is broken. Public benchmarks have been in training data for
          years, and a high reference score is not evidence of much.
        </p>
        <p>
          A bench made only of private tasks would be unfalsifiable — no reader
          could distinguish a bad model from a broken harness. That is what the
          reference tier is for.
        </p>
      </Section>

      <Section n="03" title="Why the private set stays private">
        <p>
          The usual reason given is confidentiality. The real reason is
          contamination: a published test set gets scraped, trained on, and
          stops measuring capability. That is how MMLU, GSM8K and HumanEval
          stopped being informative.
        </p>
        <p>
          Samples carry embedded canary strings, so if a future model reproduces
          them we can demonstrate contamination rather than suspect it.
        </p>
      </Section>

      <Section n="04" title="Performance is measured, not quoted">
        <p>
          Each run drives real concurrent load at the serving endpoint and
          records time-to-first-token, inter-token latency, per-stream decode
          rate, and aggregate throughput at several concurrency levels.
        </p>
        <p>
          Two details do most of the work. Every request carries a unique prefix
          so <span className="text-ink">prefix caching</span> cannot make
          prefill free and inflate the result. And output length is pinned, so
          concurrency levels finish at identical token counts and remain
          comparable. Without both, the numbers look considerably better and
          mean nothing.
        </p>
        <p>
          Single-stream and aggregate figures are both reported because they
          disagree. On our hardware, throughput rises roughly 12× from one
          stream to thirty-two while per-stream decode falls to a third — batch
          serving and interactive use want opposite configurations.
        </p>
      </Section>

      <Section n="05" title="What a capped run means">
        <p>
          Long reference datasets against a slow local model take hours, so some
          runs are capped at a sample limit. When that happens the cap is
          recorded in the result and displayed on the card. A capped score is
          not comparable to a full-dataset score and is never presented as one.
        </p>
      </Section>

      <Section n="06" title="Known limits">
        <p>
          Single hardware sample per host — no variance across identical
          machines. Perf runs are single-shot rather than averaged over repeats,
          so treat small differences between runs as noise; the run-to-run
          spread on aggregate throughput is meaningful.
        </p>
        <p>
          Rubric-graded samples depend on a judge model, which drifts as that
          model changes. Deterministic scorers are preferred wherever the
          property can be checked mechanically.
        </p>
      </Section>
    </main>
  );
}