Lumbridge Bench
This commit is contained in:
Karti Tripathi
@@ -0,0 +1,280 @@
|
||||
import Link from "next/link";
|
||||
import { ScopeTrace } from "@/components/ScopeTrace";
|
||||
import {
|
||||
fmtParams,
|
||||
getRuns,
|
||||
latestPerTarget,
|
||||
limitOf,
|
||||
type Run,
|
||||
} from "@/lib/results";
|
||||
|
||||
export const dynamic = "force-static";
|
||||
|
||||
function Metric({
|
||||
label,
|
||||
value,
|
||||
unit,
|
||||
tone = "ink",
|
||||
hint,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
unit?: string;
|
||||
tone?: "ink" | "signal" | "reference" | "dim";
|
||||
hint?: string;
|
||||
}) {
|
||||
const toneClass = {
|
||||
ink: "text-ink",
|
||||
signal: "text-signal",
|
||||
reference: "text-reference",
|
||||
dim: "text-dim",
|
||||
}[tone];
|
||||
return (
|
||||
<div className="min-w-0">
|
||||
<div className="label mb-2">{label}</div>
|
||||
<div
|
||||
className={`readout font-display text-4xl leading-none ${toneClass}`}
|
||||
>
|
||||
{value}
|
||||
{unit && (
|
||||
<span className="ml-1 font-mono text-sm text-dimmer">{unit}</span>
|
||||
)}
|
||||
</div>
|
||||
{hint && (
|
||||
<div className="mt-2 text-[11px] leading-snug text-dimmer">{hint}</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function PrimaryCard({ run }: { run: Run }) {
|
||||
const v = run.verdict;
|
||||
const limit = limitOf(run);
|
||||
const perf = run.perf?.points ?? [];
|
||||
const single = perf.find((p) => p.concurrency === 1);
|
||||
const peak = perf.length
|
||||
? perf.reduce((a, b) => (b.output_tps_total > a.output_tps_total ? b : a))
|
||||
: null;
|
||||
|
||||
return (
|
||||
<section className="rise panel relative overflow-hidden">
|
||||
<div className="absolute right-0 top-0 border-b border-l border-rule px-3 py-1.5">
|
||||
<span className="label text-signal">in production</span>
|
||||
</div>
|
||||
|
||||
<div className="border-b border-rule p-6 sm:p-8">
|
||||
<div className="label mb-3">primary target</div>
|
||||
<h2 className="font-display text-3xl leading-tight sm:text-4xl">
|
||||
{run.target.display_name}
|
||||
</h2>
|
||||
<p className="mt-3 max-w-2xl text-xs leading-relaxed text-dim">
|
||||
{run.host.hardware} · {fmtParams(run.target)} ·{" "}
|
||||
{run.target.quant?.toUpperCase()} · {run.target.serving.engine} · ctx{" "}
|
||||
{run.target.serving.max_model_len?.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-8 border-b border-rule p-6 sm:grid-cols-2 sm:p-8 lg:grid-cols-4">
|
||||
<Metric
|
||||
label="signal score"
|
||||
value={v.signal_score != null ? v.signal_score.toFixed(3) : "—"}
|
||||
tone="signal"
|
||||
hint={
|
||||
v.signal_score == null
|
||||
? "no private tasks authored yet"
|
||||
: "our own workloads"
|
||||
}
|
||||
/>
|
||||
<Metric
|
||||
label="reference"
|
||||
value={v.reference_score != null ? v.reference_score.toFixed(3) : "—"}
|
||||
tone="reference"
|
||||
hint={
|
||||
limit
|
||||
? `capped at ${limit} samples · calibration only`
|
||||
: "calibration only"
|
||||
}
|
||||
/>
|
||||
<Metric
|
||||
label="single stream"
|
||||
value={single ? single.output_tps_per_stream.toFixed(1) : "—"}
|
||||
unit="tok/s"
|
||||
hint={
|
||||
single
|
||||
? `TTFT ${Math.round(single.ttft_p50_ms ?? 0)}ms · what one user feels`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<Metric
|
||||
label="peak throughput"
|
||||
value={peak ? peak.output_tps_total.toFixed(0) : "—"}
|
||||
unit="tok/s"
|
||||
hint={
|
||||
peak
|
||||
? `at concurrency ${peak.concurrency} · what the box serves`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{perf.length > 1 && (
|
||||
<div className="p-6 sm:p-8">
|
||||
<ScopeTrace points={perf} />
|
||||
<div className="mt-4 flex flex-wrap items-baseline justify-between gap-3">
|
||||
<p className="max-w-xl text-[11px] leading-relaxed text-dimmer">
|
||||
Throughput scales{" "}
|
||||
{peak && single
|
||||
? `${(peak.output_tps_total / single.output_tps_per_stream).toFixed(1)}×`
|
||||
: "—"}{" "}
|
||||
from one stream to {peak?.concurrency ?? "—"}, but per-stream
|
||||
decode falls to {peak?.output_tps_per_stream.toFixed(1) ?? "—"}{" "}
|
||||
tok/s. Batch work and interactive work want opposite settings on
|
||||
this box.
|
||||
</p>
|
||||
<Link
|
||||
href={`/run/${run.run_id.slice(0, 8)}`}
|
||||
className="shrink-0 border border-rule-bright px-3 py-1.5 text-xs transition-colors hover:border-signal hover:text-signal"
|
||||
>
|
||||
full score card →
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Home() {
|
||||
const runs = getRuns();
|
||||
const board = latestPerTarget(runs);
|
||||
const primary = board.find((r) => r.target.tier === "production") ?? board[0];
|
||||
const lastMeasured = runs[0]?.timestamp?.slice(0, 10);
|
||||
|
||||
return (
|
||||
<main className="mx-auto max-w-6xl px-6 py-14">
|
||||
<div className="rise mb-14 max-w-3xl">
|
||||
<h1 className="font-display text-5xl leading-[1.05] sm:text-6xl">
|
||||
Does this model earn
|
||||
<br />
|
||||
its place on our hardware?
|
||||
</h1>
|
||||
<p className="mt-6 max-w-xl text-sm leading-relaxed text-dim">
|
||||
Lumbridge Bench is the evidence layer for Lumbridge Compute. Two
|
||||
numbers decide a self-hosting call: whether a model is good enough at
|
||||
the work we actually do, and whether it is fast enough on the box we
|
||||
would run it on. Most leaderboards report only the first. Every card
|
||||
here reports both, measured together, on the same machine.
|
||||
</p>
|
||||
<div className="mt-8 flex flex-wrap items-center gap-x-8 gap-y-3 text-xs text-dimmer">
|
||||
<span>
|
||||
<span className="text-ink">{board.length}</span> target
|
||||
{board.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<span>
|
||||
<span className="text-ink">{runs.length}</span> run
|
||||
{runs.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
{lastMeasured && (
|
||||
<span>
|
||||
last measured <span className="text-ink">{lastMeasured}</span>
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{primary && <PrimaryCard run={primary} />}
|
||||
|
||||
<div className="ticks mt-16 mb-8 opacity-40" />
|
||||
|
||||
<section>
|
||||
<div className="mb-6 flex items-baseline justify-between">
|
||||
<h2 className="font-display text-2xl">The board</h2>
|
||||
<span className="label">latest run per target</span>
|
||||
</div>
|
||||
|
||||
<div className="-mx-6 overflow-x-auto px-6">
|
||||
<table className="w-full min-w-[720px] border-collapse text-xs">
|
||||
<thead>
|
||||
<tr className="border-b border-rule-bright text-left">
|
||||
<th className="label py-3 pr-4 font-normal">target</th>
|
||||
<th className="label py-3 pr-4 font-normal">host</th>
|
||||
<th className="label py-3 pr-4 text-right font-normal">
|
||||
signal
|
||||
</th>
|
||||
<th className="label py-3 pr-4 text-right font-normal">
|
||||
reference
|
||||
</th>
|
||||
<th className="label py-3 pr-4 text-right font-normal">
|
||||
tok/s ×1
|
||||
</th>
|
||||
<th className="label py-3 pr-4 text-right font-normal">
|
||||
peak tok/s
|
||||
</th>
|
||||
<th className="label py-3 text-right font-normal">measured</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{board.map((run) => {
|
||||
const v = run.verdict;
|
||||
return (
|
||||
<tr
|
||||
key={run.run_id}
|
||||
className="group border-b border-rule transition-colors hover:bg-panel-2"
|
||||
>
|
||||
<td className="py-4 pr-4">
|
||||
<Link
|
||||
href={`/run/${run.run_id.slice(0, 8)}`}
|
||||
className="block"
|
||||
>
|
||||
<span className="text-ink transition-colors group-hover:text-signal">
|
||||
{run.target.display_name}
|
||||
</span>
|
||||
<span className="mt-1 block text-[11px] text-dimmer">
|
||||
{fmtParams(run.target)} ·{" "}
|
||||
{run.target.quant?.toUpperCase()}
|
||||
</span>
|
||||
</Link>
|
||||
</td>
|
||||
<td className="py-4 pr-4 text-dim">{run.host.id}</td>
|
||||
<td className="readout py-4 pr-4 text-right text-signal">
|
||||
{v.signal_score != null ? v.signal_score.toFixed(3) : "—"}
|
||||
</td>
|
||||
<td className="readout py-4 pr-4 text-right text-reference">
|
||||
{v.reference_score != null
|
||||
? v.reference_score.toFixed(3)
|
||||
: "—"}
|
||||
</td>
|
||||
<td className="readout py-4 pr-4 text-right">
|
||||
{v.single_stream_tps?.toFixed(1) ?? "—"}
|
||||
</td>
|
||||
<td className="readout py-4 pr-4 text-right">
|
||||
{v.peak_throughput_tps?.toFixed(0) ?? "—"}
|
||||
</td>
|
||||
<td className="py-4 text-right text-dimmer">
|
||||
{run.timestamp.slice(0, 10)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div className="mt-8 grid gap-4 text-[11px] leading-relaxed text-dimmer sm:grid-cols-2">
|
||||
<p className="border-l border-signal-dim pl-4">
|
||||
<span className="text-signal">Signal</span> scores come from private
|
||||
tasks built from our own workloads. They are the measurement. The
|
||||
set is not published — a public test set gets scraped into the next
|
||||
training run and stops measuring anything.
|
||||
</p>
|
||||
<p className="border-l border-reference-dim pl-4">
|
||||
<span className="text-reference">Reference</span> scores come from
|
||||
public benchmarks, unmodified. They are calibration, not a ranking:
|
||||
if one lands far from its published value, our harness is wrong, not
|
||||
the model.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user