Karti Tripathi
281 lines
10 KiB
TypeScript
281 lines
10 KiB
TypeScript
import Link from "next/link";
|
||
import { ScopeTrace } from "@/components/ScopeTrace";
|
||
import {
|
||
fmtParams,
|
||
getRuns,
|
||
latestPerTarget,
|
||
limitOf,
|
||
type Run,
|
||
} from "@/lib/results";
|
||
|
||
export const dynamic = "force-static";
|
||
|
||
function Metric({
|
||
label,
|
||
value,
|
||
unit,
|
||
tone = "ink",
|
||
hint,
|
||
}: {
|
||
label: string;
|
||
value: string;
|
||
unit?: string;
|
||
tone?: "ink" | "signal" | "reference" | "dim";
|
||
hint?: string;
|
||
}) {
|
||
const toneClass = {
|
||
ink: "text-ink",
|
||
signal: "text-signal",
|
||
reference: "text-reference",
|
||
dim: "text-dim",
|
||
}[tone];
|
||
return (
|
||
<div className="min-w-0">
|
||
<div className="label mb-2">{label}</div>
|
||
<div
|
||
className={`readout font-display text-4xl leading-none ${toneClass}`}
|
||
>
|
||
{value}
|
||
{unit && (
|
||
<span className="ml-1 font-mono text-sm text-dimmer">{unit}</span>
|
||
)}
|
||
</div>
|
||
{hint && (
|
||
<div className="mt-2 text-[11px] leading-snug text-dimmer">{hint}</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
function PrimaryCard({ run }: { run: Run }) {
|
||
const v = run.verdict;
|
||
const limit = limitOf(run);
|
||
const perf = run.perf?.points ?? [];
|
||
const single = perf.find((p) => p.concurrency === 1);
|
||
const peak = perf.length
|
||
? perf.reduce((a, b) => (b.output_tps_total > a.output_tps_total ? b : a))
|
||
: null;
|
||
|
||
return (
|
||
<section className="rise panel relative overflow-hidden">
|
||
<div className="absolute right-0 top-0 border-b border-l border-rule px-3 py-1.5">
|
||
<span className="label text-signal">in production</span>
|
||
</div>
|
||
|
||
<div className="border-b border-rule p-6 sm:p-8">
|
||
<div className="label mb-3">primary target</div>
|
||
<h2 className="font-display text-3xl leading-tight sm:text-4xl">
|
||
{run.target.display_name}
|
||
</h2>
|
||
<p className="mt-3 max-w-2xl text-xs leading-relaxed text-dim">
|
||
{run.host.hardware} · {fmtParams(run.target)} ·{" "}
|
||
{run.target.quant?.toUpperCase()} · {run.target.serving.engine} · ctx{" "}
|
||
{run.target.serving.max_model_len?.toLocaleString()}
|
||
</p>
|
||
</div>
|
||
|
||
<div className="grid gap-8 border-b border-rule p-6 sm:grid-cols-2 sm:p-8 lg:grid-cols-4">
|
||
<Metric
|
||
label="signal score"
|
||
value={v.signal_score != null ? v.signal_score.toFixed(3) : "—"}
|
||
tone="signal"
|
||
hint={
|
||
v.signal_score == null
|
||
? "no private tasks authored yet"
|
||
: "our own workloads"
|
||
}
|
||
/>
|
||
<Metric
|
||
label="reference"
|
||
value={v.reference_score != null ? v.reference_score.toFixed(3) : "—"}
|
||
tone="reference"
|
||
hint={
|
||
limit
|
||
? `capped at ${limit} samples · calibration only`
|
||
: "calibration only"
|
||
}
|
||
/>
|
||
<Metric
|
||
label="single stream"
|
||
value={single ? single.output_tps_per_stream.toFixed(1) : "—"}
|
||
unit="tok/s"
|
||
hint={
|
||
single
|
||
? `TTFT ${Math.round(single.ttft_p50_ms ?? 0)}ms · what one user feels`
|
||
: undefined
|
||
}
|
||
/>
|
||
<Metric
|
||
label="peak throughput"
|
||
value={peak ? peak.output_tps_total.toFixed(0) : "—"}
|
||
unit="tok/s"
|
||
hint={
|
||
peak
|
||
? `at concurrency ${peak.concurrency} · what the box serves`
|
||
: undefined
|
||
}
|
||
/>
|
||
</div>
|
||
|
||
{perf.length > 1 && (
|
||
<div className="p-6 sm:p-8">
|
||
<ScopeTrace points={perf} />
|
||
<div className="mt-4 flex flex-wrap items-baseline justify-between gap-3">
|
||
<p className="max-w-xl text-[11px] leading-relaxed text-dimmer">
|
||
Throughput scales{" "}
|
||
{peak && single
|
||
? `${(peak.output_tps_total / single.output_tps_per_stream).toFixed(1)}×`
|
||
: "—"}{" "}
|
||
from one stream to {peak?.concurrency ?? "—"}, but per-stream
|
||
decode falls to {peak?.output_tps_per_stream.toFixed(1) ?? "—"}{" "}
|
||
tok/s. Batch work and interactive work want opposite settings on
|
||
this box.
|
||
</p>
|
||
<Link
|
||
href={`/run/${run.run_id.slice(0, 8)}`}
|
||
className="shrink-0 border border-rule-bright px-3 py-1.5 text-xs transition-colors hover:border-signal hover:text-signal"
|
||
>
|
||
full score card →
|
||
</Link>
|
||
</div>
|
||
</div>
|
||
)}
|
||
</section>
|
||
);
|
||
}
|
||
|
||
export default function Home() {
|
||
const runs = getRuns();
|
||
const board = latestPerTarget(runs);
|
||
const primary = board.find((r) => r.target.tier === "production") ?? board[0];
|
||
const lastMeasured = runs[0]?.timestamp?.slice(0, 10);
|
||
|
||
return (
|
||
<main className="mx-auto max-w-6xl px-6 py-14">
|
||
<div className="rise mb-14 max-w-3xl">
|
||
<h1 className="font-display text-5xl leading-[1.05] sm:text-6xl">
|
||
Does this model earn
|
||
<br />
|
||
its place on our hardware?
|
||
</h1>
|
||
<p className="mt-6 max-w-xl text-sm leading-relaxed text-dim">
|
||
Lumbridge Bench is the evidence layer for Lumbridge Compute. Two
|
||
numbers decide a self-hosting call: whether a model is good enough at
|
||
the work we actually do, and whether it is fast enough on the box we
|
||
would run it on. Most leaderboards report only the first. Every card
|
||
here reports both, measured together, on the same machine.
|
||
</p>
|
||
<div className="mt-8 flex flex-wrap items-center gap-x-8 gap-y-3 text-xs text-dimmer">
|
||
<span>
|
||
<span className="text-ink">{board.length}</span> target
|
||
{board.length === 1 ? "" : "s"}
|
||
</span>
|
||
<span>
|
||
<span className="text-ink">{runs.length}</span> run
|
||
{runs.length === 1 ? "" : "s"}
|
||
</span>
|
||
{lastMeasured && (
|
||
<span>
|
||
last measured <span className="text-ink">{lastMeasured}</span>
|
||
</span>
|
||
)}
|
||
</div>
|
||
</div>
|
||
|
||
{primary && <PrimaryCard run={primary} />}
|
||
|
||
<div className="ticks mt-16 mb-8 opacity-40" />
|
||
|
||
<section>
|
||
<div className="mb-6 flex items-baseline justify-between">
|
||
<h2 className="font-display text-2xl">The board</h2>
|
||
<span className="label">latest run per target</span>
|
||
</div>
|
||
|
||
<div className="-mx-6 overflow-x-auto px-6">
|
||
<table className="w-full min-w-[720px] border-collapse text-xs">
|
||
<thead>
|
||
<tr className="border-b border-rule-bright text-left">
|
||
<th className="label py-3 pr-4 font-normal">target</th>
|
||
<th className="label py-3 pr-4 font-normal">host</th>
|
||
<th className="label py-3 pr-4 text-right font-normal">
|
||
signal
|
||
</th>
|
||
<th className="label py-3 pr-4 text-right font-normal">
|
||
reference
|
||
</th>
|
||
<th className="label py-3 pr-4 text-right font-normal">
|
||
tok/s ×1
|
||
</th>
|
||
<th className="label py-3 pr-4 text-right font-normal">
|
||
peak tok/s
|
||
</th>
|
||
<th className="label py-3 text-right font-normal">measured</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{board.map((run) => {
|
||
const v = run.verdict;
|
||
return (
|
||
<tr
|
||
key={run.run_id}
|
||
className="group border-b border-rule transition-colors hover:bg-panel-2"
|
||
>
|
||
<td className="py-4 pr-4">
|
||
<Link
|
||
href={`/run/${run.run_id.slice(0, 8)}`}
|
||
className="block"
|
||
>
|
||
<span className="text-ink transition-colors group-hover:text-signal">
|
||
{run.target.display_name}
|
||
</span>
|
||
<span className="mt-1 block text-[11px] text-dimmer">
|
||
{fmtParams(run.target)} ·{" "}
|
||
{run.target.quant?.toUpperCase()}
|
||
</span>
|
||
</Link>
|
||
</td>
|
||
<td className="py-4 pr-4 text-dim">{run.host.id}</td>
|
||
<td className="readout py-4 pr-4 text-right text-signal">
|
||
{v.signal_score != null ? v.signal_score.toFixed(3) : "—"}
|
||
</td>
|
||
<td className="readout py-4 pr-4 text-right text-reference">
|
||
{v.reference_score != null
|
||
? v.reference_score.toFixed(3)
|
||
: "—"}
|
||
</td>
|
||
<td className="readout py-4 pr-4 text-right">
|
||
{v.single_stream_tps?.toFixed(1) ?? "—"}
|
||
</td>
|
||
<td className="readout py-4 pr-4 text-right">
|
||
{v.peak_throughput_tps?.toFixed(0) ?? "—"}
|
||
</td>
|
||
<td className="py-4 text-right text-dimmer">
|
||
{run.timestamp.slice(0, 10)}
|
||
</td>
|
||
</tr>
|
||
);
|
||
})}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<div className="mt-8 grid gap-4 text-[11px] leading-relaxed text-dimmer sm:grid-cols-2">
|
||
<p className="border-l border-signal-dim pl-4">
|
||
<span className="text-signal">Signal</span> scores come from private
|
||
tasks built from our own workloads. They are the measurement. The
|
||
set is not published — a public test set gets scraped into the next
|
||
training run and stops measuring anything.
|
||
</p>
|
||
<p className="border-l border-reference-dim pl-4">
|
||
<span className="text-reference">Reference</span> scores come from
|
||
public benchmarks, unmodified. They are calibration, not a ranking:
|
||
if one lands far from its published value, our harness is wrong, not
|
||
the model.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
</main>
|
||
);
|
||
}
|