Lumbridge Bench

site/lib/model-ref.ts

@@ -0,0 +1,12 @@
+/**
+ * Normalize a public Hugging Face reference before it becomes an inert
+ * Lumbridge model-suggestion record.
+ */
+export function parseHuggingFaceRef(input: string): string | null {
+  const trimmed = input.trim().replace(/\/+$/, "");
+  const url = trimmed
+    .replace(/^https?:\/\/(www\.)?huggingface\.co\//i, "")
+    .replace(/^hf:\/\//i, "");
+  // owner/model — letters, digits, dot, dash, underscore.
+  return /^[\w.-]+\/[\w.-]+$/.test(url) ? url : null;
+}

site/lib/results.ts

@@ -0,0 +1,122 @@
+import fs from "node:fs";
+import path from "node:path";
+
+/**
+ * The leaderboard is a view over committed JSON, not a database.
+ *
+ * results/*.json is the record of every run and it lives in git, so the site
+ * has no state of its own and cannot drift from what was actually measured.
+ * Supabase enters only for submissions and auth, which are genuinely mutable.
+ */
+
+const RESULTS_DIR = path.join(process.cwd(), "..", "results");
+
+export type SampleOutcome = {
+  sample_id: string;
+  score: number;
+  passed: boolean;
+  excerpt: string | null;
+  metadata: Record<string, unknown>;
+};
+
+export type QualityResult = {
+  task: string;
+  tier: "reference" | "signal" | "example";
+  dataset_version: string | null;
+  n_samples: number;
+  metrics: Record<string, number>;
+  samples: SampleOutcome[];
+  duration_s: number | null;
+  error: string | null;
+};
+
+export type PerfPoint = {
+  concurrency: number;
+  input_tokens: number;
+  output_tokens: number;
+  completed: number;
+  failed: number;
+  output_tps_total: number;
+  output_tps_per_stream: number;
+  ttft_p50_ms: number | null;
+  ttft_p95_ms: number | null;
+  tpot_p50_ms: number | null;
+  prefill_tps: number | null;
+};
+
+export type Run = {
+  run_id: string;
+  timestamp: string;
+  target_id: string;
+  target: {
+    display_name: string;
+    family: string | null;
+    params_b: number | null;
+    active_params_b: number | null;
+    quant: string | null;
+    checkpoint: string | null;
+    tier: string;
+    host: string;
+    slug: string;
+    notes: string | null;
+    serving: {
+      engine: string;
+      model_name: string;
+      max_model_len: number | null;
+      flags: Record<string, unknown>;
+    };
+  };
+  host: { id: string; hardware: string | null; memory_gb: number | null };
+  quality: QualityResult[];
+  perf: { engine: string; points: PerfPoint[] } | null;
+  verdict: {
+    signal_score: number | null;
+    reference_score: number | null;
+    single_stream_tps?: number;
+    interactive_viable?: boolean;
+    peak_throughput_tps?: number;
+    peak_throughput_concurrency?: number;
+  };
+  schema_version: number;
+};
+
+export function getRuns(): Run[] {
+  if (!fs.existsSync(RESULTS_DIR)) return [];
+  return fs
+    .readdirSync(RESULTS_DIR)
+    .filter((f) => f.endsWith(".json"))
+    .map((f) => JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), "utf8")) as Run)
+    .sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+}
+
+export function getRun(id: string): Run | undefined {
+  return getRuns().find((r) => r.run_id.startsWith(id));
+}
+
+/**
+ * Latest run per target — what the leaderboard ranks.
+ *
+ * Ranking every run would let a target dominate the board simply by being
+ * measured more often.
+ */
+export function latestPerTarget(runs: Run[]): Run[] {
+  const seen = new Map<string, Run>();
+  for (const run of runs) {
+    if (!seen.has(run.target_id)) seen.set(run.target_id, run);
+  }
+  return [...seen.values()];
+}
+
+/** Was any task in this run capped with --limit? Must never be hidden. */
+export function limitOf(run: Run): number | null {
+  for (const q of run.quality) {
+    if (q.metrics?._limit) return q.metrics._limit;
+  }
+  return null;
+}
+
+export function fmtParams(t: Run["target"]): string {
+  if (!t.params_b) return "—";
+  const base = `${t.params_b}B`;
+  return t.active_params_b ? `${base}·${t.active_params_b}B active` : base;
+}