Lumbridge Bench

2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
@@ -0,0 +1,171 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ImageResponse } from "next/og";
+import { fmtParams, getRun, getRuns, limitOf } from "@/lib/results";
+
+/**
+ * Shareable score card as a PNG.
+ *
+ * One image serves two jobs: the download button and the OpenGraph tag, so a
+ * pasted link unfurls as the card itself. Fonts are bundled rather than
+ * fetched at request time — satori needs real font buffers, and a network
+ * dependency inside image generation fails in exactly the situations where
+ * you want a share link to work.
+ *
+ * Note: satori supports flexbox only. Every container needs an explicit
+ * display value; CSS grid silently renders nothing.
+ */
+
+export const dynamic = "force-static";
+
+export function generateStaticParams() {
+  return getRuns().map((r) => ({ id: r.run_id.slice(0, 8) }));
+}
+
+const FONT_DIR = path.join(process.cwd(), "assets", "fonts");
+const font = (file: string) => fs.readFileSync(path.join(FONT_DIR, file));
+
+const VOID = "#08090b";
+const RULE = "#1e232b";
+const INK = "#e8e6e1";
+const DIM = "#79818d";
+const DIMMER = "#4a515b";
+const SIGNAL = "#f2b134";
+const REFERENCE = "#5896ae";
+
+function Stat({
+  label,
+  value,
+  unit,
+  color = INK,
+  hint,
+}: {
+  label: string;
+  value: string;
+  unit?: string;
+  color?: string;
+  hint?: string;
+}) {
+  return (
+    <div style={{ display: "flex", flexDirection: "column", flex: 1 }}>
+      <div style={{ fontSize: 15, letterSpacing: 3, color: DIMMER, textTransform: "uppercase" }}>
+        {label}
+      </div>
+      <div style={{ display: "flex", alignItems: "baseline", marginTop: 12 }}>
+        <span style={{ fontFamily: "Instrument", fontSize: 62, color, lineHeight: 1 }}>{value}</span>
+        {unit && <span style={{ fontSize: 17, color: DIMMER, marginLeft: 7 }}>{unit}</span>}
+      </div>
+      {hint && <div style={{ fontSize: 14, color: DIMMER, marginTop: 8 }}>{hint}</div>}
+    </div>
+  );
+}
+
+export async function GET(
+  _req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) {
+  const { id } = await params;
+  const run = getRun(id);
+  if (!run) return new Response("not found", { status: 404 });
+
+  const v = run.verdict;
+  const limit = limitOf(run);
+  const perf = run.perf?.points ?? [];
+  const single = perf.find((p) => p.concurrency === 1);
+  const peak = perf.length
+    ? perf.reduce((a, b) => (b.output_tps_total > a.output_tps_total ? b : a))
+    : null;
+
+  return new ImageResponse(
+    (
+      <div
+        style={{
+          width: "100%",
+          height: "100%",
+          display: "flex",
+          flexDirection: "column",
+          background: VOID,
+          fontFamily: "Plex",
+          color: INK,
+          padding: 56,
+        }}
+      >
+        {/* header */}
+        <div style={{ display: "flex", justifyContent: "space-between", alignItems: "baseline" }}>
+          <div style={{ display: "flex", alignItems: "baseline" }}>
+            <span style={{ fontFamily: "Instrument", fontSize: 30 }}>bench</span>
+            <span style={{ fontFamily: "Instrument", fontSize: 30, color: DIMMER }}>.karti.ai</span>
+          </div>
+          <span style={{ fontSize: 15, letterSpacing: 3, color: DIMMER, textTransform: "uppercase" }}>
+            {run.timestamp.slice(0, 10)}
+          </span>
+        </div>
+
+        <div style={{ display: "flex", height: 1, background: RULE, marginTop: 24, marginBottom: 40 }} />
+
+        {/* title */}
+        <div style={{ display: "flex", flexDirection: "column" }}>
+          <span style={{ fontFamily: "Instrument", fontSize: 68, lineHeight: 1.05 }}>
+            {run.target.display_name}
+          </span>
+          <span style={{ fontSize: 18, color: DIM, marginTop: 16 }}>
+            {run.host.hardware} · {fmtParams(run.target)} ·{" "}
+            {run.target.quant?.toUpperCase()} · {run.target.serving.engine}
+          </span>
+        </div>
+
+        {/* stats */}
+        <div style={{ display: "flex", marginTop: "auto", gap: 24 }}>
+          <Stat
+            label="signal"
+            value={v.signal_score != null ? v.signal_score.toFixed(3) : "—"}
+            color={SIGNAL}
+            hint="our workloads"
+          />
+          <Stat
+            label="reference"
+            value={v.reference_score != null ? v.reference_score.toFixed(3) : "—"}
+            color={REFERENCE}
+            hint={limit ? `capped ${limit}` : "calibration"}
+          />
+          <Stat
+            label="single stream"
+            value={single ? single.output_tps_per_stream.toFixed(1) : "—"}
+            unit="tok/s"
+            hint={single?.ttft_p50_ms ? `ttft ${Math.round(single.ttft_p50_ms)}ms` : undefined}
+          />
+          <Stat
+            label="peak"
+            value={peak ? peak.output_tps_total.toFixed(0) : "—"}
+            unit="tok/s"
+            hint={peak ? `at ×${peak.concurrency}` : undefined}
+          />
+        </div>
+
+        {/* tick strip — a measurement scale across the full plate */}
+        <div style={{ display: "flex", marginTop: 40, justifyContent: "space-between", width: "100%" }}>
+          {Array.from({ length: 96 }).map((_, i) => (
+            <div
+              key={i}
+              style={{
+                display: "flex",
+                width: 1,
+                height: 8,
+                background: i % 6 === 0 ? SIGNAL : RULE,
+              }}
+            />
+          ))}
+        </div>
+      </div>
+    ),
+    {
+      width: 1200,
+      height: 630,
+      fonts: [
+        { name: "Instrument", data: font("InstrumentSerif-Regular.ttf"), style: "normal", weight: 400 },
+        { name: "Plex", data: font("IBMPlexMono-Regular.ttf"), style: "normal", weight: 400 },
+        { name: "Plex", data: font("IBMPlexMono-SemiBold.ttf"), style: "normal", weight: 600 },
+      ],
+    },
+  );
+}
@@ -0,0 +1,13 @@
+import { getRuns } from "@/lib/results";
+
+export const dynamic = "force-dynamic";
+
+/** Liveness probe used by the deploy verify step. */
+export async function GET() {
+  const runs = getRuns();
+  return Response.json({
+    ok: true,
+    runs: runs.length,
+    latest: runs[0]?.timestamp ?? null,
+  });
+}
@@ -0,0 +1,73 @@
+import { NextResponse } from "next/server";
+import { parseHuggingFaceRef } from "@/lib/model-ref";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+const controlPlaneUrl = (
+  process.env.LUMBRIDGE_CONTROL_PLANE_URL ?? "http://127.0.0.1:8902"
+).replace(/\/+$/, "");
+
+/**
+ * Record a Hugging Face model suggestion for Karti to review manually.
+ *
+ * This endpoint only forwards an inert review record to the Lumbridge control
+ * plane. Neither service downloads, imports, schedules, or executes a model.
+ */
+
+export async function POST(req: Request) {
+  let body: { model?: string; notes?: string };
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: "invalid JSON" }, { status: 400 });
+  }
+
+  const ref = parseHuggingFaceRef(body.model ?? "");
+  if (!ref) {
+    return NextResponse.json(
+      { error: "Expected a Hugging Face model reference like `owner/model`." },
+      { status: 400 },
+    );
+  }
+
+  try {
+    const response = await fetch(`${controlPlaneUrl}/api/model-suggestions`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        sourceUrl: `https://huggingface.co/${ref}`,
+        reason: "Suggested via Lumbridge Bench for manual model review.",
+        notes: (body.notes ?? "").slice(0, 2000) || undefined,
+      }),
+      cache: "no-store",
+      signal: AbortSignal.timeout(5_000),
+    });
+    if (!response.ok) {
+      const result = await response.json().catch(() => null) as {
+        error?: string;
+      } | null;
+      return NextResponse.json(
+        {
+          error:
+            result?.error ??
+            "The Lumbridge review inbox could not accept that suggestion.",
+        },
+        { status: response.status },
+      );
+    }
+  } catch {
+    return NextResponse.json(
+      { error: "The Lumbridge review inbox is temporarily unavailable." },
+      { status: 503 },
+    );
+  }
+
+  return NextResponse.json({
+    ok: true,
+    model: ref,
+    message:
+      "Suggestion received. Karti reviews every model manually. Nothing is " +
+      "downloaded or run automatically; approved results appear on the board later.",
+  });
+}