Lumbridge Bench

2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
@@ -0,0 +1,167 @@
+"""Model registry.
+
+A *target* is not a model. It is a specific way of serving a specific model:
+
+    (model × host × quantization × serving config × checkpoint)
+
+This is the central design decision of the bench and it cannot be retrofitted
+later without invalidating every result already recorded. On unified-memory
+boxes like the DGX Spark, serving flags move throughput more than a model swap
+does -- `--enforce-eager` alone can cost 30%+, and `--gpu-memory-utilization`
+determines how much KV cache exists, which sets the concurrency ceiling. A
+registry keyed on model name alone produces numbers nobody can reproduce.
+
+`checkpoint` is first-class for the same reason: once we fine-tune our own
+models, we will compare `karti-7b@step2000` against `karti-7b@step4000` far
+more often than we compare Qwen against Llama.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+REGISTRY_PATH = Path(__file__).parent / "models.yaml"
+
+
+@dataclass(frozen=True)
+class Host:
+    """A machine that can serve models."""
+
+    id: str
+    ssh: str | None = None
+    hardware: str | None = None
+    memory_gb: int | None = None
+    memory_bandwidth_gbs: int | None = None
+    notes: str | None = None
+
+
+@dataclass(frozen=True)
+class Serving:
+    """How a target is served. Part of the identity of a result."""
+
+    engine: str  # vllm | sglang | llama.cpp | anthropic | openai | ...
+    model_name: str  # value sent as "model" in the API request
+    base_url: str | None = None
+    service: str | None = None  # inspect openai-api service prefix
+    max_model_len: int | None = None
+    flags: dict[str, Any] = field(default_factory=dict)
+    # How to sample this model during quality runs. Part of the identity of a result: the
+    # same model at temperature 1 and at temperature 0.6 is two different measurements.
+    #
+    # Not a global constant, because the right value is a property of the model. Greedy
+    # decoding looks like the obvious choice for reproducibility and is actively wrong for
+    # reasoning models -- Qwen3 in thinking mode degenerates into repetition at
+    # temperature 0, and a 2-sample probe that took two minutes ran past twelve without
+    # terminating. Reproducibility comes from pinning the seed, not from killing entropy.
+    sampling: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def is_local(self) -> bool:
+        return self.engine in {"vllm", "sglang", "llama.cpp", "ollama"}
+
+
+@dataclass(frozen=True)
+class Target:
+    """One benchmarkable configuration."""
+
+    id: str
+    display_name: str
+    serving: Serving
+    host: str
+    family: str | None = None
+    params_b: float | None = None
+    active_params_b: float | None = None  # MoE: params active per token
+    quant: str | None = None
+    checkpoint: str | None = None
+    tier: str = "candidate"  # production | candidate | reference | baseline
+    aliases: list[str] = field(default_factory=list)
+    notes: str | None = None
+
+    @property
+    def inspect_model(self) -> str:
+        """The model string Inspect uses to address this target."""
+        s = self.serving
+        if s.engine in {"anthropic", "openai", "google", "openrouter", "grok"}:
+            return f"{s.engine}/{s.model_name}"
+        if s.is_local:
+            if not s.service:
+                raise ValueError(f"target {self.id}: local serving requires a 'service'")
+            # Inspect convention: openai-api/<service>/<model>, with base_url and
+            # api key read from <SERVICE>_BASE_URL / <SERVICE>_API_KEY.
+            return f"openai-api/{s.service}/{s.model_name}"
+        raise ValueError(f"target {self.id}: unsupported engine {s.engine!r}")
+
+    @property
+    def env(self) -> dict[str, str]:
+        """Environment variables Inspect needs to reach this target."""
+        s = self.serving
+        if not s.is_local or not s.service:
+            return {}
+        prefix = s.service.upper().replace("-", "_")
+        env = {f"{prefix}_API_KEY": os.environ.get(f"{prefix}_API_KEY", "no-key-required")}
+        if s.base_url:
+            env[f"{prefix}_BASE_URL"] = s.base_url
+        return env
+
+    def apply_env(self) -> None:
+        """Export this target's env vars into the current process."""
+        os.environ.update(self.env)
+
+    @property
+    def slug(self) -> str:
+        """Filesystem-safe identifier used in results filenames."""
+        return self.id.replace("/", "_").replace("@", "__")
+
+
+@dataclass(frozen=True)
+class Registry:
+    hosts: dict[str, Host]
+    targets: dict[str, Target]
+
+    def target(self, ident: str) -> Target:
+        """Look up a target by id or alias."""
+        if ident in self.targets:
+            return self.targets[ident]
+        for t in self.targets.values():
+            if ident in t.aliases:
+                return t
+        known = ", ".join(sorted(self.targets))
+        raise KeyError(f"unknown target {ident!r}. known targets: {known}")
+
+    def host(self, ident: str) -> Host:
+        if ident not in self.hosts:
+            raise KeyError(f"unknown host {ident!r}")
+        return self.hosts[ident]
+
+    def by_tier(self, tier: str) -> list[Target]:
+        return [t for t in self.targets.values() if t.tier == tier]
+
+
+def load_registry(path: Path | None = None) -> Registry:
+    raw = yaml.safe_load((path or REGISTRY_PATH).read_text())
+
+    hosts = {
+        hid: Host(id=hid, **(spec or {}))
+        for hid, spec in (raw.get("hosts") or {}).items()
+    }
+
+    targets: dict[str, Target] = {}
+    for spec in raw.get("targets") or []:
+        spec = dict(spec)
+        serving = Serving(**spec.pop("serving"))
+        target = Target(serving=serving, **spec)
+        if target.host not in hosts:
+            raise ValueError(f"target {target.id}: unknown host {target.host!r}")
+        if target.id in targets:
+            raise ValueError(f"duplicate target id {target.id!r}")
+        targets[target.id] = target
+
+    return Registry(hosts=hosts, targets=targets)
+
+
+__all__ = ["Host", "Registry", "Serving", "Target", "load_registry"]
@@ -0,0 +1,109 @@
+# Lumbridge Bench target registry
+#
+# A target = (model x host x quant x serving config x checkpoint).
+# See kbench/registry/__init__.py for why the key is this wide.
+#
+# `serving.flags` records the ACTUAL flags the server was started with. Snapshot
+# them from a live host with:  kbench snapshot <host>
+# Results are only comparable between targets with identical flags.
+
+hosts:
+  your-node:
+    ssh: your-node
+    hardware: NVIDIA GB10 Grace Blackwell (DGX Spark)
+    memory_gb: 121
+    memory_bandwidth_gbs: 273
+    notes: >
+      Unified LPDDR5X. Memory bandwidth is the binding constraint on decode
+      throughput, not compute -- large dense models are bandwidth-starved here
+      while MoE models with small active-param counts do comparatively well.
+
+  metal:
+    ssh: your-second-node
+    hardware: TBD
+    notes: Secondary inference box. Not yet benchmarked.
+
+  your-second-node:
+    ssh: root@your-second-node
+    hardware: TBD
+    notes: Storage origin box, 761GB. Not yet benchmarked.
+
+  api:
+    hardware: hosted
+    notes: Frontier APIs. Used as quality ceiling and as judge models.
+
+targets:
+  # ---------------------------------------------------------------------
+  # PRODUCTION -- currently serving live traffic, so regressions here matter
+  # ---------------------------------------------------------------------
+  - id: qwen3.6-35b-a3b-nvfp4@your-node
+    display_name: Qwen3.6-35B-A3B (NVFP4)
+    family: qwen3.6
+    params_b: 35
+    active_params_b: 3
+    quant: nvfp4
+    host: your-node
+    tier: production
+    aliases: [brain, spark]
+    notes: >
+      Live behind the `brain` and `local-moe` aliases. This is the
+      model the bench exists to interrogate: it was deployed without
+      measurement.
+
+      Flags below were snapshotted from the live process on 2026-08-03 and had
+      drifted badly from what was recorded here: gpu_memory_utilization was
+      0.25 not 0.55, the MoE backend was flashinfer_cutlass not flashinfer_b12x,
+      and MTP speculative decoding was running but undocumented. Since target
+      identity IS the serving config, score cards taken against the old entry
+      would have been mislabeled. Re-snapshot before trusting a comparison.
+
+      FLAGS OF CONCERN, in the order worth testing:
+      (1) --max-num-seqs 4 caps concurrency at four sequences, so a throughput
+          sweep past 4 measures queueing, not the server.
+      (2) --gpu-memory-utilization 0.25 leaves ~75% of unified memory unused,
+          capping KV cache and therefore concurrency.
+      (3) --enforce-eager disables CUDA graphs, which costs throughput.
+      (4) MTP is on; report spec_acceptance_rate, since a low acceptance rate
+          means the draft tokens are wasted work.
+    serving:
+      engine: vllm
+      service: spark
+      base_url: http://your-node:8001/v1
+      model_name: brain
+      max_model_len: 65536
+      # Qwen3's documented values for thinking mode. Do NOT set temperature 0 here: this
+      # model degenerates into endless repetition under greedy decoding, and a two-sample
+      # probe that normally finishes in two minutes ran past twelve without terminating.
+      # The seed is what makes a re-run a re-run.
+      sampling:
+        temperature: 0.6
+        top_p: 0.95
+        top_k: 20
+        seed: 20260804
+        max_tokens: 4096
+      flags:
+        async_scheduling: true
+        enforce_eager: true
+        gpu_memory_utilization: 0.25
+        kv_cache_dtype: fp8
+        max_num_batched_tokens: 4096
+        max_num_seqs: 4
+        moe_backend: flashinfer_cutlass
+        reasoning_parser: qwen3
+        speculative_config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
+        tool_call_parser: qwen3_coder
+        enable_auto_tool_choice: true
+
+  # ---------------------------------------------------------------------
+  # BASELINE -- frontier APIs. Quality ceiling, and the judge for graded tasks.
+  # Uncomment once the corresponding API key is in .env.
+  # ---------------------------------------------------------------------
+  # - id: claude-opus-5@api
+  #   display_name: Claude Opus 5
+  #   family: claude
+  #   host: api
+  #   tier: baseline
+  #   aliases: [opus]
+  #   serving:
+  #     engine: anthropic
+  #     model_name: claude-opus-5