Lumbridge Bench

2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
@@ -0,0 +1,109 @@
+# Lumbridge Bench target registry
+#
+# A target = (model x host x quant x serving config x checkpoint).
+# See kbench/registry/__init__.py for why the key is this wide.
+#
+# `serving.flags` records the ACTUAL flags the server was started with. Snapshot
+# them from a live host with:  kbench snapshot <host>
+# Results are only comparable between targets with identical flags.
+
+hosts:
+  your-node:
+    ssh: your-node
+    hardware: NVIDIA GB10 Grace Blackwell (DGX Spark)
+    memory_gb: 121
+    memory_bandwidth_gbs: 273
+    notes: >
+      Unified LPDDR5X. Memory bandwidth is the binding constraint on decode
+      throughput, not compute -- large dense models are bandwidth-starved here
+      while MoE models with small active-param counts do comparatively well.
+
+  metal:
+    ssh: your-second-node
+    hardware: TBD
+    notes: Secondary inference box. Not yet benchmarked.
+
+  your-second-node:
+    ssh: root@your-second-node
+    hardware: TBD
+    notes: Storage origin box, 761GB. Not yet benchmarked.
+
+  api:
+    hardware: hosted
+    notes: Frontier APIs. Used as quality ceiling and as judge models.
+
+targets:
+  # ---------------------------------------------------------------------
+  # PRODUCTION -- currently serving live traffic, so regressions here matter
+  # ---------------------------------------------------------------------
+  - id: qwen3.6-35b-a3b-nvfp4@your-node
+    display_name: Qwen3.6-35B-A3B (NVFP4)
+    family: qwen3.6
+    params_b: 35
+    active_params_b: 3
+    quant: nvfp4
+    host: your-node
+    tier: production
+    aliases: [brain, spark]
+    notes: >
+      Live behind the `brain` and `local-moe` aliases. This is the
+      model the bench exists to interrogate: it was deployed without
+      measurement.
+
+      Flags below were snapshotted from the live process on 2026-08-03 and had
+      drifted badly from what was recorded here: gpu_memory_utilization was
+      0.25 not 0.55, the MoE backend was flashinfer_cutlass not flashinfer_b12x,
+      and MTP speculative decoding was running but undocumented. Since target
+      identity IS the serving config, score cards taken against the old entry
+      would have been mislabeled. Re-snapshot before trusting a comparison.
+
+      FLAGS OF CONCERN, in the order worth testing:
+      (1) --max-num-seqs 4 caps concurrency at four sequences, so a throughput
+          sweep past 4 measures queueing, not the server.
+      (2) --gpu-memory-utilization 0.25 leaves ~75% of unified memory unused,
+          capping KV cache and therefore concurrency.
+      (3) --enforce-eager disables CUDA graphs, which costs throughput.
+      (4) MTP is on; report spec_acceptance_rate, since a low acceptance rate
+          means the draft tokens are wasted work.
+    serving:
+      engine: vllm
+      service: spark
+      base_url: http://your-node:8001/v1
+      model_name: brain
+      max_model_len: 65536
+      # Qwen3's documented values for thinking mode. Do NOT set temperature 0 here: this
+      # model degenerates into endless repetition under greedy decoding, and a two-sample
+      # probe that normally finishes in two minutes ran past twelve without terminating.
+      # The seed is what makes a re-run a re-run.
+      sampling:
+        temperature: 0.6
+        top_p: 0.95
+        top_k: 20
+        seed: 20260804
+        max_tokens: 4096
+      flags:
+        async_scheduling: true
+        enforce_eager: true
+        gpu_memory_utilization: 0.25
+        kv_cache_dtype: fp8
+        max_num_batched_tokens: 4096
+        max_num_seqs: 4
+        moe_backend: flashinfer_cutlass
+        reasoning_parser: qwen3
+        speculative_config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
+        tool_call_parser: qwen3_coder
+        enable_auto_tool_choice: true
+
+  # ---------------------------------------------------------------------
+  # BASELINE -- frontier APIs. Quality ceiling, and the judge for graded tasks.
+  # Uncomment once the corresponding API key is in .env.
+  # ---------------------------------------------------------------------
+  # - id: claude-opus-5@api
+  #   display_name: Claude Opus 5
+  #   family: claude
+  #   host: api
+  #   tier: baseline
+  #   aliases: [opus]
+  #   serving:
+  #     engine: anthropic
+  #     model_name: claude-opus-5