# Lumbridge Bench target registry
#
# A target = (model x host x quant x serving config x checkpoint).
# See kbench/registry/__init__.py for why the key is this wide.
#
# `serving.flags` records the ACTUAL flags the server was started with. Snapshot
# them from a live host with:  kbench snapshot <host>
# Results are only comparable between targets with identical flags.

hosts:
  your-node:
    ssh: your-node
    hardware: NVIDIA GB10 Grace Blackwell (DGX Spark)
    memory_gb: 121
    memory_bandwidth_gbs: 273
    notes: >
      Unified LPDDR5X. Memory bandwidth is the binding constraint on decode
      throughput, not compute -- large dense models are bandwidth-starved here
      while MoE models with small active-param counts do comparatively well.

  metal:
    ssh: your-second-node
    hardware: TBD
    notes: Secondary inference box. Not yet benchmarked.

  your-second-node:
    ssh: root@your-second-node
    hardware: TBD
    notes: Storage origin box, 761GB. Not yet benchmarked.

  api:
    hardware: hosted
    notes: Frontier APIs. Used as quality ceiling and as judge models.

targets:
  # ---------------------------------------------------------------------
  # PRODUCTION -- currently serving live traffic, so regressions here matter
  # ---------------------------------------------------------------------
  - id: qwen3.6-35b-a3b-nvfp4@your-node
    display_name: Qwen3.6-35B-A3B (NVFP4)
    family: qwen3.6
    params_b: 35
    active_params_b: 3
    quant: nvfp4
    host: your-node
    tier: production
    aliases: [brain, spark]
    notes: >
      Live behind the `brain` and `local-moe` aliases. This is the
      model the bench exists to interrogate: it was deployed without
      measurement.

      Flags below were snapshotted from the live process on 2026-08-03 and had
      drifted badly from what was recorded here: gpu_memory_utilization was
      0.25 not 0.55, the MoE backend was flashinfer_cutlass not flashinfer_b12x,
      and MTP speculative decoding was running but undocumented. Since target
      identity IS the serving config, score cards taken against the old entry
      would have been mislabeled. Re-snapshot before trusting a comparison.

      FLAGS OF CONCERN, in the order worth testing:
      (1) --max-num-seqs 4 caps concurrency at four sequences, so a throughput
          sweep past 4 measures queueing, not the server.
      (2) --gpu-memory-utilization 0.25 leaves ~75% of unified memory unused,
          capping KV cache and therefore concurrency.
      (3) --enforce-eager disables CUDA graphs, which costs throughput.
      (4) MTP is on; report spec_acceptance_rate, since a low acceptance rate
          means the draft tokens are wasted work.
    serving:
      engine: vllm
      service: spark
      base_url: http://your-node:8001/v1
      model_name: brain
      max_model_len: 65536
      # Qwen3's documented values for thinking mode. Do NOT set temperature 0 here: this
      # model degenerates into endless repetition under greedy decoding, and a two-sample
      # probe that normally finishes in two minutes ran past twelve without terminating.
      # The seed is what makes a re-run a re-run.
      sampling:
        temperature: 0.6
        top_p: 0.95
        top_k: 20
        seed: 20260804
        max_tokens: 4096
      flags:
        async_scheduling: true
        enforce_eager: true
        gpu_memory_utilization: 0.25
        kv_cache_dtype: fp8
        max_num_batched_tokens: 4096
        max_num_seqs: 4
        moe_backend: flashinfer_cutlass
        reasoning_parser: qwen3
        speculative_config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
        tool_call_parser: qwen3_coder
        enable_auto_tool_choice: true

  # ---------------------------------------------------------------------
  # BASELINE -- frontier APIs. Quality ceiling, and the judge for graded tasks.
  # Uncomment once the corresponding API key is in .env.
  # ---------------------------------------------------------------------
  # - id: claude-opus-5@api
  #   display_name: Claude Opus 5
  #   family: claude
  #   host: api
  #   tier: baseline
  #   aliases: [opus]
  #   serving:
  #     engine: anthropic
  #     model_name: claude-opus-5