compute/registry/models.yaml

apiVersion: lumbridge/v1
kind: Registry
version: 1

# Local to this box. Ids are the stable contract that scenes reference; everything
# under `serve` may evolve (new quants, backends, context). Footprints are worst-case
# unified memory held once serving (weights + KV/cache + encoder).

models:

  ears:
    name: "Nemotron 3.5 ASR streaming (0.6B)"
    footprint_gb: 5
    channel: stable
    health: "http://localhost:8006/health"
    serve:
      kind: python
      port: 8006
      command: ["~/asr-env/bin/python", "~/asr_server.py"]
      env:
        ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b"

  brain:
    name: "Qwen3.6 35B-A3B MoE (NVFP4)"
    # Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by
    # gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built
    # 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping
    # max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost.
    #
    # MTP (multi-token prediction) is ON as of 2026-08-02 and measured
    # 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4
    # (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is
    # declared for headroom.
    #
    # IMPORTANT — this model MUST start before ears/voice. Its KV-cache
    # profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable
    # dips to ~7GB against a 3GB watchdog floor even when it loads alone. That
    # is why the primary scene uses `order: listed` with brain first. Three earlier
    # attempts with ears+voice already resident were watchdog-killed.
    footprint_gb: 40
    channel: stable
    health: "http://localhost:8001/v1/models"
    health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
    serve:
      kind: vllm
      executable: "~/vllm-env/bin/vllm"
      port: 8001
      weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
      served_name:                # stable aliases so downstream agents survive a weight swap
        - brain
        - "local-moe"
        - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
        - "unsloth/Qwen3.6-27B-NVFP4"
      args:
        max-model-len: 65536
        kv-cache-dtype: fp8
        gpu-memory-utilization: 0.25     # 0.40 + MTP tripped the watchdog
        max-num-seqs: 4                  # bounds KV; the single biggest memory lever
        enforce-eager: true
        max-num-batched-tokens: 4096
        async-scheduling: true
        # The checkpoint already ships 19 MTP tensors (mtp.fc.weight,
        # mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0
        # deprecates the method name qwen3_5_mtp -> "mtp" and resolves
        # Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok
        # per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM
        # warns >1 re-runs the same MTP layer and can lower per-position
        # acceptance, so the gain is not linear — after any change check
        # vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"}
        # and go back to 1 if the second position accepts poorly.
        speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
        limit-mm-per-prompt: '{"image": 2, "video": 0}'
        enable-auto-tool-choice: true
        tool-call-parser: qwen3_coder
        reasoning-parser: qwen3
        # MUST be flashinfer_cutlass, NOT the Unsloth b12x recipe. b12x supports
        # only *quantized* NVFP4 MoE and hard-fails on the unquantized MTP draft
        # layers: "moe_backend='flashinfer_b12x' is not supported for
        # unquantized MoE. Expected one of ['triton','flashinfer_trtllm',
        # 'flashinfer_cutlass','aiter']".
        moe-backend: flashinfer_cutlass
      env:
        CUTE_DSL_ARCH: sm_121a

  voice:
    # Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on
    # 2026-08-02 — and the name is misleading. Turbo is not a degraded fast
    # variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a
    # 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` +
    # s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`.
    # The vocoder is what produces the waveform, so that is what you hear.
    # It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade.
    #
    # Both checkpoints stay resident (CBX_VARIANTS) so the /voice A/B Lab can
    # re-test per request without a ~30s reload; that costs ~3.6GB extra.
    name: "Chatterbox Turbo TTS (meanflow vocoder, voice cloning)"
    footprint_gb: 8
    channel: stable
    health: "http://localhost:8095/health"
    serve:
      kind: uvicorn
      port: 8095
      command:
        - "~/cbx-env/bin/uvicorn"
        - "voice_api:app"
        - "--host"
        - "0.0.0.0"
        - "--port"
        - "8095"
        - "--app-dir"
        - "~/chatterbox"
      env:
        # Turbo is the served default. Both stay loaded so the /voice A/B Lab
        # can compare per request; this only decides what an unqualified
        # request gets — i.e. what the downstream caller hears.
        CBX_VARIANT: turbo
        CBX_VARIANTS: "full,turbo"
        # Local copies, not from_pretrained(): snapshot_download needs the
        # network and would silently follow upstream if the repo moved.
        CBX_TURBO_DIR: "/srv/models/chatterbox-turbo"
        CBX_FULL_DIR: "/srv/models/chatterbox-full"

  brain-laguna:
    name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)"
    # 71 GB main weights + draft, runtime/JIT overhead, and a conservative
    # 64k FP8 KV allowance. This is a replacement for `brain`, never a peer.
    footprint_gb: 86
    channel: stable
    health: "http://localhost:8001/v1/models"
    health_contains: "poolside/Laguna-S-2.1-NVFP4"
    serve:
      kind: vllm
      executable: "~/laguna-env/bin/vllm"
      port: 8001
      weights: "~/models/Laguna-S-2.1-NVFP4"
      served_name:
        - brain
        - "local-moe"
        - "poolside/Laguna-S-2.1-NVFP4"
      args:
        default-chat-template-kwargs: '{"enable_thinking": false}'
        speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}'
        max-model-len: 65536
        kv-cache-dtype: fp8
        gpu-memory-utilization: 0.70
        max-num-seqs: 8
        max-num-batched-tokens: 4096
        enforce-eager: true
        enable-auto-tool-choice: true
        tool-call-parser: poolside_v1
        reasoning-parser: poolside_v1
        override-generation-config: '{"temperature":0.7,"top_p":0.95}'
      env:
        CUTE_DSL_ARCH: sm_121a
        MAX_JOBS: "4"
        PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"

  brain-gemma:
    name: "Gemma 4 31B-IT (NVFP4)"
    # 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation
    # headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window
    # attention (10 of 60 layers are full-attention, rest capped at a 1024-token
    # window) means its real KV cost at 64k context is far lighter than brain/
    # brain-laguna's dense-attention KV — this cap is a conservative first pass,
    # not yet confirmed by an actual load on the reference node.
    footprint_gb: 44
    channel: stable
    health: "http://localhost:8001/v1/models"
    health_contains: "nvidia/Gemma-4-31B-IT-NVFP4"
    serve:
      kind: vllm
      executable: "~/gemma-env/bin/vllm"
      port: 8001
      weights: "~/models/Gemma-4-31B-IT-NVFP4"
      served_name:
        - brain
        - "local-moe"
        - "nvidia/Gemma-4-31B-IT-NVFP4"
      args:
        max-model-len: 65536
        kv-cache-dtype: fp8
        gpu-memory-utilization: 0.35
        quantization: modelopt
        max-num-seqs: 8
        max-num-batched-tokens: 4096
        enforce-eager: true
        # Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM
        # as of this writing — verify before relying on function-calling.
      env:
        CUTE_DSL_ARCH: sm_121a

  music:
    name: "ACE-Step 1.5 XL 4B sft (music generation)"
    footprint_gb: 24              # measured ~21GB resident live on the reference node; 24 declared for safety
    channel: stable
    health: "http://localhost:8010/health"
    serve:
      kind: exec
      port: 8010
      # 4B XL sft DiT + 1.7B LM (pt backend). Custom FastAPI wrapper (POST /generate -> WAV).
      # Runs as a systemd --user unit on the node today; this is the equivalent launch.
      command: ["~/ACE-Step-1.5/.venv/bin/python", "~/ace_step_server.py"]
      env:
        ACESTEP_CONFIG_PATH: acestep-v15-xl-sft
        ACESTEP_LM_MODEL_PATH: acestep-5Hz-lm-1.7B
        ACESTEP_LM_BACKEND: pt
        ACESTEP_API_PORT: "8010"
        ACESTEP_API_HOST: 0.0.0.0

  image:
    name: "FLUX.2-klein-9B (BF16)"
    # BFL's own card claims ~29GB VRAM, but an independent benchmark that
    # actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident
    # (95s/image) — trust the on-hardware number for admission control until
    # we re-measure it ourselves. Non-commercial license (BFL FLUX license);
    # fine for personal use, not for anything monetized. Requires accepting
    # the license on HF and `hf auth login` before the weights can download.
    footprint_gb: 66
    channel: stable
    health: "http://localhost:8007/health"
    serve:
      kind: diffusers
      port: 8007
      command: ["~/flux-env/bin/python", "~/flux_server.py"]
      env:
        FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B"
        FLUX_API_PORT: "8007"

  embed:
    name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval"
    # 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim,
    # 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by
    # the gpu-memory-utilization reservation, not the tiny weights.
    footprint_gb: 2
    channel: stable
    health: "http://localhost:8012/v1/models"
    health_contains: "embed"
    serve:
      kind: vllm
      executable: "~/vllm-env/bin/vllm"
      port: 8012
      weights: "~/models/Nemotron-3-Embed-1B-NVFP4"
      served_name:
        - embed
        - nemotron-embed
      args:
        runner: pooling
        max-model-len: 32768
        gpu-memory-utilization: 0.05
        enforce-eager: true

  # Alternatives, ready to register when wanted:
  #   image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark
  #   image-qwen     -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering