2302 lines
115 KiB
YAML
2302 lines
115 KiB
YAML
apiVersion: lumbridge/v1
|
|
kind: Registry
|
|
version: 1
|
|
|
|
# Local to this box. Ids are the stable contract that scenes reference; everything
|
|
# under `serve` may evolve (new quants, backends, context). Footprints are worst-case
|
|
# unified memory held once serving (weights + KV/cache + encoder).
|
|
|
|
models:
|
|
|
|
ears:
|
|
name: "Nemotron 3.5 ASR streaming (0.6B)"
|
|
# MEASURED 2026-08-16: 9.9 GB via `nvidia-smi --query-compute-apps`, was
|
|
# declared 5. This was a 2x UNDER-declaration — the same class of error
|
|
# that OOM-killed this very process on 2026-08-11, pointing the other way.
|
|
# The old 5 came from RSS (~1.9-2.1 GB), which does not measure GPU memory
|
|
# on this unified-memory box. Measure with nvidia-smi, never RSS.
|
|
#
|
|
# It is FIXED, not leaking: identical MiB at idle, during and after 59 s of
|
|
# audio in one request. `ears` is position-limited, not memory-limited —
|
|
# it refuses past 6m39s of audio (HTTP 500 from max_position_embeddings)
|
|
# long before it could grow. Declared at 10 with slack.
|
|
footprint_gb: 10
|
|
channel: stable
|
|
health: "http://localhost:8006/health"
|
|
# A listening socket is not a transcriber. The server binds :8006 before
|
|
# the checkpoint and Silero VAD are loaded (~5-14s), so match on the model
|
|
# name the way `voice` matches on model_loaded.
|
|
health_contains: '"model":"nemotron-3.5-asr-streaming-0.6b"'
|
|
# Compute is the ONLY supervisor for this process as of 2026-08-12.
|
|
#
|
|
# It used to be double-managed: this entry AND a user systemd unit
|
|
# (nemotron-asr.service, Restart=always, enabled at default.target) ran the
|
|
# exact same command on the same port. systemd won every race, so whenever
|
|
# it restarted the ASR — a deploy, a crash, the OOM-kill on 2026-08-11 —
|
|
# the pid recorded here went stale and every `scene activate` failed with
|
|
# "'ears' is already serving but is not identity-owned by Compute".
|
|
#
|
|
# The unit is now stopped and disabled (`systemctl --user disable --now
|
|
# nemotron-asr.service`). This supervision block replaces what it provided:
|
|
# Restart=always / RestartSec=5 becomes restart: always with backoff, which
|
|
# is strictly better here — a plain 5s restart loop against a memory-
|
|
# pressure failure is how you turn one OOM into a thrash.
|
|
#
|
|
# If you ever want systemd to own it again, delete this block AND remove
|
|
# `ears` from every scene, or you are right back where you started.
|
|
#
|
|
# Tuned 2026-08-12 by kill -9'ing the process and timing recovery. At
|
|
# check_interval 30 / threshold 3 it came back in ~60s, which is a long
|
|
# time to be deaf in a voice loop. At 10/2 it is ~25s (2 missed probes +
|
|
# ~14s to load the checkpoint and Silero VAD). The backoff is what keeps
|
|
# this from becoming systemd's 5s hot loop under memory pressure, so it
|
|
# stays generous even though detection is now fast.
|
|
supervision:
|
|
restart: always
|
|
check_interval_sec: 10
|
|
failure_threshold: 2
|
|
backoff_sec: 30
|
|
max_backoff_sec: 600
|
|
startup_timeout_sec: 120
|
|
serve:
|
|
kind: python
|
|
port: 8006
|
|
command: ["~/asr-env/bin/python", "~/asr_server.py"]
|
|
env:
|
|
ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b"
|
|
|
|
brain:
|
|
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
|
|
# Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by
|
|
# gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built
|
|
# 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping
|
|
# max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost.
|
|
#
|
|
# MTP (multi-token prediction) is ON as of 2026-08-02 and measured
|
|
# 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4
|
|
# (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is
|
|
# declared for headroom.
|
|
#
|
|
# IMPORTANT — this model MUST start before ears/voice. Its KV-cache
|
|
# profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable
|
|
# dips to ~7GB against a 3GB watchdog floor even when it loads alone. That
|
|
# is why the primary scene uses `order: listed` with brain first. Three earlier
|
|
# attempts with ears+voice already resident were watchdog-killed.
|
|
footprint_gb: 40
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
served_name: # stable aliases so downstream agents survive a weight swap
|
|
- brain
|
|
- "local-moe"
|
|
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
- "unsloth/Qwen3.6-27B-NVFP4"
|
|
args:
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.25 # 0.40 + MTP tripped the watchdog
|
|
max-num-seqs: 4 # bounds KV; the single biggest memory lever
|
|
enforce-eager: true
|
|
max-num-batched-tokens: 4096
|
|
async-scheduling: true
|
|
# The checkpoint already ships 19 MTP tensors (mtp.fc.weight,
|
|
# mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0
|
|
# deprecates the method name qwen3_5_mtp -> "mtp" and resolves
|
|
# Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok
|
|
# per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM
|
|
# warns >1 re-runs the same MTP layer and can lower per-position
|
|
# acceptance, so the gain is not linear — after any change check
|
|
# vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"}
|
|
# and go back to 1 if the second position accepts poorly.
|
|
speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
|
|
limit-mm-per-prompt: '{"image": 2, "video": 0}'
|
|
enable-auto-tool-choice: true
|
|
tool-call-parser: qwen3_coder
|
|
reasoning-parser: qwen3
|
|
# NO explicit moe-backend — let vLLM's oracle choose. Pinning one is a
|
|
# version trap.
|
|
#
|
|
# History: under vLLM 0.25 this HAD to be flashinfer_cutlass, because the
|
|
# Unsloth b12x recipe supports only *quantized* NVFP4 MoE and hard-failed on
|
|
# the unquantized MTP draft layers ("Expected one of ['triton',
|
|
# 'flashinfer_trtllm','flashinfer_cutlass','aiter']").
|
|
#
|
|
# Under vLLM 0.26 that same pin is FATAL — the NvFp4 oracle rejects it on
|
|
# GB10: "NvFp4 MoE backend 'FLASHINFER_CUTLASS' does not support the
|
|
# deployment configuration since kernel does not support current device
|
|
# cuda." Rebuilding the reference node on 2026-08-07 hit exactly this, because
|
|
# vllm-setup.sh asks for `vllm>=0.25.0` and a clean install now gets 0.26.0.
|
|
#
|
|
# Left alone, 0.26 picks VLLM_CUTLASS from ['FLASHINFER_TRTLLM',
|
|
# 'FLASHINFER_CUTEDSL','FLASHINFER_CUTEDSL_BATCHED','FLASHINFER_CUTLASS',
|
|
# 'VLLM_CUTLASS','MARLIN','HUMMING','EMULATION'] and serves correctly with
|
|
# MTP. Verified on the 4TB FE: coherent completions, spec-decode metrics live.
|
|
#
|
|
# ⚠️ VLLM_CUTLASS JIT-compiles kernels and needs `ninja` on PATH. It is NOT
|
|
# pulled in by build-essential/cmake — install ninja-build. Without it the
|
|
# engine dies with FileNotFoundError: 'ninja' AFTER passing the backend
|
|
# check, which reads like an unrelated bug.
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
|
|
voice:
|
|
# Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on
|
|
# 2026-08-02 — and the name is misleading. Turbo is not a degraded fast
|
|
# variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a
|
|
# 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` +
|
|
# s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`.
|
|
# The vocoder is what produces the waveform, so that is what you hear.
|
|
# It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade.
|
|
#
|
|
# 2026-08-16: the FULL CHECKPOINT IS SUNSET at the user's request. This
|
|
# entry is turbo-only. It reverts the 2026-08-13 flip to full, which had
|
|
# doubled unqualified /voice latency (RTF ~0.50 vs ~0.26) against the blind
|
|
# A/B recorded above, and it stops paying the ~3.6 GB that keeping full
|
|
# resident for the /voice A/B Lab cost.
|
|
#
|
|
# ⚠️ CONSEQUENCE: the /voice A/B Lab can no longer serve `variant: "full"`.
|
|
# `_model_for()` in ~/chatterbox/voice_api.py answers a clean HTTP 400
|
|
# (`unknown variant 'full'; loaded: ['turbo']`) rather than silently
|
|
# substituting turbo — an A/B result labelled with the wrong checkpoint
|
|
# being worse than none. The weights are still on disk at CBX_FULL_DIR, so
|
|
# this is reversible by putting `full` back in CBX_VARIANTS.
|
|
name: "Chatterbox TTS — turbo only (voice cloning)"
|
|
# MEASURED 2026-08-16 turbo-only: 3.2 GB GPU via
|
|
# `nvidia-smi --query-compute-apps` (2.8 fresh, 3.2 after two renders).
|
|
# Was 8 with both checkpoints resident. Declared at 5 with slack.
|
|
#
|
|
# ⚠️⚠️ THIS NUMBER DOES NOT BOUND WHAT THIS SERVICE COSTS. `voice` LEAKS
|
|
# HOST RAM, which no GPU measurement sees. On 2026-08-14 it was found
|
|
# holding 57.1 GB RSS against an 8 GB declaration after ~4 hours, having
|
|
# squeezed `brain-nemotron` and `music` off the box; restarting it freed
|
|
# 72 GB. The growth is TIME-based, not request-based — ~8.4 GB/hour while
|
|
# completely idle — and the GPU allocation stays flat throughout, which is
|
|
# exactly why footprint_gb looks innocent here.
|
|
#
|
|
# That leak was characterised with BOTH checkpoints resident; the leading
|
|
# suspect was accumulating per-connection state from persistent TCP
|
|
# connections held open by two downstream clients, never confirmed.
|
|
#
|
|
# RE-MEASURED 2026-08-16 turbo-only: NOT REPRODUCING. RSS sampled every
|
|
# 90 s for 13 min went 2.57 -> 2.65 GB and then sat FLAT at 2.65 across the
|
|
# last ~6 min of true idle. The old rate would have added ~0.84 GB over
|
|
# that window. One downstream persistent connection was ESTABLISHED
|
|
# throughout, so the leading suspect was present and produced no growth,
|
|
# which points at the dual-checkpoint residency as the actual cause.
|
|
# ⚠️ 6 min against a 4-hour original observation is NOT a clearance.
|
|
# Re-check RSS after this has been up overnight before trusting it.
|
|
#
|
|
# ⚠️ DO NOT paper over this with a systemd timer or an out-of-band killer.
|
|
# `voice` is Compute-managed (supervision.restart below); a second
|
|
# supervisor recreates the double-management failure that broke `ears`.
|
|
# The real fixes are (a) memory-based supervision in lumbridge-compute,
|
|
# which today keys on health only and never reads RSS, or (b) finding what
|
|
# those connections retain.
|
|
footprint_gb: 5
|
|
channel: stable
|
|
health: "http://localhost:8095/health"
|
|
# A listening uvicorn socket is not a narrator. Both checkpoints must have
|
|
# loaded before the Scene or the resident supervisor calls this healthy.
|
|
health_contains: '"model_loaded":true'
|
|
# Opt-in and local to this vetted registry entry. Three missed probes avoid
|
|
# restarting across one busy response; backoff prevents a memory-pressure
|
|
# failure from becoming a hot restart loop.
|
|
supervision:
|
|
restart: always
|
|
check_interval_sec: 30
|
|
failure_threshold: 3
|
|
backoff_sec: 60
|
|
max_backoff_sec: 900
|
|
startup_timeout_sec: 180
|
|
serve:
|
|
kind: uvicorn
|
|
port: 8095
|
|
command:
|
|
- "~/cbx-env/bin/uvicorn"
|
|
- "voice_api:app"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8095"
|
|
- "--app-dir"
|
|
- "~/chatterbox"
|
|
env:
|
|
# Turbo is the only checkpoint loaded, and so is what every unqualified
|
|
# request gets — i.e. what the downstream caller hears.
|
|
CBX_VARIANT: turbo
|
|
CBX_VARIANTS: "turbo"
|
|
# Local copies, not from_pretrained(): snapshot_download needs the
|
|
# network and would silently follow upstream if the repo moved.
|
|
CBX_TURBO_DIR: "/srv/models/chatterbox-turbo"
|
|
CBX_FULL_DIR: "/srv/models/chatterbox-full"
|
|
|
|
brain-laguna:
|
|
name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)"
|
|
# 71 GB main weights + draft, runtime/JIT overhead, and a conservative
|
|
# 64k FP8 KV allowance. This is a replacement for `brain`, never a peer.
|
|
footprint_gb: 86
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "poolside/Laguna-S-2.1-NVFP4"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/laguna-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Laguna-S-2.1-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
- "poolside/Laguna-S-2.1-NVFP4"
|
|
args:
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}'
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.70
|
|
max-num-seqs: 8
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-auto-tool-choice: true
|
|
tool-call-parser: poolside_v1
|
|
reasoning-parser: poolside_v1
|
|
override-generation-config: '{"temperature":0.7,"top_p":0.95}'
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
MAX_JOBS: "4"
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
brain-nemotron:
|
|
name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark)"
|
|
# A REPLACEMENT for `brain`, never a peer — it takes :8001. Two MoE brains
|
|
# co-resident does not fit: 68GB is already held by the rest of the primary scene
|
|
# stack plus brain, and both engines spike during KV profiling.
|
|
#
|
|
# Why this model is interesting on THIS box specifically: it is a hybrid
|
|
# Mamba-2 + MoE, and NVIDIA's own reference single-GPU target for it is
|
|
# "1x DGX Spark (GB10)". `brain`'s entire config is contorted around KV
|
|
# cache — max-num-seqs is pinned to 4 purely to stop vLLM building 3.2M
|
|
# tokens of it. Mamba layers carry recurrent state instead of a growing KV
|
|
# cache, so context length costs far less memory here. That is what buys
|
|
# the 1M window NVIDIA advertises on the same 121GB we currently ration
|
|
# at 64k.
|
|
#
|
|
# Weights measured on disk 2026-08-11: 21GB main + 1.3GB DSpark draft.
|
|
#
|
|
# ⚠️ REQUIRES vLLM >= 0.27.1 — its own venv, ~/vllm27-env (vllm 0.27.1,
|
|
# torch 2.13.0+cu132). Do NOT upgrade ~/vllm-env to get this: that env
|
|
# serves brain, embed and ocr, and the 0.25 -> 0.26 jump already made
|
|
# brain's old moe-backend pin fatal on GB10 once. The flags below
|
|
# (--mamba-backend, --mamba-cache-mode, --moe-backend, and the
|
|
# nemotron_v3 reasoning parser) do not exist in 0.26.0 at all.
|
|
#
|
|
# footprint: MEASURED on the reference node, two data points:
|
|
# max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS
|
|
# max-model-len 65536, gpu-memory-utilization 0.28 -> 35.5 GB RSS
|
|
#
|
|
# The first configuration was declared at 46 GB from a paper estimate, ran
|
|
# 11 GB over it, and the kernel OOM-killed `ears` on the next scene
|
|
# transition. Do not estimate this entry — boot it and read RSS.
|
|
#
|
|
# The overhead above vLLM's pool is NOT a constant: 0.38*121.7 = 46.2 GB
|
|
# pool vs 57 GB observed (+10.8), but 0.28*121.7 = 34.1 GB pool vs 35.5 GB
|
|
# observed (+1.4). It scales with max-model-len, because the mamba conv/SSM
|
|
# state is allocated per-sequence OUTSIDE the fraction that
|
|
# gpu-memory-utilization bounds. Long context is cheap in the mamba layers
|
|
# and expensive here — that is the opposite of the intuition the 1M-context
|
|
# headline gives you.
|
|
#
|
|
# 38 declared against 35.5 measured, leaving a little slack for the
|
|
# allocator. Re-measure after ANY change to context, max-num-seqs, or the
|
|
# speculative config.
|
|
footprint_gb: 38
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm27-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
served_name: # same aliases as brain, so downstream agents swap blind
|
|
- brain
|
|
- "local-moe"
|
|
- "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
args:
|
|
# Thinking OFF by default. Measured 2026-08-11: with it on, a trivial
|
|
# P&L question burned 347 completion tokens of correct reasoning before
|
|
# the answer, so every eval case with a max_tokens cap truncated
|
|
# mid-thought and scored as a wrong answer. The model had the right
|
|
# number ($550) the whole time. `brain` serves with thinking off too,
|
|
# so leaving this on also made the A/B against it meaningless.
|
|
# Callers that want it can pass chat_template_kwargs per request.
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
# 64k, matching `brain`. The window is nearly free in the mamba layers
|
|
# but NOT in the interleaved attention layers, which hold real KV, and
|
|
# 256k is what drove this process to 57GB. The 1M window is real but it
|
|
# is not free on a 121GB box that also carries eyes/ears/voice — raise
|
|
# this only against measured headroom in `status`.
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.28
|
|
max-num-seqs: 4
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-prefix-caching: true
|
|
# DSpark: a semi-autoregressive drafter that proposes a whole block per
|
|
# forward pass. NVIDIA recommends it over MTP and DFlash for GB10 and
|
|
# for low-concurrency serving, which is exactly this box.
|
|
speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}'
|
|
# Mamba-2 specific. `align` trades a little memory for aligned state
|
|
# layout; both flags are NVIDIA's published GB10 recipe.
|
|
mamba-backend: flashinfer
|
|
mamba-cache-mode: align
|
|
# Pinned deliberately, against the rule in `brain`'s entry that says to
|
|
# let the oracle choose. NVIDIA names marlin explicitly for GB10 here
|
|
# because the NVFP4 recipe is W4A16 on the experts, which is marlin's
|
|
# case. If a future vLLM rejects this pin the way 0.26 rejected
|
|
# brain's flashinfer_cutlass, drop the line before debugging anything else.
|
|
moe-backend: marlin
|
|
reasoning-parser: nemotron_v3
|
|
tool-call-parser: qwen3_coder
|
|
enable-auto-tool-choice: true
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
brain-gemma:
|
|
name: "Gemma 4 31B-IT (NVFP4)"
|
|
# 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation
|
|
# headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window
|
|
# attention (10 of 60 layers are full-attention, rest capped at a 1024-token
|
|
# window) means its real KV cost at 64k context is far lighter than brain/
|
|
# brain-laguna's dense-attention KV — this cap is a conservative first pass,
|
|
# not yet confirmed by an actual load on the reference node.
|
|
footprint_gb: 44
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "nvidia/Gemma-4-31B-IT-NVFP4"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/gemma-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Gemma-4-31B-IT-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
- "nvidia/Gemma-4-31B-IT-NVFP4"
|
|
args:
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.35
|
|
quantization: modelopt
|
|
max-num-seqs: 8
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
# Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM
|
|
# as of this writing — verify before relying on function-calling.
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
|
|
music:
|
|
name: "MiniMax Music 3 (BF16) — full-song generation"
|
|
# Replaced ACE-Step 1.5 XL 4B on 2026-08-13. The ACE-Step entry had been
|
|
# unbootable for some time: its command pointed at ~/ACE-Step-1.5/.venv,
|
|
# and neither that venv nor the acestep-v15-xl-sft / acestep-5Hz-lm-1.7B
|
|
# weights existed anywhere on the reference node. This slot is a repair, not a trade.
|
|
#
|
|
# Architecture: 8B Global LLM (Qwen3-8B init) predicting RVQ codebook 0
|
|
# frame-by-frame, 0.6B Local LLM for the remaining 7 codebooks, then
|
|
# 2.4B Flow Matching -> 123M Flow-VAE decoder -> 32kHz 16-bit stereo.
|
|
# ~11.1B params total. Up to 5 minutes per generation.
|
|
#
|
|
# SERVING PATH: diffusers ModularPipeline, NOT the SGLang reference repo.
|
|
# That repo hardcodes two CUDA devices (GPU0 for Qwen3+RVQ autoregression,
|
|
# GPU1 for flow matching and waveform decode) and cannot run on a
|
|
# single-GPU GB10. diffusers is single-device. SGLang-Omni was also
|
|
# rejected because prebuilt aarch64 + sm_121 wheels do not exist and a
|
|
# source build on this box is not worth it for a text-to-audio slot.
|
|
#
|
|
# Pinned to the pre-merge PR commit (huggingface/diffusers#14456). Once
|
|
# that merges, ~/mm3-env can move to a released diffusers and this note
|
|
# can go.
|
|
#
|
|
# footprint_gb: MEASURED — see the boot log. MiniMax's card claims "fits
|
|
# under 24GB VRAM", but a vendor VRAM claim has already understated this
|
|
# box once (FLUX.2-klein: 29GB claimed, 66GB measured on GB10), and the
|
|
# repo ships flowmatching_vae.pth as a 9.8GB fp32 blob alongside a 9.7GB
|
|
# bf16 transformer/, so a load that skips the cast would land far above
|
|
# the claim. Do not estimate this entry — boot it and read RSS.
|
|
# MEASURED on the reference node 2026-08-13, diffusers path, MM3_OFFLOAD=none:
|
|
# idle resident 23,028 MiB (22.5 GB)
|
|
# peak during generation 23,349 MiB (+321 MiB)
|
|
# Generation barely moves it — the AR decode builds no large KV cache — so
|
|
# unlike brain-nemotron this footprint does not scale with request size.
|
|
# MiniMaxs under 24GB claim was accurate here, unlike FLUXs.
|
|
# 26 declared against 23 measured, leaving slack for the allocator.
|
|
footprint_gb: 26
|
|
channel: experimental
|
|
health: "http://localhost:8010/health"
|
|
health_contains: '"ready":true'
|
|
serve:
|
|
kind: diffusers
|
|
port: 8010
|
|
# Request schema deliberately mirrors the old ACE-Step wrapper
|
|
# (prompt/tags/lyrics/duration/seed/audio_format) so anything already
|
|
# calling :8010 survives the engine swap unchanged.
|
|
command: ["~/mm3-env/bin/python", "~/mm3_server.py"]
|
|
env:
|
|
MM3_MODEL_PATH: "~/models/MiniMax-Music3"
|
|
MM3_API_PORT: "8010"
|
|
MM3_API_HOST: 0.0.0.0
|
|
# "none" keeps the whole pipeline resident; we have the headroom and
|
|
# CPU offload costs real latency on an already-slow autoregressive
|
|
# path. Set to "model" only if the scene budget gets tight.
|
|
MM3_OFFLOAD: none
|
|
|
|
eye:
|
|
name: "NVIDIA Cosmos 3 Edge 4B (BF16) — vision reasoner"
|
|
# The "eye": camera frames in, text reasoning out. Cosmos3-Edge is an
|
|
# omnimodal world model (Mixture-of-Transformers: an autoregressive
|
|
# reasoner tower + a diffusion generator tower). Plain vLLM serves the
|
|
# REASONER tower only, which is exactly the half we want; the diffusion
|
|
# side (video/action generation) needs the vllm-omni container and is not
|
|
# wired up here.
|
|
#
|
|
# BF16 ONLY. NVIDIA explicitly lists FP4/FP8/FP16 as untested/unsupported,
|
|
# so unlike every other model in this registry it does not quantize down.
|
|
# Measured 2026-08-12: 12.1 GB RSS at gpu-memory-utilization 0.12.
|
|
#
|
|
# Context is 4096 — that is the model's own text limit, not a tuning
|
|
# choice. It is an eye, not a brain: send it a frame and a short question.
|
|
#
|
|
# ⚠️ IMAGES ONLY (`video: 0`). Video is disabled deliberately, not by
|
|
# oversight — see the launcher note below.
|
|
footprint_gb: 14
|
|
channel: experimental
|
|
health: "http://localhost:8014/v1/models"
|
|
health_contains: "cosmos3-edge"
|
|
serve:
|
|
kind: vllm
|
|
port: 8014
|
|
# NOT a bare `vllm serve` — it goes through ~/cosmos_eye_launch.py, which
|
|
# applies one import shim before handing off to the vLLM CLI. vLLM 0.27.1
|
|
# requires transformers>=5.5.3 but its own Cosmos processor imports
|
|
# `get_image_size` from a qwen3_vl module that stopped re-exporting it by
|
|
# transformers 5.15.0 — vLLM's two pins are mutually unsatisfiable for
|
|
# this one model. The launcher re-injects the (unchanged, merely moved)
|
|
# function. Read the docstring in that file before touching this.
|
|
#
|
|
# A SECOND incompatibility from the same version skew is NOT shimmed:
|
|
# transformers 5.15 added required `factor`/`temporal_factor` args to
|
|
# Qwen3VLVideoProcessor.resize(), which vLLM's Cosmos video path calls
|
|
# with the old signature. That is a changed contract, not a moved symbol,
|
|
# so video is switched off instead of patched around. Engine startup
|
|
# profiling exercises the video path, so with `video: 1` this model does
|
|
# not boot at all. Still frames are unaffected and are what the webcam
|
|
# eye needs. Revisit when vLLM ships a Cosmos fix.
|
|
command:
|
|
- "~/vllm27-env/bin/python3"
|
|
- "~/cosmos_eye_launch.py"
|
|
- "serve"
|
|
- "~/models/Cosmos3-Edge"
|
|
- "--served-model-name"
|
|
- "eye"
|
|
- "cosmos3-edge"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8014"
|
|
- "--gpu-memory-utilization"
|
|
- "0.12"
|
|
- "--max-model-len"
|
|
- "4096"
|
|
- "--max-num-seqs"
|
|
- "2"
|
|
- "--enforce-eager"
|
|
- "--limit-mm-per-prompt"
|
|
- '{"image": 2, "video": 0}'
|
|
# Thinking OFF by default — the single biggest lever on this model.
|
|
#
|
|
# vLLM 0.27.1 has no reasoning parser for cosmos3_edge, so its
|
|
# chain-of-thought is not split into a `reasoning` field the way the
|
|
# brain's is: it lands in `content` and eats the whole token budget.
|
|
# No prompting trick suppresses it. Measured 2026-08-12 on one webcam
|
|
# frame, thinking on -> off:
|
|
# "describe in two sentences" 200 tok / 7.0s -> 49 tok / 2.1s
|
|
# "is there a person? yes/no" 39 tok / 1.8s -> 3 tok / 0.2s
|
|
# The 200-token case never even reached its answer; it was still
|
|
# narrating when the cap hit. Off, it answers cleanly and correctly.
|
|
#
|
|
# The chat template exposes `enable_thinking` (default True), so a
|
|
# caller that wants deliberation can pass chat_template_kwargs per
|
|
# request. For a camera loop you almost never want it.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
|
|
image:
|
|
name: "FLUX.2-klein-9B (BF16)"
|
|
# BFL's own card claims ~29GB VRAM, but an independent benchmark that
|
|
# actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident
|
|
# (95s/image) — trust the on-hardware number for admission control until
|
|
# we re-measure it ourselves. Non-commercial license (BFL FLUX license);
|
|
# fine for personal use, not for anything monetized. Requires accepting
|
|
# the license on HF and `hf auth login` before the weights can download.
|
|
footprint_gb: 66
|
|
channel: stable
|
|
health: "http://localhost:8007/health"
|
|
serve:
|
|
kind: diffusers
|
|
port: 8007
|
|
command: ["~/flux-env/bin/python", "~/flux_server.py"]
|
|
env:
|
|
FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B"
|
|
FLUX_API_PORT: "8007"
|
|
|
|
ocr:
|
|
name: "Unlimited-OCR 3B (NVFP4) — whole-document OCR"
|
|
# baidu/Unlimited-OCR, NVFP4 via sahilchachra. Added 2026-08-07.
|
|
#
|
|
# Chosen over GLM-OCR and DeepSeek-OCR on the axis that actually matters here.
|
|
# GLM-OCR scores higher PER PAGE (94.6 vs 93.9 OmniDocBench v1.6), but
|
|
# Unlimited-OCR parses dozens of pages in ONE forward pass, and on whole-
|
|
# document parsing it beats DeepSeek-OCR by 6.2 points (93.23 vs 87.01 on
|
|
# v1.5). For document-shaped work the single-pass number is the real one.
|
|
# MIT licensed.
|
|
#
|
|
# Arch UnlimitedOCRForCausalLM is natively supported by vLLM 0.26 — no custom
|
|
# code path, same `kind: vllm` treatment as brain/embed.
|
|
#
|
|
# 2.8GB of weights, but 13 declared. gpu-memory-utilization is a fraction of
|
|
# TOTAL memory (121.7GB), not of what is free, so 0.10 => ~12.2GB and the
|
|
# declaration has to match or the governor's arithmetic lies.
|
|
#
|
|
# 0.06 was tried first and failed with "No available memory for the cache
|
|
# blocks": 7.3GB could not cover 2.8GB of weights plus the 1024px vision
|
|
# tower's activation peak, leaving zero for KV. This is a lodger in the primary scene
|
|
# scene, not a tenant, but it is not free either.
|
|
footprint_gb: 13
|
|
channel: stable
|
|
health: "http://localhost:8013/v1/models"
|
|
health_contains: "ocr"
|
|
# ⚠️ REQUEST CONTRACT — this model has NO chat template. Callers must build the
|
|
# prompt by hand or they get an empty completion (1 token, no error):
|
|
# 1. text part FIRST, image part second
|
|
# 2. the text MUST begin with the literal "<image>" — e.g.
|
|
# "<image>document parsing." (single page)
|
|
# "<image>Multi page parsing." (multi-page / PDF)
|
|
# 3. extra_body: {"skip_special_tokens": false,
|
|
# "vllm_xargs": {"ngram_size": 35, "window_size": 128}}
|
|
# window_size 1024 for multi-page/PDF.
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm-env/bin/vllm"
|
|
port: 8013
|
|
weights: "~/models/Unlimited-OCR-NVFP4"
|
|
served_name:
|
|
- ocr
|
|
- unlimited-ocr
|
|
args:
|
|
max-model-len: 32768 # model max_position_embeddings
|
|
gpu-memory-utilization: 0.10
|
|
max-num-seqs: 2
|
|
enforce-eager: true
|
|
limit-mm-per-prompt: '{"image": 4, "video": 0}' # 8 inflates the vision activation peak
|
|
trust-remote-code: true
|
|
# MANDATORY per the official vLLM recipe — without it the model loops on
|
|
# coordinate tokens. Not an optimisation; it is load-bearing.
|
|
logits_processors: "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor"
|
|
# Both caches must be OFF for this model (recipe requirement).
|
|
no-enable-prefix-caching: true
|
|
mm-processor-cache-gb: 0
|
|
|
|
embed:
|
|
name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval"
|
|
# 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim,
|
|
# 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by
|
|
# the gpu-memory-utilization reservation, not the tiny weights.
|
|
footprint_gb: 2
|
|
channel: stable
|
|
health: "http://localhost:8012/v1/models"
|
|
health_contains: "embed"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm-env/bin/vllm"
|
|
port: 8012
|
|
weights: "~/models/Nemotron-3-Embed-1B-NVFP4"
|
|
served_name:
|
|
- embed
|
|
- nemotron-embed
|
|
args:
|
|
runner: pooling
|
|
max-model-len: 32768
|
|
gpu-memory-utilization: 0.05
|
|
enforce-eager: true
|
|
|
|
brain-ds4:
|
|
name: "DeepSeek V4 Flash 0731 (EXL3 3.0bpw, REAP K216) via SparkInfer"
|
|
# PARKED 2026-08-06 — registered, never pulled. Earmarked for the second
|
|
# Spark, where it can have the whole box. Nothing is downloaded yet: first
|
|
# activation pays ~1h of download + coalesce (see DISK below).
|
|
#
|
|
# A WHOLE-BOX tenant, not a peer of `brain`. The EXL3/Trellis weights alone
|
|
# load at 95.39 GiB (upstream VALIDATION.md, "Weight load"), and no context
|
|
# setting moves that number — only KV shrinks. On this 121.6 GB box that
|
|
# leaves ~20 GB for the OS and every other model, which is why the
|
|
# `deepseek` Scene carries only `ears` and drops voice/embed/music.
|
|
#
|
|
# NOT faster than `brain`. Upstream's own gate is five 512-token coding
|
|
# trials at concurrency 1: min 34.30, median 38.12 tok/s — and their own
|
|
# >=35 tok/s floor is still marked FAILING in VALIDATION.md. Measured on
|
|
# THIS box 2026-08-06, `brain` (Qwen3.6 35B-A3B NVFP4 + 2-token MTP) did
|
|
# 62.49 / 64.37 / 64.38 tok/s on the same shape of prompt. So this is
|
|
# ~1.7x SLOWER for ~2.7x the memory. Choose it for what a DeepSeek V4
|
|
# checkpoint says, never for how fast it says it.
|
|
#
|
|
# Also unevaluated for quality: the HF card and VALIDATION.md publish only
|
|
# functional gates (does it generate, does JSON schema parse) — there is no
|
|
# benchmark score anywhere for the 3.0bpw REAP build vs the fp8 original.
|
|
#
|
|
# ALIAS GAP — this does NOT answer to `local-moe`. The image's
|
|
# entrypoint takes a single SERVED_MODEL_NAME and vLLM 404s unknown model
|
|
# names, so unlike `brain-fable` (llama.cpp, which ignores the requested
|
|
# name) the downstream agents will NOT transparently fall through to this. Point
|
|
# them at `deepseek-v4-flash-0731-spark` explicitly, or confirm the in-image
|
|
# serve-ds4-flash.sh forwards a space-separated list to --served-model-name
|
|
# before relying on an alias.
|
|
#
|
|
# Runs as a pinned third-party container, so `kind: docker` with an explicit
|
|
# `command` — proc.rs only builds argv for `kind: vllm`. Foreground
|
|
# `docker run --rm` so the supervisor gets a real PID to track and the
|
|
# container is removed when it stops; upstream's compose.yaml uses
|
|
# `restart: unless-stopped` + `-d`, which would fight the Governor. The
|
|
# server binds 8000 inside the image (hardcoded in serve-ds4-flash.sh), so
|
|
# it is published as 8001 to take `brain`'s port and keep the gateway's
|
|
# `--upstream 127.0.0.1:8001` working unchanged, exactly like brain-fable.
|
|
#
|
|
# DISK, first start only: downloads ~99.5 GiB of source weights to
|
|
# data/source, then coalesces the 172 rank-sliced `exl3-layer-*-tp4-rankN`
|
|
# files to TP1 (~83 GiB of NEW files; the 5 `carried-*.safetensors` are
|
|
# hard-linked by --link-carried, so those cost nothing) and builds a 3 GB
|
|
# K64 draft. Peak ~186 GiB against 207 GiB free. It fits, and nothing else
|
|
# lands on this box while it does. data/source is reusable-but-dead weight
|
|
# afterward; the entrypoint re-enters the download branch only when
|
|
# tp1/rank-sliced-tp1-manifest.json is missing.
|
|
footprint_gb: 101
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "deepseek-v4-flash"
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-ds4"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=16g"
|
|
- "-p"
|
|
- "8001:8000"
|
|
- "-v"
|
|
- "/srv/models/deepseek-v4-flash-spark/data:/models"
|
|
- "-v"
|
|
- "/srv/models/deepseek-v4-flash-spark/cache:/cache"
|
|
- "-e"
|
|
- "MODEL_REPO=0xSero/deepseek-v4-flash-0731-spark"
|
|
- "-e"
|
|
- "MODEL_REVISION=22f28d32b9b29b4352eaa380ff8c2c170b2847ab"
|
|
# 65536, not upstream's 262144. We do not need 262k, and it is the only
|
|
# lever that exists: it drops KV from the measured 8.01 GiB to ~2 GiB.
|
|
# The 95.39 GiB of weights is unaffected.
|
|
- "-e"
|
|
- "MAX_MODEL_LEN=65536"
|
|
- "-e"
|
|
- "MAX_NUM_SEQS=4"
|
|
- "-e"
|
|
- "MAX_NUM_BATCHED_TOKENS=8224"
|
|
- "-e"
|
|
- "MODE=dspark"
|
|
# Fixed K5, dynamic depth OFF. This is upstream's default and it is
|
|
# load-bearing, not cosmetic: dynamic K1-K5 was faster on some prompts
|
|
# but tripped a grammar-mask cardinality assertion on strict JSON
|
|
# schema. That is the "xgrammar issue" being reported on X. Leave at 0.
|
|
- "-e"
|
|
- "DSPARK_TOKENS=5"
|
|
- "-e"
|
|
- "DSPARK_CAPACITY=0"
|
|
- "-e"
|
|
- "DSPARK_DYNAMIC_DRAFT_DEPTH=0"
|
|
- "-e"
|
|
- "DSPARK_DYNAMIC_DRAFT_DEPTH_WINDOW=8"
|
|
- "-e"
|
|
- "DSPARK_DRAFT_EXPERTS=64"
|
|
- "-e"
|
|
- "DSPARK_STRUCTURED_EXPERTS_PER_CATEGORY=32"
|
|
- "-e"
|
|
- "VLLM_USE_B12X_WO_PROJECTION=1"
|
|
# 0.83 of 121.6 GB = ~101 GB, matching footprint_gb above. Upstream ships
|
|
# 0.9465, which reserves about one hundred fifteen GB and would leave nothing for `ears`.
|
|
- "-e"
|
|
- "GPU_MEMORY_UTILIZATION=0.83"
|
|
- "-e"
|
|
- "VERIFY_MODEL_CHECKSUMS=1"
|
|
- "-e"
|
|
- "SERVED_MODEL_NAME=deepseek-v4-flash-0731-spark"
|
|
- "ghcr.io/0xsero/deepseek-v4-flash-0731-spark-sparkinfer@sha256:2e077489a83a0360952828051fe7f7a32c1801e5ce8436d85f7267583d614ff4"
|
|
|
|
# Alternatives, ready to register when wanted:
|
|
# image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark
|
|
# image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering
|
|
|
|
brain-nemotron-256k:
|
|
# The 256k sibling of `brain-nemotron`. A REPLACEMENT for it and for `brain`,
|
|
# never a peer — all three take :8001.
|
|
#
|
|
# Everything in the `brain-nemotron` entry above applies: the vllm27-env
|
|
# requirement, the flag set, why thinking is off, why marlin is pinned. Read
|
|
# that entry first; only the context window and its cost differ here.
|
|
#
|
|
# footprint: MEASURED on the reference node at exactly this configuration —
|
|
# max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS
|
|
#
|
|
# 0.38 * 121.7 = 46.2 GB of pool against 57.0 GB observed, so **10.8 GB sits
|
|
# outside the fraction** — the mamba conv/SSM state is allocated per-sequence
|
|
# and gpu-memory-utilization does not bound it. At 64k the same gap is only
|
|
# 1.4 GB. That is why this is declared at 57 and not at 46, and why raising
|
|
# the window again means re-measuring rather than scaling the old number.
|
|
#
|
|
# This configuration is what OOM-killed `ears` on 2026-08-11, when it was
|
|
# declared at 46 GB from a paper estimate. 57 is the measurement that
|
|
# replaced the estimate. Do not lower it without booting and reading RSS.
|
|
name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 256k)"
|
|
footprint_gb: 57
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# ⚠️ Matches on the 256k-only alias, NOT the shared HF name. Compute decides
|
|
# "is it running" by probing the port and matching this string, so an entry
|
|
# that shares :8001 AND the generic name with `brain-nemotron` makes BOTH
|
|
# look alive at once — observed 2026-08-16 as committed 128 GB across five
|
|
# models and headroom 0. Same trick `brain-fable` uses, same reason.
|
|
health_contains: "brain-nemotron-256k"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm27-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
# The bare HF name is deliberately NOT advertised here: it is what
|
|
# `brain-nemotron` (64k) matches health on, and sharing it makes that
|
|
# entry look alive whenever this one runs.
|
|
- "brain-nemotron-256k"
|
|
args:
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
max-model-len: 262144
|
|
kv-cache-dtype: fp8
|
|
# Explicit, so it wins over the fraction Compute would derive from
|
|
# footprint_gb (57/121.7 = 0.468, which would reserve 11 GB more pool
|
|
# than this configuration was measured with).
|
|
gpu-memory-utilization: 0.38
|
|
max-num-seqs: 4
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-prefix-caching: true
|
|
speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}'
|
|
mamba-backend: flashinfer
|
|
mamba-cache-mode: align
|
|
moe-backend: marlin
|
|
reasoning-parser: nemotron_v3
|
|
tool-call-parser: qwen3_coder
|
|
enable-auto-tool-choice: true
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
voice-vox:
|
|
# VoxCPM 2 as an alternative narrator to `voice` (Chatterbox). A REPLACEMENT,
|
|
# never a peer — they are both a TTS on this box and there is no reason to
|
|
# hold two. Chatterbox is the one to keep if latency matters: measured
|
|
# 2026-08-16 under a warm protocol, Chatterbox turbo is RTF 0.28 here and
|
|
# VoxCPM is 1.35, i.e. VoxCPM is SLOWER THAN REAL TIME on this box. Pick it
|
|
# for what it is better at — 48 kHz output, 30 languages, and it will read
|
|
# 1800 characters in one call where Chatterbox drops 98% of the text.
|
|
#
|
|
# footprint: MEASURED on the reference node. Unlike a vLLM model this one does NOT
|
|
# pre-allocate — the audio decoder allocates in proportion to the LENGTH OF
|
|
# THE AUDIO, so the footprint is a function of the longest call you allow:
|
|
# 450 chars -> 7.8 GB 3600 chars -> 17.7 GB
|
|
# 7200 chars -> 23.7 GB 14400 chars -> 49.2 GB
|
|
# The service pins VOX_MAX_CHARS=3600 and serialises renders behind a lock,
|
|
# so 17.7 GB is the real ceiling. 20 declared leaves slack for the allocator.
|
|
# ⚠️ Raising VOX_MAX_CHARS without raising this is how you OOM-kill `ears`.
|
|
#
|
|
# 3600 is also where quality stops, not just memory: at 7200 characters the
|
|
# model silently stopped reading (27.8 chars/sec against ~15 for real
|
|
# speech). The cap returns 413 rather than truncating in silence.
|
|
#
|
|
# Defaults are cfg_value 3.0 / inference_timesteps 20, chosen by listening on
|
|
# 2026-08-16. Higher guidance is also CHEAPER here — at cfg 2.0 the model
|
|
# produces bad cases and retries them (rtf 5.2), at cfg 3.0-4.0 with 20 steps
|
|
# it settles at rtf ~2.15 on the Radeon. Ear and clock agree for once.
|
|
name: "VoxCPM 2 (48 kHz, 30 languages, voice cloning)"
|
|
footprint_gb: 20
|
|
channel: experimental
|
|
# A listening socket is not a narrator: the server warms one render before it
|
|
# flips model_loaded, because the first call after a load is ~3x slower.
|
|
health: "http://localhost:8096/health"
|
|
health_contains: '"model_loaded":true'
|
|
supervision:
|
|
restart: always
|
|
check_interval_sec: 30
|
|
failure_threshold: 3
|
|
backoff_sec: 60
|
|
max_backoff_sec: 900
|
|
# Weights are 4.7 GB and it warms a render before reporting healthy.
|
|
startup_timeout_sec: 300
|
|
serve:
|
|
kind: python
|
|
port: 8096
|
|
command: ["~/vox-env/bin/python", "~/vox_api.py"]
|
|
env:
|
|
VOX_PORT: "8096"
|
|
VOX_MAX_CHARS: "3600"
|
|
VOX_STEPS: "20"
|
|
VOX_CFG: "3.0"
|
|
VOX_VOICES: "/var/lib/lumbridge/vox-voices.json"
|
|
|
|
brain-nemotron-128k:
|
|
# The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`.
|
|
# A REPLACEMENT for both — all three take :8001. Read the 64k entry for the
|
|
# vllm27-env requirement and the flag rationale; only the window differs.
|
|
#
|
|
# MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps`
|
|
# (util 0.32). The declared 45 was interpolated from the 64k/256k points and
|
|
# was ~5 GB conservative; corrected to 41. The overhead over the pool is the
|
|
# per-sequence mamba conv/SSM state, which gpu-memory-utilization does not
|
|
# bound and which grows with the window.
|
|
#
|
|
# ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from
|
|
# RSS, which does NOT measure GPU memory on this unified-memory box — the
|
|
# brain reads 8 GB RSS against 40 GB actual. They are not comparable to
|
|
# this number and should be re-measured with nvidia-smi before being used.
|
|
name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k)"
|
|
footprint_gb: 41
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# 128k-only alias, so Compute can tell this apart from its 64k and 256k
|
|
# siblings, which share the port and the generic served names.
|
|
# ⚠️ The trailing quote is load-bearing. `health_contains` is a raw substring
|
|
# test against the /v1/models body (governor.rs: response.contains(marker)),
|
|
# and "brain-nemotron-128k" is a PREFIX of "brain-nemotron-128k-seqs16".
|
|
# Without the closing quote this entry matched the seqs16 server, the Governor
|
|
# reported a phantom sixth model as [ext], committed 131 GB against a 100 GB
|
|
# budget, and refused `scene activate stock` with "already serving but is not
|
|
# identity-owned by Compute". Matching the JSON string terminator makes the
|
|
# two ids distinguishable. Found 2026-08-19.
|
|
health_contains: 'brain-nemotron-128k"'
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm27-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
# The bare HF name is deliberately NOT advertised here: it is what
|
|
# `brain-nemotron` (64k) matches health on, and sharing it makes that
|
|
# entry look alive whenever this one runs.
|
|
- "brain-nemotron-128k"
|
|
args:
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
max-model-len: 131072
|
|
kv-cache-dtype: fp8
|
|
# Explicit, so it wins over the fraction Compute derives from footprint.
|
|
gpu-memory-utilization: 0.32
|
|
max-num-seqs: 4
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-prefix-caching: true
|
|
speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}'
|
|
mamba-backend: flashinfer
|
|
mamba-cache-mode: align
|
|
moe-backend: marlin
|
|
reasoning-parser: nemotron_v3
|
|
tool-call-parser: qwen3_coder
|
|
enable-auto-tool-choice: true
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
brain-nemotron-128k-seqs16:
|
|
name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k, seqs 16)"
|
|
# EXPERIMENT, 2026-08-19. Identical to brain-nemotron-128k in every respect
|
|
# except max-num-seqs: 16 instead of 4. It exists to answer whether the
|
|
# "ceiling at four" measured 2026-08-16 is this box or is this flag. That
|
|
# sweep ran 8 client lanes against a cap of 4, so its 8-lane level was
|
|
# queueing rather than concurrency, and the finding is confounded by its
|
|
# own config. Peak occupancy never exceeded 4 in its own recording.
|
|
#
|
|
# The cap is NOT forced by memory. This model logged 3,503,197 tokens of KV
|
|
# cache at util 0.32 and reported "Maximum concurrency for 131,072 tokens
|
|
# per request: 26.73x". The 4 was inherited from `brain`, where it existed
|
|
# to stop Qwen building 3.2M tokens of KV on a single-user box; that reason
|
|
# does not transfer to a hybrid mamba model whose window is cheap.
|
|
#
|
|
# ⚠️ footprint_gb below is INHERITED FROM THE 4-SEQ ENTRY, NOT MEASURED.
|
|
# Raising the cap adds per-sequence mamba conv/SSM state, allocated OUTSIDE
|
|
# the fraction gpu-memory-utilization bounds. Re-measure with
|
|
# `nvidia-smi --query-compute-apps` and correct this line. Under-declaring
|
|
# this exact model is what OOM-killed `ears` on 2026-08-11.
|
|
# The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`.
|
|
# A REPLACEMENT for both — all three take :8001. Read the 64k entry for the
|
|
# vllm27-env requirement and the flag rationale; only the window differs.
|
|
#
|
|
# MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps`
|
|
# (util 0.32). The declared 45 was interpolated from the 64k/256k points and
|
|
# was ~5 GB conservative; corrected to 41. The overhead over the pool is the
|
|
# per-sequence mamba conv/SSM state, which gpu-memory-utilization does not
|
|
# bound and which grows with the window.
|
|
#
|
|
# ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from
|
|
# RSS, which does NOT measure GPU memory on this unified-memory box — the
|
|
# brain reads 8 GB RSS against 40 GB actual. They are not comparable to
|
|
# this number and should be re-measured with nvidia-smi before being used.
|
|
footprint_gb: 41
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# 128k-only alias, so Compute can tell this apart from its 64k and 256k
|
|
# siblings, which share the port and the generic served names.
|
|
health_contains: 'brain-nemotron-128k-seqs16"'
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm27-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
# The bare HF name is deliberately NOT advertised here: it is what
|
|
# `brain-nemotron` (64k) matches health on, and sharing it makes that
|
|
# entry look alive whenever this one runs.
|
|
- "brain-nemotron-128k-seqs16"
|
|
args:
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
max-model-len: 131072
|
|
kv-cache-dtype: fp8
|
|
# Explicit, so it wins over the fraction Compute derives from footprint.
|
|
gpu-memory-utilization: 0.32
|
|
max-num-seqs: 16
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-prefix-caching: true
|
|
speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}'
|
|
mamba-backend: flashinfer
|
|
mamba-cache-mode: align
|
|
moe-backend: marlin
|
|
reasoning-parser: nemotron_v3
|
|
tool-call-parser: qwen3_coder
|
|
enable-auto-tool-choice: true
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
brain-qwen38:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, VL)"
|
|
footprint_gb: 56
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
- "-p"
|
|
- "8001:30000"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
- "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "30000"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "auto"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
- "--mem-fraction-static"
|
|
- "0.46"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.59"
|
|
- "--max-running-requests"
|
|
- "8"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
- "--speculative-algorithm"
|
|
- "EAGLE"
|
|
- "--speculative-num-steps"
|
|
- "3"
|
|
- "--speculative-eagle-topk"
|
|
- "1"
|
|
- "--speculative-num-draft-tokens"
|
|
- "4"
|
|
|
|
brain-qwen38-mia:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, MiaAI-Lab tuning, VL)"
|
|
footprint_gb: 56
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-mia"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-mia"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
# --network host removes the docker-proxy userland hop; SGLang then binds
|
|
# 8001 directly (see --port below).
|
|
- "--network=host"
|
|
# GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency
|
|
# cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs
|
|
# on the 2.8GHz little cores.
|
|
- "--cpuset-cpus=5-9,15-19"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
- "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-mia"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8001"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "fp8_e4m3"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
- "--mem-fraction-static"
|
|
- "0.46"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.21"
|
|
- "--mamba-ssm-dtype"
|
|
- "bfloat16"
|
|
- "--mamba-radix-cache-strategy"
|
|
- "extra_buffer_lazy"
|
|
- "--max-mamba-cache-size"
|
|
- "40"
|
|
- "--sampling-defaults"
|
|
- "model"
|
|
- "--max-running-requests"
|
|
- "10"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
- "--speculative-algorithm"
|
|
- "EAGLE"
|
|
- "--speculative-num-steps"
|
|
- "3"
|
|
- "--speculative-eagle-topk"
|
|
- "1"
|
|
- "--speculative-num-draft-tokens"
|
|
- "4"
|
|
|
|
brain-qwen38-dspark:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4, SGLang DSpark, MiaAI-Lab tuning, VL)"
|
|
footprint_gb: 56
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-dspark"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-dspark"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
# --network host removes the docker-proxy userland hop; SGLang then binds
|
|
# 8001 directly (see --port below).
|
|
- "--network=host"
|
|
# GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency
|
|
# cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs
|
|
# on the 2.8GHz little cores.
|
|
- "--cpuset-cpus=5-9,15-19"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
- "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-dspark"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8001"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "fp8_e4m3"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
- "--mem-fraction-static"
|
|
- "0.46"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.21"
|
|
- "--mamba-ssm-dtype"
|
|
- "bfloat16"
|
|
- "--mamba-radix-cache-strategy"
|
|
- "extra_buffer_lazy"
|
|
- "--max-mamba-cache-size"
|
|
- "40"
|
|
- "--sampling-defaults"
|
|
- "model"
|
|
- "--max-running-requests"
|
|
- "10"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
# DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the
|
|
# published GB10 config from hasso5703/dgx-spark-qwen38) except
|
|
# --mem-fraction-static, which the Governor budget owns.
|
|
- "--speculative-algorithm"
|
|
- "DSPARK"
|
|
- "--speculative-draft-model-path"
|
|
- "/draft-dspark"
|
|
- "--speculative-dspark-block-size"
|
|
- "7"
|
|
- "--speculative-draft-model-quantization"
|
|
- "unquant"
|
|
- "--speculative-num-draft-tokens"
|
|
- "8"
|
|
|
|
# Decode-loop options we were missing on EVERY previous profile:
|
|
# torch.compile was off entirely, and continuous-decode-steps was 1.
|
|
- "--enable-torch-compile"
|
|
- "--torch-compile-max-bs"
|
|
- "4"
|
|
- "--cuda-graph-max-bs-decode"
|
|
- "4"
|
|
- "--num-continuous-decode-steps"
|
|
- "2"
|
|
|
|
brain-qwen38-dspark-solo:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4, SGLang DSpark, 0.90 pool, BENCHMARK ONLY)"
|
|
footprint_gb: 110
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-dspark-solo"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-dspark-solo"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
# --network host removes the docker-proxy userland hop; SGLang then binds
|
|
# 8001 directly (see --port below).
|
|
- "--network=host"
|
|
# GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency
|
|
# cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs
|
|
# on the 2.8GHz little cores.
|
|
- "--cpuset-cpus=5-9,15-19"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
- "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-dspark-solo"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8001"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "fp8_e4m3"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
# Mia's value. ~110 GB of 121.7 — the brain owns the whole box.
|
|
- "--mem-fraction-static"
|
|
- "0.90"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.21"
|
|
- "--mamba-ssm-dtype"
|
|
- "bfloat16"
|
|
- "--mamba-radix-cache-strategy"
|
|
- "extra_buffer_lazy"
|
|
- "--max-mamba-cache-size"
|
|
- "40"
|
|
- "--sampling-defaults"
|
|
- "model"
|
|
- "--max-running-requests"
|
|
- "10"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
# DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the
|
|
# published GB10 config from hasso5703/dgx-spark-qwen38) except
|
|
# --mem-fraction-static, which the Governor budget owns.
|
|
- "--speculative-algorithm"
|
|
- "DSPARK"
|
|
- "--speculative-draft-model-path"
|
|
- "/draft-dspark"
|
|
- "--speculative-dspark-block-size"
|
|
- "7"
|
|
- "--speculative-draft-model-quantization"
|
|
- "unquant"
|
|
- "--speculative-num-draft-tokens"
|
|
- "8"
|
|
# Decode-loop options we were missing on EVERY previous profile:
|
|
# torch.compile was off entirely, and continuous-decode-steps was 1.
|
|
- "--enable-torch-compile"
|
|
- "--torch-compile-max-bs"
|
|
- "4"
|
|
- "--cuda-graph-max-bs-decode"
|
|
- "4"
|
|
- "--num-continuous-decode-steps"
|
|
- "2"
|
|
|
|
brain-qwen38-mem:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, big pool, VL)"
|
|
footprint_gb: 75
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-mem"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-mem"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
- "-p"
|
|
- "8001:30000"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
- "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-mem"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "30000"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "auto"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
# 0.62 to match brain-qwen38-dflash-mem exactly — this entry exists only
|
|
# to tell "DFlash2 is better" apart from "a bigger KV pool is better".
|
|
- "--mem-fraction-static"
|
|
- "0.62"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.59"
|
|
- "--max-running-requests"
|
|
- "8"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
- "--speculative-algorithm"
|
|
- "EAGLE"
|
|
- "--speculative-num-steps"
|
|
- "3"
|
|
- "--speculative-eagle-topk"
|
|
- "1"
|
|
- "--speculative-num-draft-tokens"
|
|
- "4"
|
|
|
|
brain-qwen38-dflash:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# The runtime profile is pinned to the measured 0.46 pool and a C4 ceiling.
|
|
# Telemetry must show sustained queueing or a throughput win before either
|
|
# value changes again.
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, VL)"
|
|
footprint_gb: 56
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-dflash"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-dflash"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
- "-p"
|
|
- "8001:30000"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
# Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but
|
|
# has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker
|
|
# with "Cannot find model module". This image is that same digest plus the
|
|
# 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark
|
|
# patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4
|
|
# quantized-lm_head fix, needed because our checkpoint quantizes lm_head).
|
|
# Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull
|
|
# and to keep the A/B differing only in the DFlash2 code.
|
|
# Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh
|
|
- "lmsysorg/sglang:qwen38-27b-dflash2-local"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-dflash"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "30000"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "auto"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
- "--mem-fraction-static"
|
|
- "0.46"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.59"
|
|
# C4 is the measured useful ceiling for the solo Spark workload. Compute
|
|
# records both running and queued requests before this is tuned again.
|
|
- "--max-running-requests"
|
|
- "4"
|
|
# Runtime-native metrics are the source of truth for tokens, queueing,
|
|
# latency, cache behaviour, and multimodal encoder use. Labels are
|
|
# deliberately bounded; job ids never become Prometheus labels.
|
|
- "--enable-metrics"
|
|
- "--enable-cache-report"
|
|
- "--tokenizer-metrics-allowed-custom-labels"
|
|
- "client"
|
|
- "agent"
|
|
- "workload"
|
|
- "--uvicorn-access-log-exclude-prefixes"
|
|
- "/metrics"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
# DFlash2 instead of EAGLE. The stock image advertises DFLASH but lacks
|
|
# DFlash2DraftModel, so the pinned local image carries the verified
|
|
# upstream overlay and NVFP4 lm_head fix described above.
|
|
#
|
|
# WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class:
|
|
# probe EAGLE/MTP DSpark DFlash2
|
|
# code 34.5 51.5 50.9
|
|
# long essay 24.1 18.3 25.4
|
|
# short chat 21.0 23.2 66.6
|
|
# DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats
|
|
# MTP on essays, and is 3x on short chat. The downstream agents do short
|
|
# tool-calling turns, so short chat is the regime that matters here.
|
|
#
|
|
# NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: the
|
|
# live C1-C4 A/B showed no useful gain, while 0.46 leaves roughly 55 GB
|
|
# available for page cache and safe transient work.
|
|
- "--speculative-algorithm"
|
|
- "DFLASH"
|
|
- "--speculative-draft-model-path"
|
|
- "/draft"
|
|
- "--mamba-radix-cache-strategy"
|
|
- "extra_buffer"
|
|
# 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4).
|
|
- "--speculative-num-draft-tokens"
|
|
- "8"
|
|
|
|
brain-qwen38-dflash-mem:
|
|
# Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker.
|
|
# A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001.
|
|
#
|
|
# WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family
|
|
# (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration
|
|
# and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark
|
|
# cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head
|
|
# on THIS box class, same checkpoint, same harness:
|
|
#
|
|
# tok/s dedicated c1 c1(1024->256) c2 c4 c8
|
|
# vLLM MTP K3 27.83 19.24 32.00 34.61 82.89
|
|
# SGLang EAGLE 25.62 27.65 43.74 74.31 123.90
|
|
#
|
|
# vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins
|
|
# short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do
|
|
# short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM
|
|
# 29.12) — if a workload is long thinking-on monologues, use vLLM instead.
|
|
#
|
|
# ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM
|
|
# (config.json carries a live vision_config; language_model_only: false) and
|
|
# SGLang serves the vision tower in-process. The `qwen38` scene therefore
|
|
# drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this
|
|
# checkpoint is NOT verified by anyone — that is a second reason for SGLang.
|
|
#
|
|
# ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted
|
|
# the hardlinked body shards; a tree with only shard 4 loads and produces
|
|
# garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17.
|
|
#
|
|
# ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing —
|
|
# this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto`
|
|
# honours it. Do not "fix" auto to something else without re-running the canary.
|
|
#
|
|
# ⚠️ NO PREFIX CACHING and no `<think>` special tokens in the vLLM path. On
|
|
# SGLang the chat template does emit them, so `--reasoning-parser qwen3` is
|
|
# correct HERE and wrong on vLLM. The two runtimes genuinely differ.
|
|
#
|
|
# MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB),
|
|
# not of what is free, and SGLang reserves it as a STATIC pool at boot. So
|
|
# this model must start FIRST (the scene uses `order: listed`), exactly like
|
|
# `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56.
|
|
# Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100.
|
|
# At 62 the margin pushed the total to 105 and the Governor refused the
|
|
# admit. The margin is NOT optional — budget for it up front.
|
|
#
|
|
# ⚠️ A too-small pool fails at BOOT, not under load, and the error names the
|
|
# wrong first remedy. At 0.28 it died with
|
|
# "Not enough GPU memory for hybrid (mamba/linear-attention) state cache.
|
|
# Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)"
|
|
# i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise
|
|
# --mem-fraction-static; do not start shaving --speculative-num-draft-tokens.
|
|
#
|
|
# Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours:
|
|
# --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget)
|
|
# and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins).
|
|
# VERIFIED LIVE 2026-08-17, first boot:
|
|
# - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed)
|
|
# - VISION WORKS. A generated test image came back with both shapes and
|
|
# the exact string read off it. This is the empirical basis for dropping
|
|
# `eye` from the scene, not an assumption.
|
|
# - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER
|
|
# than the 0.23-0.52 published for this checkpoint.
|
|
# - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage.
|
|
# Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to
|
|
# take as the pool fills). Do NOT cut the declaration to the observed
|
|
# 40 — under-declaring a growing pool is how `ears` got OOM-killed.
|
|
#
|
|
# ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`,
|
|
# whose entry warns about exactly this), SGLang does NOT 404 a request whose
|
|
# `model` field it does not recognise — a request for `brain` was served
|
|
# normally. So the gateway and the downstream agents need no alias and no change.
|
|
#
|
|
# ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on
|
|
# 2026-08-17 ran against 2-3 concurrent live requests from the agents, which
|
|
# find :8001 the moment it is healthy. Client-side single-stream read 18.6
|
|
# tok/s think-off / 15.2 think-on, but server-side `gen throughput` was
|
|
# 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a
|
|
# ceiling. To get a real c1 number, quiesce the agents first.
|
|
name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, big pool, VL)"
|
|
footprint_gb: 75
|
|
channel: experimental
|
|
health: "http://localhost:8001/v1/models"
|
|
# Anchored on the closing quote: /v1/models returns {"id":"<name>"}, and a bare
|
|
# "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form
|
|
# made this entry report healthy whenever the dflash variant was serving :8001.
|
|
# The Governor then double-counted both as running (147 GB committed) and blocked
|
|
# scene switches with "is serving but is not identity-owned by Compute".
|
|
health_contains: 'brain-qwen38-dflash-mem"'
|
|
serve:
|
|
kind: docker
|
|
port: 8001
|
|
command:
|
|
- "docker"
|
|
- "run"
|
|
- "--rm"
|
|
- "--init"
|
|
- "--name=brain-qwen38-dflash-mem"
|
|
- "--gpus=all"
|
|
- "--ipc=host"
|
|
- "--shm-size=32g"
|
|
- "-p"
|
|
- "8001:30000"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro"
|
|
- "-v"
|
|
- "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro"
|
|
# Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build
|
|
# and the cookbook numbers do not transfer to it.
|
|
# Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but
|
|
# has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker
|
|
# with "Cannot find model module". This image is that same digest plus the
|
|
# 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark
|
|
# patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4
|
|
# quantized-lm_head fix, needed because our checkpoint quantizes lm_head).
|
|
# Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull
|
|
# and to keep the A/B differing only in the DFlash2 code.
|
|
# Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh
|
|
- "lmsysorg/sglang:qwen38-27b-dflash2-local"
|
|
- "sglang"
|
|
- "serve"
|
|
- "--trust-remote-code"
|
|
- "--model-path"
|
|
- "/model"
|
|
- "--served-model-name"
|
|
- "brain-qwen38-dflash-mem"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "30000"
|
|
# trtllm_mha is SM100-only; flashinfer is the GB10 path.
|
|
- "--attention-backend"
|
|
- "flashinfer"
|
|
- "--kv-cache-dtype"
|
|
- "auto"
|
|
- "--chunked-prefill-size"
|
|
- "8192"
|
|
- "--max-prefill-tokens"
|
|
- "8192"
|
|
- "--context-length"
|
|
- "262144"
|
|
# 0.62 (~75 GB) instead of 0.46. See this entry's docstring: the point is
|
|
# to separate "DFlash2 is not better" from "DFlash2 was starved". Mia's
|
|
# profile wants 0.90; the Governor budget will not allow that with ears
|
|
# and voice resident.
|
|
- "--mem-fraction-static"
|
|
- "0.62"
|
|
- "--disable-prefill-cuda-graph"
|
|
- "--reasoning-parser"
|
|
- "qwen3"
|
|
# THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack.
|
|
# Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77)
|
|
# sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came
|
|
# back "length" and content was EMPTY - so news/squawk narration silently
|
|
# produced nothing at all. Every brain-nemotron* entry in this registry already
|
|
# carries the equivalent default-chat-template-kwargs; this one was missing it.
|
|
# Per-request chat_template_kwargs still takes precedence, so an agent that
|
|
# wants thinking simply asks for it.
|
|
# CANARY: a max_tokens:80 request must return NON-EMPTY content.
|
|
- "--default-chat-template-kwargs"
|
|
- '{"enable_thinking": false}'
|
|
- "--tool-call-parser"
|
|
- "qwen3_coder"
|
|
- "--mamba-full-memory-ratio"
|
|
- "4.59"
|
|
- "--max-running-requests"
|
|
- "8"
|
|
# EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second
|
|
# download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and
|
|
# measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62).
|
|
# DFlash2 instead of EAGLE. Our pinned image already lists DFLASH as a
|
|
# builtin (`sglang serve --help`: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM,
|
|
# DFLASH, DSPARK), so unlike MiaAI-Lab's recipe we need NO derived image —
|
|
# only the drafter (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB, at /draft).
|
|
#
|
|
# WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class:
|
|
# probe EAGLE/MTP DSpark DFlash2
|
|
# code 34.5 51.5 50.9
|
|
# long essay 24.1 18.3 25.4
|
|
# short chat 21.0 23.2 66.6
|
|
# DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats
|
|
# MTP on essays, and is 3x on short chat. The downstream agents do short
|
|
# tool-calling turns, so short chat is the regime that matters here.
|
|
#
|
|
# NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: that
|
|
# is ~110 GB and assumes the whole box. Lumbridge runs ears+voice+voice-vox
|
|
# alongside, so we keep 0.46 and let the Governor hold the budget.
|
|
- "--speculative-algorithm"
|
|
- "DFLASH"
|
|
- "--speculative-draft-model-path"
|
|
- "/draft"
|
|
- "--mamba-radix-cache-strategy"
|
|
- "extra_buffer"
|
|
# 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4).
|
|
- "--speculative-num-draft-tokens"
|
|
- "8"
|