apiVersion: lumbridge/v1 kind: Registry version: 1 # Local to this box. Ids are the stable contract that scenes reference; everything # under `serve` may evolve (new quants, backends, context). Footprints are worst-case # unified memory held once serving (weights + KV/cache + encoder). models: ears: name: "Nemotron 3.5 ASR streaming (0.6B)" # MEASURED 2026-08-16: 9.9 GB via `nvidia-smi --query-compute-apps`, was # declared 5. This was a 2x UNDER-declaration — the same class of error # that OOM-killed this very process on 2026-08-11, pointing the other way. # The old 5 came from RSS (~1.9-2.1 GB), which does not measure GPU memory # on this unified-memory box. Measure with nvidia-smi, never RSS. # # It is FIXED, not leaking: identical MiB at idle, during and after 59 s of # audio in one request. `ears` is position-limited, not memory-limited — # it refuses past 6m39s of audio (HTTP 500 from max_position_embeddings) # long before it could grow. Declared at 10 with slack. footprint_gb: 10 channel: stable health: "http://localhost:8006/health" # A listening socket is not a transcriber. The server binds :8006 before # the checkpoint and Silero VAD are loaded (~5-14s), so match on the model # name the way `voice` matches on model_loaded. health_contains: '"model":"nemotron-3.5-asr-streaming-0.6b"' # Compute is the ONLY supervisor for this process as of 2026-08-12. # # It used to be double-managed: this entry AND a user systemd unit # (nemotron-asr.service, Restart=always, enabled at default.target) ran the # exact same command on the same port. systemd won every race, so whenever # it restarted the ASR — a deploy, a crash, the OOM-kill on 2026-08-11 — # the pid recorded here went stale and every `scene activate` failed with # "'ears' is already serving but is not identity-owned by Compute". # # The unit is now stopped and disabled (`systemctl --user disable --now # nemotron-asr.service`). This supervision block replaces what it provided: # Restart=always / RestartSec=5 becomes restart: always with backoff, which # is strictly better here — a plain 5s restart loop against a memory- # pressure failure is how you turn one OOM into a thrash. # # If you ever want systemd to own it again, delete this block AND remove # `ears` from every scene, or you are right back where you started. # # Tuned 2026-08-12 by kill -9'ing the process and timing recovery. At # check_interval 30 / threshold 3 it came back in ~60s, which is a long # time to be deaf in a voice loop. At 10/2 it is ~25s (2 missed probes + # ~14s to load the checkpoint and Silero VAD). The backoff is what keeps # this from becoming systemd's 5s hot loop under memory pressure, so it # stays generous even though detection is now fast. supervision: restart: always check_interval_sec: 10 failure_threshold: 2 backoff_sec: 30 max_backoff_sec: 600 startup_timeout_sec: 120 serve: kind: python port: 8006 command: ["~/asr-env/bin/python", "~/asr_server.py"] env: ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b" brain: name: "Qwen3.6 35B-A3B MoE (NVFP4)" # Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by # gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built # 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping # max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost. # # MTP (multi-token prediction) is ON as of 2026-08-02 and measured # 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4 # (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is # declared for headroom. # # IMPORTANT — this model MUST start before ears/voice. Its KV-cache # profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable # dips to ~7GB against a 3GB watchdog floor even when it loads alone. That # is why the primary scene uses `order: listed` with brain first. Three earlier # attempts with ears+voice already resident were watchdog-killed. footprint_gb: 40 channel: stable health: "http://localhost:8001/v1/models" health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" serve: kind: vllm executable: "~/vllm-env/bin/vllm" port: 8001 weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast" served_name: # stable aliases so downstream agents survive a weight swap - brain - "local-moe" - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" - "unsloth/Qwen3.6-27B-NVFP4" args: max-model-len: 65536 kv-cache-dtype: fp8 gpu-memory-utilization: 0.25 # 0.40 + MTP tripped the watchdog max-num-seqs: 4 # bounds KV; the single biggest memory lever enforce-eager: true max-num-batched-tokens: 4096 async-scheduling: true # The checkpoint already ships 19 MTP tensors (mtp.fc.weight, # mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0 # deprecates the method name qwen3_5_mtp -> "mtp" and resolves # Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok # per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM # warns >1 re-runs the same MTP layer and can lower per-position # acceptance, so the gain is not linear — after any change check # vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"} # and go back to 1 if the second position accepts poorly. speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}' limit-mm-per-prompt: '{"image": 2, "video": 0}' enable-auto-tool-choice: true tool-call-parser: qwen3_coder reasoning-parser: qwen3 # NO explicit moe-backend — let vLLM's oracle choose. Pinning one is a # version trap. # # History: under vLLM 0.25 this HAD to be flashinfer_cutlass, because the # Unsloth b12x recipe supports only *quantized* NVFP4 MoE and hard-failed on # the unquantized MTP draft layers ("Expected one of ['triton', # 'flashinfer_trtllm','flashinfer_cutlass','aiter']"). # # Under vLLM 0.26 that same pin is FATAL — the NvFp4 oracle rejects it on # GB10: "NvFp4 MoE backend 'FLASHINFER_CUTLASS' does not support the # deployment configuration since kernel does not support current device # cuda." Rebuilding the reference node on 2026-08-07 hit exactly this, because # vllm-setup.sh asks for `vllm>=0.25.0` and a clean install now gets 0.26.0. # # Left alone, 0.26 picks VLLM_CUTLASS from ['FLASHINFER_TRTLLM', # 'FLASHINFER_CUTEDSL','FLASHINFER_CUTEDSL_BATCHED','FLASHINFER_CUTLASS', # 'VLLM_CUTLASS','MARLIN','HUMMING','EMULATION'] and serves correctly with # MTP. Verified on the 4TB FE: coherent completions, spec-decode metrics live. # # ⚠️ VLLM_CUTLASS JIT-compiles kernels and needs `ninja` on PATH. It is NOT # pulled in by build-essential/cmake — install ninja-build. Without it the # engine dies with FileNotFoundError: 'ninja' AFTER passing the backend # check, which reads like an unrelated bug. env: CUTE_DSL_ARCH: sm_121a voice: # Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on # 2026-08-02 — and the name is misleading. Turbo is not a degraded fast # variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a # 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` + # s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`. # The vocoder is what produces the waveform, so that is what you hear. # It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade. # # 2026-08-16: the FULL CHECKPOINT IS SUNSET at the user's request. This # entry is turbo-only. It reverts the 2026-08-13 flip to full, which had # doubled unqualified /voice latency (RTF ~0.50 vs ~0.26) against the blind # A/B recorded above, and it stops paying the ~3.6 GB that keeping full # resident for the /voice A/B Lab cost. # # ⚠️ CONSEQUENCE: the /voice A/B Lab can no longer serve `variant: "full"`. # `_model_for()` in ~/chatterbox/voice_api.py answers a clean HTTP 400 # (`unknown variant 'full'; loaded: ['turbo']`) rather than silently # substituting turbo — an A/B result labelled with the wrong checkpoint # being worse than none. The weights are still on disk at CBX_FULL_DIR, so # this is reversible by putting `full` back in CBX_VARIANTS. name: "Chatterbox TTS — turbo only (voice cloning)" # MEASURED 2026-08-16 turbo-only: 3.2 GB GPU via # `nvidia-smi --query-compute-apps` (2.8 fresh, 3.2 after two renders). # Was 8 with both checkpoints resident. Declared at 5 with slack. # # ⚠️⚠️ THIS NUMBER DOES NOT BOUND WHAT THIS SERVICE COSTS. `voice` LEAKS # HOST RAM, which no GPU measurement sees. On 2026-08-14 it was found # holding 57.1 GB RSS against an 8 GB declaration after ~4 hours, having # squeezed `brain-nemotron` and `music` off the box; restarting it freed # 72 GB. The growth is TIME-based, not request-based — ~8.4 GB/hour while # completely idle — and the GPU allocation stays flat throughout, which is # exactly why footprint_gb looks innocent here. # # That leak was characterised with BOTH checkpoints resident; the leading # suspect was accumulating per-connection state from persistent TCP # connections held open by two downstream clients, never confirmed. # # RE-MEASURED 2026-08-16 turbo-only: NOT REPRODUCING. RSS sampled every # 90 s for 13 min went 2.57 -> 2.65 GB and then sat FLAT at 2.65 across the # last ~6 min of true idle. The old rate would have added ~0.84 GB over # that window. One downstream persistent connection was ESTABLISHED # throughout, so the leading suspect was present and produced no growth, # which points at the dual-checkpoint residency as the actual cause. # ⚠️ 6 min against a 4-hour original observation is NOT a clearance. # Re-check RSS after this has been up overnight before trusting it. # # ⚠️ DO NOT paper over this with a systemd timer or an out-of-band killer. # `voice` is Compute-managed (supervision.restart below); a second # supervisor recreates the double-management failure that broke `ears`. # The real fixes are (a) memory-based supervision in lumbridge-compute, # which today keys on health only and never reads RSS, or (b) finding what # those connections retain. footprint_gb: 5 channel: stable health: "http://localhost:8095/health" # A listening uvicorn socket is not a narrator. Both checkpoints must have # loaded before the Scene or the resident supervisor calls this healthy. health_contains: '"model_loaded":true' # Opt-in and local to this vetted registry entry. Three missed probes avoid # restarting across one busy response; backoff prevents a memory-pressure # failure from becoming a hot restart loop. supervision: restart: always check_interval_sec: 30 failure_threshold: 3 backoff_sec: 60 max_backoff_sec: 900 startup_timeout_sec: 180 serve: kind: uvicorn port: 8095 command: - "~/cbx-env/bin/uvicorn" - "voice_api:app" - "--host" - "0.0.0.0" - "--port" - "8095" - "--app-dir" - "~/chatterbox" env: # Turbo is the only checkpoint loaded, and so is what every unqualified # request gets — i.e. what the downstream caller hears. CBX_VARIANT: turbo CBX_VARIANTS: "turbo" # Local copies, not from_pretrained(): snapshot_download needs the # network and would silently follow upstream if the repo moved. CBX_TURBO_DIR: "/srv/models/chatterbox-turbo" CBX_FULL_DIR: "/srv/models/chatterbox-full" brain-laguna: name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)" # 71 GB main weights + draft, runtime/JIT overhead, and a conservative # 64k FP8 KV allowance. This is a replacement for `brain`, never a peer. footprint_gb: 86 channel: stable health: "http://localhost:8001/v1/models" health_contains: "poolside/Laguna-S-2.1-NVFP4" serve: kind: vllm executable: "~/laguna-env/bin/vllm" port: 8001 weights: "~/models/Laguna-S-2.1-NVFP4" served_name: - brain - "local-moe" - "poolside/Laguna-S-2.1-NVFP4" args: default-chat-template-kwargs: '{"enable_thinking": false}' speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}' max-model-len: 65536 kv-cache-dtype: fp8 gpu-memory-utilization: 0.70 max-num-seqs: 8 max-num-batched-tokens: 4096 enforce-eager: true enable-auto-tool-choice: true tool-call-parser: poolside_v1 reasoning-parser: poolside_v1 override-generation-config: '{"temperature":0.7,"top_p":0.95}' env: CUTE_DSL_ARCH: sm_121a MAX_JOBS: "4" PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" brain-nemotron: name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark)" # A REPLACEMENT for `brain`, never a peer — it takes :8001. Two MoE brains # co-resident does not fit: 68GB is already held by the rest of the primary scene # stack plus brain, and both engines spike during KV profiling. # # Why this model is interesting on THIS box specifically: it is a hybrid # Mamba-2 + MoE, and NVIDIA's own reference single-GPU target for it is # "1x DGX Spark (GB10)". `brain`'s entire config is contorted around KV # cache — max-num-seqs is pinned to 4 purely to stop vLLM building 3.2M # tokens of it. Mamba layers carry recurrent state instead of a growing KV # cache, so context length costs far less memory here. That is what buys # the 1M window NVIDIA advertises on the same 121GB we currently ration # at 64k. # # Weights measured on disk 2026-08-11: 21GB main + 1.3GB DSpark draft. # # ⚠️ REQUIRES vLLM >= 0.27.1 — its own venv, ~/vllm27-env (vllm 0.27.1, # torch 2.13.0+cu132). Do NOT upgrade ~/vllm-env to get this: that env # serves brain, embed and ocr, and the 0.25 -> 0.26 jump already made # brain's old moe-backend pin fatal on GB10 once. The flags below # (--mamba-backend, --mamba-cache-mode, --moe-backend, and the # nemotron_v3 reasoning parser) do not exist in 0.26.0 at all. # # footprint: MEASURED on the reference node, two data points: # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS # max-model-len 65536, gpu-memory-utilization 0.28 -> 35.5 GB RSS # # The first configuration was declared at 46 GB from a paper estimate, ran # 11 GB over it, and the kernel OOM-killed `ears` on the next scene # transition. Do not estimate this entry — boot it and read RSS. # # The overhead above vLLM's pool is NOT a constant: 0.38*121.7 = 46.2 GB # pool vs 57 GB observed (+10.8), but 0.28*121.7 = 34.1 GB pool vs 35.5 GB # observed (+1.4). It scales with max-model-len, because the mamba conv/SSM # state is allocated per-sequence OUTSIDE the fraction that # gpu-memory-utilization bounds. Long context is cheap in the mamba layers # and expensive here — that is the opposite of the intuition the 1M-context # headline gives you. # # 38 declared against 35.5 measured, leaving a little slack for the # allocator. Re-measure after ANY change to context, max-num-seqs, or the # speculative config. footprint_gb: 38 channel: experimental health: "http://localhost:8001/v1/models" health_contains: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" serve: kind: vllm executable: "~/vllm27-env/bin/vllm" port: 8001 weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" served_name: # same aliases as brain, so downstream agents swap blind - brain - "local-moe" - "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" args: # Thinking OFF by default. Measured 2026-08-11: with it on, a trivial # P&L question burned 347 completion tokens of correct reasoning before # the answer, so every eval case with a max_tokens cap truncated # mid-thought and scored as a wrong answer. The model had the right # number ($550) the whole time. `brain` serves with thinking off too, # so leaving this on also made the A/B against it meaningless. # Callers that want it can pass chat_template_kwargs per request. default-chat-template-kwargs: '{"enable_thinking": false}' # 64k, matching `brain`. The window is nearly free in the mamba layers # but NOT in the interleaved attention layers, which hold real KV, and # 256k is what drove this process to 57GB. The 1M window is real but it # is not free on a 121GB box that also carries eyes/ears/voice — raise # this only against measured headroom in `status`. max-model-len: 65536 kv-cache-dtype: fp8 gpu-memory-utilization: 0.28 max-num-seqs: 4 max-num-batched-tokens: 4096 enforce-eager: true enable-prefix-caching: true # DSpark: a semi-autoregressive drafter that proposes a whole block per # forward pass. NVIDIA recommends it over MTP and DFlash for GB10 and # for low-concurrency serving, which is exactly this box. speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' # Mamba-2 specific. `align` trades a little memory for aligned state # layout; both flags are NVIDIA's published GB10 recipe. mamba-backend: flashinfer mamba-cache-mode: align # Pinned deliberately, against the rule in `brain`'s entry that says to # let the oracle choose. NVIDIA names marlin explicitly for GB10 here # because the NVFP4 recipe is W4A16 on the experts, which is marlin's # case. If a future vLLM rejects this pin the way 0.26 rejected # brain's flashinfer_cutlass, drop the line before debugging anything else. moe-backend: marlin reasoning-parser: nemotron_v3 tool-call-parser: qwen3_coder enable-auto-tool-choice: true env: CUTE_DSL_ARCH: sm_121a PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" brain-gemma: name: "Gemma 4 31B-IT (NVFP4)" # 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation # headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window # attention (10 of 60 layers are full-attention, rest capped at a 1024-token # window) means its real KV cost at 64k context is far lighter than brain/ # brain-laguna's dense-attention KV — this cap is a conservative first pass, # not yet confirmed by an actual load on the reference node. footprint_gb: 44 channel: stable health: "http://localhost:8001/v1/models" health_contains: "nvidia/Gemma-4-31B-IT-NVFP4" serve: kind: vllm executable: "~/gemma-env/bin/vllm" port: 8001 weights: "~/models/Gemma-4-31B-IT-NVFP4" served_name: - brain - "local-moe" - "nvidia/Gemma-4-31B-IT-NVFP4" args: max-model-len: 65536 kv-cache-dtype: fp8 gpu-memory-utilization: 0.35 quantization: modelopt max-num-seqs: 8 max-num-batched-tokens: 4096 enforce-eager: true # Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM # as of this writing — verify before relying on function-calling. env: CUTE_DSL_ARCH: sm_121a music: name: "MiniMax Music 3 (BF16) — full-song generation" # Replaced ACE-Step 1.5 XL 4B on 2026-08-13. The ACE-Step entry had been # unbootable for some time: its command pointed at ~/ACE-Step-1.5/.venv, # and neither that venv nor the acestep-v15-xl-sft / acestep-5Hz-lm-1.7B # weights existed anywhere on the reference node. This slot is a repair, not a trade. # # Architecture: 8B Global LLM (Qwen3-8B init) predicting RVQ codebook 0 # frame-by-frame, 0.6B Local LLM for the remaining 7 codebooks, then # 2.4B Flow Matching -> 123M Flow-VAE decoder -> 32kHz 16-bit stereo. # ~11.1B params total. Up to 5 minutes per generation. # # SERVING PATH: diffusers ModularPipeline, NOT the SGLang reference repo. # That repo hardcodes two CUDA devices (GPU0 for Qwen3+RVQ autoregression, # GPU1 for flow matching and waveform decode) and cannot run on a # single-GPU GB10. diffusers is single-device. SGLang-Omni was also # rejected because prebuilt aarch64 + sm_121 wheels do not exist and a # source build on this box is not worth it for a text-to-audio slot. # # Pinned to the pre-merge PR commit (huggingface/diffusers#14456). Once # that merges, ~/mm3-env can move to a released diffusers and this note # can go. # # footprint_gb: MEASURED — see the boot log. MiniMax's card claims "fits # under 24GB VRAM", but a vendor VRAM claim has already understated this # box once (FLUX.2-klein: 29GB claimed, 66GB measured on GB10), and the # repo ships flowmatching_vae.pth as a 9.8GB fp32 blob alongside a 9.7GB # bf16 transformer/, so a load that skips the cast would land far above # the claim. Do not estimate this entry — boot it and read RSS. # MEASURED on the reference node 2026-08-13, diffusers path, MM3_OFFLOAD=none: # idle resident 23,028 MiB (22.5 GB) # peak during generation 23,349 MiB (+321 MiB) # Generation barely moves it — the AR decode builds no large KV cache — so # unlike brain-nemotron this footprint does not scale with request size. # MiniMaxs under 24GB claim was accurate here, unlike FLUXs. # 26 declared against 23 measured, leaving slack for the allocator. footprint_gb: 26 channel: experimental health: "http://localhost:8010/health" health_contains: '"ready":true' serve: kind: diffusers port: 8010 # Request schema deliberately mirrors the old ACE-Step wrapper # (prompt/tags/lyrics/duration/seed/audio_format) so anything already # calling :8010 survives the engine swap unchanged. command: ["~/mm3-env/bin/python", "~/mm3_server.py"] env: MM3_MODEL_PATH: "~/models/MiniMax-Music3" MM3_API_PORT: "8010" MM3_API_HOST: 0.0.0.0 # "none" keeps the whole pipeline resident; we have the headroom and # CPU offload costs real latency on an already-slow autoregressive # path. Set to "model" only if the scene budget gets tight. MM3_OFFLOAD: none eye: name: "NVIDIA Cosmos 3 Edge 4B (BF16) — vision reasoner" # The "eye": camera frames in, text reasoning out. Cosmos3-Edge is an # omnimodal world model (Mixture-of-Transformers: an autoregressive # reasoner tower + a diffusion generator tower). Plain vLLM serves the # REASONER tower only, which is exactly the half we want; the diffusion # side (video/action generation) needs the vllm-omni container and is not # wired up here. # # BF16 ONLY. NVIDIA explicitly lists FP4/FP8/FP16 as untested/unsupported, # so unlike every other model in this registry it does not quantize down. # Measured 2026-08-12: 12.1 GB RSS at gpu-memory-utilization 0.12. # # Context is 4096 — that is the model's own text limit, not a tuning # choice. It is an eye, not a brain: send it a frame and a short question. # # ⚠️ IMAGES ONLY (`video: 0`). Video is disabled deliberately, not by # oversight — see the launcher note below. footprint_gb: 14 channel: experimental health: "http://localhost:8014/v1/models" health_contains: "cosmos3-edge" serve: kind: vllm port: 8014 # NOT a bare `vllm serve` — it goes through ~/cosmos_eye_launch.py, which # applies one import shim before handing off to the vLLM CLI. vLLM 0.27.1 # requires transformers>=5.5.3 but its own Cosmos processor imports # `get_image_size` from a qwen3_vl module that stopped re-exporting it by # transformers 5.15.0 — vLLM's two pins are mutually unsatisfiable for # this one model. The launcher re-injects the (unchanged, merely moved) # function. Read the docstring in that file before touching this. # # A SECOND incompatibility from the same version skew is NOT shimmed: # transformers 5.15 added required `factor`/`temporal_factor` args to # Qwen3VLVideoProcessor.resize(), which vLLM's Cosmos video path calls # with the old signature. That is a changed contract, not a moved symbol, # so video is switched off instead of patched around. Engine startup # profiling exercises the video path, so with `video: 1` this model does # not boot at all. Still frames are unaffected and are what the webcam # eye needs. Revisit when vLLM ships a Cosmos fix. command: - "~/vllm27-env/bin/python3" - "~/cosmos_eye_launch.py" - "serve" - "~/models/Cosmos3-Edge" - "--served-model-name" - "eye" - "cosmos3-edge" - "--host" - "0.0.0.0" - "--port" - "8014" - "--gpu-memory-utilization" - "0.12" - "--max-model-len" - "4096" - "--max-num-seqs" - "2" - "--enforce-eager" - "--limit-mm-per-prompt" - '{"image": 2, "video": 0}' # Thinking OFF by default — the single biggest lever on this model. # # vLLM 0.27.1 has no reasoning parser for cosmos3_edge, so its # chain-of-thought is not split into a `reasoning` field the way the # brain's is: it lands in `content` and eats the whole token budget. # No prompting trick suppresses it. Measured 2026-08-12 on one webcam # frame, thinking on -> off: # "describe in two sentences" 200 tok / 7.0s -> 49 tok / 2.1s # "is there a person? yes/no" 39 tok / 1.8s -> 3 tok / 0.2s # The 200-token case never even reached its answer; it was still # narrating when the cap hit. Off, it answers cleanly and correctly. # # The chat template exposes `enable_thinking` (default True), so a # caller that wants deliberation can pass chat_template_kwargs per # request. For a camera loop you almost never want it. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' env: CUTE_DSL_ARCH: sm_121a image: name: "FLUX.2-klein-9B (BF16)" # BFL's own card claims ~29GB VRAM, but an independent benchmark that # actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident # (95s/image) — trust the on-hardware number for admission control until # we re-measure it ourselves. Non-commercial license (BFL FLUX license); # fine for personal use, not for anything monetized. Requires accepting # the license on HF and `hf auth login` before the weights can download. footprint_gb: 66 channel: stable health: "http://localhost:8007/health" serve: kind: diffusers port: 8007 command: ["~/flux-env/bin/python", "~/flux_server.py"] env: FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B" FLUX_API_PORT: "8007" ocr: name: "Unlimited-OCR 3B (NVFP4) — whole-document OCR" # baidu/Unlimited-OCR, NVFP4 via sahilchachra. Added 2026-08-07. # # Chosen over GLM-OCR and DeepSeek-OCR on the axis that actually matters here. # GLM-OCR scores higher PER PAGE (94.6 vs 93.9 OmniDocBench v1.6), but # Unlimited-OCR parses dozens of pages in ONE forward pass, and on whole- # document parsing it beats DeepSeek-OCR by 6.2 points (93.23 vs 87.01 on # v1.5). For document-shaped work the single-pass number is the real one. # MIT licensed. # # Arch UnlimitedOCRForCausalLM is natively supported by vLLM 0.26 — no custom # code path, same `kind: vllm` treatment as brain/embed. # # 2.8GB of weights, but 13 declared. gpu-memory-utilization is a fraction of # TOTAL memory (121.7GB), not of what is free, so 0.10 => ~12.2GB and the # declaration has to match or the governor's arithmetic lies. # # 0.06 was tried first and failed with "No available memory for the cache # blocks": 7.3GB could not cover 2.8GB of weights plus the 1024px vision # tower's activation peak, leaving zero for KV. This is a lodger in the primary scene # scene, not a tenant, but it is not free either. footprint_gb: 13 channel: stable health: "http://localhost:8013/v1/models" health_contains: "ocr" # ⚠️ REQUEST CONTRACT — this model has NO chat template. Callers must build the # prompt by hand or they get an empty completion (1 token, no error): # 1. text part FIRST, image part second # 2. the text MUST begin with the literal "" — e.g. # "document parsing." (single page) # "Multi page parsing." (multi-page / PDF) # 3. extra_body: {"skip_special_tokens": false, # "vllm_xargs": {"ngram_size": 35, "window_size": 128}} # window_size 1024 for multi-page/PDF. serve: kind: vllm executable: "~/vllm-env/bin/vllm" port: 8013 weights: "~/models/Unlimited-OCR-NVFP4" served_name: - ocr - unlimited-ocr args: max-model-len: 32768 # model max_position_embeddings gpu-memory-utilization: 0.10 max-num-seqs: 2 enforce-eager: true limit-mm-per-prompt: '{"image": 4, "video": 0}' # 8 inflates the vision activation peak trust-remote-code: true # MANDATORY per the official vLLM recipe — without it the model loops on # coordinate tokens. Not an optimisation; it is load-bearing. logits_processors: "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor" # Both caches must be OFF for this model (recipe requirement). no-enable-prefix-caching: true mm-processor-cache-gb: 0 embed: name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval" # 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim, # 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by # the gpu-memory-utilization reservation, not the tiny weights. footprint_gb: 2 channel: stable health: "http://localhost:8012/v1/models" health_contains: "embed" serve: kind: vllm executable: "~/vllm-env/bin/vllm" port: 8012 weights: "~/models/Nemotron-3-Embed-1B-NVFP4" served_name: - embed - nemotron-embed args: runner: pooling max-model-len: 32768 gpu-memory-utilization: 0.05 enforce-eager: true brain-ds4: name: "DeepSeek V4 Flash 0731 (EXL3 3.0bpw, REAP K216) via SparkInfer" # PARKED 2026-08-06 — registered, never pulled. Earmarked for the second # Spark, where it can have the whole box. Nothing is downloaded yet: first # activation pays ~1h of download + coalesce (see DISK below). # # A WHOLE-BOX tenant, not a peer of `brain`. The EXL3/Trellis weights alone # load at 95.39 GiB (upstream VALIDATION.md, "Weight load"), and no context # setting moves that number — only KV shrinks. On this 121.6 GB box that # leaves ~20 GB for the OS and every other model, which is why the # `deepseek` Scene carries only `ears` and drops voice/embed/music. # # NOT faster than `brain`. Upstream's own gate is five 512-token coding # trials at concurrency 1: min 34.30, median 38.12 tok/s — and their own # >=35 tok/s floor is still marked FAILING in VALIDATION.md. Measured on # THIS box 2026-08-06, `brain` (Qwen3.6 35B-A3B NVFP4 + 2-token MTP) did # 62.49 / 64.37 / 64.38 tok/s on the same shape of prompt. So this is # ~1.7x SLOWER for ~2.7x the memory. Choose it for what a DeepSeek V4 # checkpoint says, never for how fast it says it. # # Also unevaluated for quality: the HF card and VALIDATION.md publish only # functional gates (does it generate, does JSON schema parse) — there is no # benchmark score anywhere for the 3.0bpw REAP build vs the fp8 original. # # ALIAS GAP — this does NOT answer to `local-moe`. The image's # entrypoint takes a single SERVED_MODEL_NAME and vLLM 404s unknown model # names, so unlike `brain-fable` (llama.cpp, which ignores the requested # name) the downstream agents will NOT transparently fall through to this. Point # them at `deepseek-v4-flash-0731-spark` explicitly, or confirm the in-image # serve-ds4-flash.sh forwards a space-separated list to --served-model-name # before relying on an alias. # # Runs as a pinned third-party container, so `kind: docker` with an explicit # `command` — proc.rs only builds argv for `kind: vllm`. Foreground # `docker run --rm` so the supervisor gets a real PID to track and the # container is removed when it stops; upstream's compose.yaml uses # `restart: unless-stopped` + `-d`, which would fight the Governor. The # server binds 8000 inside the image (hardcoded in serve-ds4-flash.sh), so # it is published as 8001 to take `brain`'s port and keep the gateway's # `--upstream 127.0.0.1:8001` working unchanged, exactly like brain-fable. # # DISK, first start only: downloads ~99.5 GiB of source weights to # data/source, then coalesces the 172 rank-sliced `exl3-layer-*-tp4-rankN` # files to TP1 (~83 GiB of NEW files; the 5 `carried-*.safetensors` are # hard-linked by --link-carried, so those cost nothing) and builds a 3 GB # K64 draft. Peak ~186 GiB against 207 GiB free. It fits, and nothing else # lands on this box while it does. data/source is reusable-but-dead weight # afterward; the entrypoint re-enters the download branch only when # tp1/rank-sliced-tp1-manifest.json is missing. footprint_gb: 101 channel: experimental health: "http://localhost:8001/v1/models" health_contains: "deepseek-v4-flash" serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-ds4" - "--gpus=all" - "--ipc=host" - "--shm-size=16g" - "-p" - "8001:8000" - "-v" - "/srv/models/deepseek-v4-flash-spark/data:/models" - "-v" - "/srv/models/deepseek-v4-flash-spark/cache:/cache" - "-e" - "MODEL_REPO=0xSero/deepseek-v4-flash-0731-spark" - "-e" - "MODEL_REVISION=22f28d32b9b29b4352eaa380ff8c2c170b2847ab" # 65536, not upstream's 262144. We do not need 262k, and it is the only # lever that exists: it drops KV from the measured 8.01 GiB to ~2 GiB. # The 95.39 GiB of weights is unaffected. - "-e" - "MAX_MODEL_LEN=65536" - "-e" - "MAX_NUM_SEQS=4" - "-e" - "MAX_NUM_BATCHED_TOKENS=8224" - "-e" - "MODE=dspark" # Fixed K5, dynamic depth OFF. This is upstream's default and it is # load-bearing, not cosmetic: dynamic K1-K5 was faster on some prompts # but tripped a grammar-mask cardinality assertion on strict JSON # schema. That is the "xgrammar issue" being reported on X. Leave at 0. - "-e" - "DSPARK_TOKENS=5" - "-e" - "DSPARK_CAPACITY=0" - "-e" - "DSPARK_DYNAMIC_DRAFT_DEPTH=0" - "-e" - "DSPARK_DYNAMIC_DRAFT_DEPTH_WINDOW=8" - "-e" - "DSPARK_DRAFT_EXPERTS=64" - "-e" - "DSPARK_STRUCTURED_EXPERTS_PER_CATEGORY=32" - "-e" - "VLLM_USE_B12X_WO_PROJECTION=1" # 0.83 of 121.6 GB = ~101 GB, matching footprint_gb above. Upstream ships # 0.9465, which reserves about one hundred fifteen GB and would leave nothing for `ears`. - "-e" - "GPU_MEMORY_UTILIZATION=0.83" - "-e" - "VERIFY_MODEL_CHECKSUMS=1" - "-e" - "SERVED_MODEL_NAME=deepseek-v4-flash-0731-spark" - "ghcr.io/0xsero/deepseek-v4-flash-0731-spark-sparkinfer@sha256:2e077489a83a0360952828051fe7f7a32c1801e5ce8436d85f7267583d614ff4" # Alternatives, ready to register when wanted: # image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark # image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering brain-nemotron-256k: # The 256k sibling of `brain-nemotron`. A REPLACEMENT for it and for `brain`, # never a peer — all three take :8001. # # Everything in the `brain-nemotron` entry above applies: the vllm27-env # requirement, the flag set, why thinking is off, why marlin is pinned. Read # that entry first; only the context window and its cost differ here. # # footprint: MEASURED on the reference node at exactly this configuration — # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS # # 0.38 * 121.7 = 46.2 GB of pool against 57.0 GB observed, so **10.8 GB sits # outside the fraction** — the mamba conv/SSM state is allocated per-sequence # and gpu-memory-utilization does not bound it. At 64k the same gap is only # 1.4 GB. That is why this is declared at 57 and not at 46, and why raising # the window again means re-measuring rather than scaling the old number. # # This configuration is what OOM-killed `ears` on 2026-08-11, when it was # declared at 46 GB from a paper estimate. 57 is the measurement that # replaced the estimate. Do not lower it without booting and reading RSS. name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 256k)" footprint_gb: 57 channel: experimental health: "http://localhost:8001/v1/models" # ⚠️ Matches on the 256k-only alias, NOT the shared HF name. Compute decides # "is it running" by probing the port and matching this string, so an entry # that shares :8001 AND the generic name with `brain-nemotron` makes BOTH # look alive at once — observed 2026-08-16 as committed 128 GB across five # models and headroom 0. Same trick `brain-fable` uses, same reason. health_contains: "brain-nemotron-256k" serve: kind: vllm executable: "~/vllm27-env/bin/vllm" port: 8001 weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" served_name: - brain - "local-moe" # The bare HF name is deliberately NOT advertised here: it is what # `brain-nemotron` (64k) matches health on, and sharing it makes that # entry look alive whenever this one runs. - "brain-nemotron-256k" args: default-chat-template-kwargs: '{"enable_thinking": false}' max-model-len: 262144 kv-cache-dtype: fp8 # Explicit, so it wins over the fraction Compute would derive from # footprint_gb (57/121.7 = 0.468, which would reserve 11 GB more pool # than this configuration was measured with). gpu-memory-utilization: 0.38 max-num-seqs: 4 max-num-batched-tokens: 4096 enforce-eager: true enable-prefix-caching: true speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' mamba-backend: flashinfer mamba-cache-mode: align moe-backend: marlin reasoning-parser: nemotron_v3 tool-call-parser: qwen3_coder enable-auto-tool-choice: true env: CUTE_DSL_ARCH: sm_121a PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" voice-vox: # VoxCPM 2 as an alternative narrator to `voice` (Chatterbox). A REPLACEMENT, # never a peer — they are both a TTS on this box and there is no reason to # hold two. Chatterbox is the one to keep if latency matters: measured # 2026-08-16 under a warm protocol, Chatterbox turbo is RTF 0.28 here and # VoxCPM is 1.35, i.e. VoxCPM is SLOWER THAN REAL TIME on this box. Pick it # for what it is better at — 48 kHz output, 30 languages, and it will read # 1800 characters in one call where Chatterbox drops 98% of the text. # # footprint: MEASURED on the reference node. Unlike a vLLM model this one does NOT # pre-allocate — the audio decoder allocates in proportion to the LENGTH OF # THE AUDIO, so the footprint is a function of the longest call you allow: # 450 chars -> 7.8 GB 3600 chars -> 17.7 GB # 7200 chars -> 23.7 GB 14400 chars -> 49.2 GB # The service pins VOX_MAX_CHARS=3600 and serialises renders behind a lock, # so 17.7 GB is the real ceiling. 20 declared leaves slack for the allocator. # ⚠️ Raising VOX_MAX_CHARS without raising this is how you OOM-kill `ears`. # # 3600 is also where quality stops, not just memory: at 7200 characters the # model silently stopped reading (27.8 chars/sec against ~15 for real # speech). The cap returns 413 rather than truncating in silence. # # Defaults are cfg_value 3.0 / inference_timesteps 20, chosen by listening on # 2026-08-16. Higher guidance is also CHEAPER here — at cfg 2.0 the model # produces bad cases and retries them (rtf 5.2), at cfg 3.0-4.0 with 20 steps # it settles at rtf ~2.15 on the Radeon. Ear and clock agree for once. name: "VoxCPM 2 (48 kHz, 30 languages, voice cloning)" footprint_gb: 20 channel: experimental # A listening socket is not a narrator: the server warms one render before it # flips model_loaded, because the first call after a load is ~3x slower. health: "http://localhost:8096/health" health_contains: '"model_loaded":true' supervision: restart: always check_interval_sec: 30 failure_threshold: 3 backoff_sec: 60 max_backoff_sec: 900 # Weights are 4.7 GB and it warms a render before reporting healthy. startup_timeout_sec: 300 serve: kind: python port: 8096 command: ["~/vox-env/bin/python", "~/vox_api.py"] env: VOX_PORT: "8096" VOX_MAX_CHARS: "3600" VOX_STEPS: "20" VOX_CFG: "3.0" VOX_VOICES: "/var/lib/lumbridge/vox-voices.json" brain-nemotron-128k: # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the # vllm27-env requirement and the flag rationale; only the window differs. # # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` # (util 0.32). The declared 45 was interpolated from the 64k/256k points and # was ~5 GB conservative; corrected to 41. The overhead over the pool is the # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not # bound and which grows with the window. # # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from # RSS, which does NOT measure GPU memory on this unified-memory box — the # brain reads 8 GB RSS against 40 GB actual. They are not comparable to # this number and should be re-measured with nvidia-smi before being used. name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k)" footprint_gb: 41 channel: experimental health: "http://localhost:8001/v1/models" # 128k-only alias, so Compute can tell this apart from its 64k and 256k # siblings, which share the port and the generic served names. # ⚠️ The trailing quote is load-bearing. `health_contains` is a raw substring # test against the /v1/models body (governor.rs: response.contains(marker)), # and "brain-nemotron-128k" is a PREFIX of "brain-nemotron-128k-seqs16". # Without the closing quote this entry matched the seqs16 server, the Governor # reported a phantom sixth model as [ext], committed 131 GB against a 100 GB # budget, and refused `scene activate stock` with "already serving but is not # identity-owned by Compute". Matching the JSON string terminator makes the # two ids distinguishable. Found 2026-08-19. health_contains: 'brain-nemotron-128k"' serve: kind: vllm executable: "~/vllm27-env/bin/vllm" port: 8001 weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" served_name: - brain - "local-moe" # The bare HF name is deliberately NOT advertised here: it is what # `brain-nemotron` (64k) matches health on, and sharing it makes that # entry look alive whenever this one runs. - "brain-nemotron-128k" args: default-chat-template-kwargs: '{"enable_thinking": false}' max-model-len: 131072 kv-cache-dtype: fp8 # Explicit, so it wins over the fraction Compute derives from footprint. gpu-memory-utilization: 0.32 max-num-seqs: 4 max-num-batched-tokens: 4096 enforce-eager: true enable-prefix-caching: true speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' mamba-backend: flashinfer mamba-cache-mode: align moe-backend: marlin reasoning-parser: nemotron_v3 tool-call-parser: qwen3_coder enable-auto-tool-choice: true env: CUTE_DSL_ARCH: sm_121a PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" brain-nemotron-128k-seqs16: name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k, seqs 16)" # EXPERIMENT, 2026-08-19. Identical to brain-nemotron-128k in every respect # except max-num-seqs: 16 instead of 4. It exists to answer whether the # "ceiling at four" measured 2026-08-16 is this box or is this flag. That # sweep ran 8 client lanes against a cap of 4, so its 8-lane level was # queueing rather than concurrency, and the finding is confounded by its # own config. Peak occupancy never exceeded 4 in its own recording. # # The cap is NOT forced by memory. This model logged 3,503,197 tokens of KV # cache at util 0.32 and reported "Maximum concurrency for 131,072 tokens # per request: 26.73x". The 4 was inherited from `brain`, where it existed # to stop Qwen building 3.2M tokens of KV on a single-user box; that reason # does not transfer to a hybrid mamba model whose window is cheap. # # ⚠️ footprint_gb below is INHERITED FROM THE 4-SEQ ENTRY, NOT MEASURED. # Raising the cap adds per-sequence mamba conv/SSM state, allocated OUTSIDE # the fraction gpu-memory-utilization bounds. Re-measure with # `nvidia-smi --query-compute-apps` and correct this line. Under-declaring # this exact model is what OOM-killed `ears` on 2026-08-11. # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the # vllm27-env requirement and the flag rationale; only the window differs. # # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` # (util 0.32). The declared 45 was interpolated from the 64k/256k points and # was ~5 GB conservative; corrected to 41. The overhead over the pool is the # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not # bound and which grows with the window. # # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from # RSS, which does NOT measure GPU memory on this unified-memory box — the # brain reads 8 GB RSS against 40 GB actual. They are not comparable to # this number and should be re-measured with nvidia-smi before being used. footprint_gb: 41 channel: experimental health: "http://localhost:8001/v1/models" # 128k-only alias, so Compute can tell this apart from its 64k and 256k # siblings, which share the port and the generic served names. health_contains: 'brain-nemotron-128k-seqs16"' serve: kind: vllm executable: "~/vllm27-env/bin/vllm" port: 8001 weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" served_name: - brain - "local-moe" # The bare HF name is deliberately NOT advertised here: it is what # `brain-nemotron` (64k) matches health on, and sharing it makes that # entry look alive whenever this one runs. - "brain-nemotron-128k-seqs16" args: default-chat-template-kwargs: '{"enable_thinking": false}' max-model-len: 131072 kv-cache-dtype: fp8 # Explicit, so it wins over the fraction Compute derives from footprint. gpu-memory-utilization: 0.32 max-num-seqs: 16 max-num-batched-tokens: 4096 enforce-eager: true enable-prefix-caching: true speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' mamba-backend: flashinfer mamba-cache-mode: align moe-backend: marlin reasoning-parser: nemotron_v3 tool-call-parser: qwen3_coder enable-auto-tool-choice: true env: CUTE_DSL_ARCH: sm_121a PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" brain-qwen38: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, VL)" footprint_gb: 56 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" - "-p" - "8001:30000" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38" - "--host" - "0.0.0.0" - "--port" - "30000" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "auto" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" - "--mem-fraction-static" - "0.46" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.59" - "--max-running-requests" - "8" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). - "--speculative-algorithm" - "EAGLE" - "--speculative-num-steps" - "3" - "--speculative-eagle-topk" - "1" - "--speculative-num-draft-tokens" - "4" brain-qwen38-mia: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, MiaAI-Lab tuning, VL)" footprint_gb: 56 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-mia"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-mia" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" # --network host removes the docker-proxy userland hop; SGLang then binds # 8001 directly (see --port below). - "--network=host" # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs # on the 2.8GHz little cores. - "--cpuset-cpus=5-9,15-19" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-mia" - "--host" - "0.0.0.0" - "--port" - "8001" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "fp8_e4m3" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" - "--mem-fraction-static" - "0.46" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.21" - "--mamba-ssm-dtype" - "bfloat16" - "--mamba-radix-cache-strategy" - "extra_buffer_lazy" - "--max-mamba-cache-size" - "40" - "--sampling-defaults" - "model" - "--max-running-requests" - "10" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). - "--speculative-algorithm" - "EAGLE" - "--speculative-num-steps" - "3" - "--speculative-eagle-topk" - "1" - "--speculative-num-draft-tokens" - "4" brain-qwen38-dspark: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4, SGLang DSpark, MiaAI-Lab tuning, VL)" footprint_gb: 56 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-dspark"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-dspark" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" # --network host removes the docker-proxy userland hop; SGLang then binds # 8001 directly (see --port below). - "--network=host" # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs # on the 2.8GHz little cores. - "--cpuset-cpus=5-9,15-19" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" - "-v" - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-dspark" - "--host" - "0.0.0.0" - "--port" - "8001" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "fp8_e4m3" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" - "--mem-fraction-static" - "0.46" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.21" - "--mamba-ssm-dtype" - "bfloat16" - "--mamba-radix-cache-strategy" - "extra_buffer_lazy" - "--max-mamba-cache-size" - "40" - "--sampling-defaults" - "model" - "--max-running-requests" - "10" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the # published GB10 config from hasso5703/dgx-spark-qwen38) except # --mem-fraction-static, which the Governor budget owns. - "--speculative-algorithm" - "DSPARK" - "--speculative-draft-model-path" - "/draft-dspark" - "--speculative-dspark-block-size" - "7" - "--speculative-draft-model-quantization" - "unquant" - "--speculative-num-draft-tokens" - "8" # Decode-loop options we were missing on EVERY previous profile: # torch.compile was off entirely, and continuous-decode-steps was 1. - "--enable-torch-compile" - "--torch-compile-max-bs" - "4" - "--cuda-graph-max-bs-decode" - "4" - "--num-continuous-decode-steps" - "2" brain-qwen38-dspark-solo: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4, SGLang DSpark, 0.90 pool, BENCHMARK ONLY)" footprint_gb: 110 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-dspark-solo"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-dspark-solo" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" # --network host removes the docker-proxy userland hop; SGLang then binds # 8001 directly (see --port below). - "--network=host" # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs # on the 2.8GHz little cores. - "--cpuset-cpus=5-9,15-19" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" - "-v" - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-dspark-solo" - "--host" - "0.0.0.0" - "--port" - "8001" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "fp8_e4m3" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" # Mia's value. ~110 GB of 121.7 — the brain owns the whole box. - "--mem-fraction-static" - "0.90" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.21" - "--mamba-ssm-dtype" - "bfloat16" - "--mamba-radix-cache-strategy" - "extra_buffer_lazy" - "--max-mamba-cache-size" - "40" - "--sampling-defaults" - "model" - "--max-running-requests" - "10" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the # published GB10 config from hasso5703/dgx-spark-qwen38) except # --mem-fraction-static, which the Governor budget owns. - "--speculative-algorithm" - "DSPARK" - "--speculative-draft-model-path" - "/draft-dspark" - "--speculative-dspark-block-size" - "7" - "--speculative-draft-model-quantization" - "unquant" - "--speculative-num-draft-tokens" - "8" # Decode-loop options we were missing on EVERY previous profile: # torch.compile was off entirely, and continuous-decode-steps was 1. - "--enable-torch-compile" - "--torch-compile-max-bs" - "4" - "--cuda-graph-max-bs-decode" - "4" - "--num-continuous-decode-steps" - "2" brain-qwen38-mem: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, big pool, VL)" footprint_gb: 75 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-mem"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-mem" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" - "-p" - "8001:30000" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-mem" - "--host" - "0.0.0.0" - "--port" - "30000" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "auto" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" # 0.62 to match brain-qwen38-dflash-mem exactly — this entry exists only # to tell "DFlash2 is better" apart from "a bigger KV pool is better". - "--mem-fraction-static" - "0.62" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.59" - "--max-running-requests" - "8" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). - "--speculative-algorithm" - "EAGLE" - "--speculative-num-steps" - "3" - "--speculative-eagle-topk" - "1" - "--speculative-num-draft-tokens" - "4" brain-qwen38-dflash: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # The runtime profile is pinned to the measured 0.46 pool and a C4 ceiling. # Telemetry must show sustained queueing or a throughput win before either # value changes again. # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, VL)" footprint_gb: 56 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-dflash"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-dflash" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" - "-p" - "8001:30000" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" - "-v" - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker # with "Cannot find model module". This image is that same digest plus the # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull # and to keep the A/B differing only in the DFlash2 code. # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh - "lmsysorg/sglang:qwen38-27b-dflash2-local" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-dflash" - "--host" - "0.0.0.0" - "--port" - "30000" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "auto" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" - "--mem-fraction-static" - "0.46" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.59" # C4 is the measured useful ceiling for the solo Spark workload. Compute # records both running and queued requests before this is tuned again. - "--max-running-requests" - "4" # Runtime-native metrics are the source of truth for tokens, queueing, # latency, cache behaviour, and multimodal encoder use. Labels are # deliberately bounded; job ids never become Prometheus labels. - "--enable-metrics" - "--enable-cache-report" - "--tokenizer-metrics-allowed-custom-labels" - "client" - "agent" - "workload" - "--uvicorn-access-log-exclude-prefixes" - "/metrics" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). # DFlash2 instead of EAGLE. The stock image advertises DFLASH but lacks # DFlash2DraftModel, so the pinned local image carries the verified # upstream overlay and NVFP4 lm_head fix described above. # # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: # probe EAGLE/MTP DSpark DFlash2 # code 34.5 51.5 50.9 # long essay 24.1 18.3 25.4 # short chat 21.0 23.2 66.6 # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats # MTP on essays, and is 3x on short chat. The downstream agents do short # tool-calling turns, so short chat is the regime that matters here. # # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: the # live C1-C4 A/B showed no useful gain, while 0.46 leaves roughly 55 GB # available for page cache and safe transient work. - "--speculative-algorithm" - "DFLASH" - "--speculative-draft-model-path" - "/draft" - "--mamba-radix-cache-strategy" - "extra_buffer" # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). - "--speculative-num-draft-tokens" - "8" brain-qwen38-dflash-mem: # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. # # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head # on THIS box class, same checkpoint, same harness: # # tok/s dedicated c1 c1(1024->256) c2 c4 c8 # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 # # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. # # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM # (config.json carries a live vision_config; language_model_only: false) and # SGLang serves the vision tower in-process. The `qwen38` scene therefore # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this # checkpoint is NOT verified by anyone — that is a second reason for SGLang. # # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted # the hardlinked body shards; a tree with only shard 4 loads and produces # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. # # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` # honours it. Do not "fix" auto to something else without re-running the canary. # # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is # correct HERE and wrong on vLLM. The two runtimes genuinely differ. # # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), # not of what is free, and SGLang reserves it as a STATIC pool at boot. So # this model must start FIRST (the scene uses `order: listed`), exactly like # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. # At 62 the margin pushed the total to 105 and the Governor refused the # admit. The margin is NOT optional — budget for it up front. # # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the # wrong first remedy. At 0.28 it died with # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. # # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). # VERIFIED LIVE 2026-08-17, first boot: # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) # - VISION WORKS. A generated test image came back with both shapes and # the exact string read off it. This is the empirical basis for dropping # `eye` from the scene, not an assumption. # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER # than the 0.23-0.52 published for this checkpoint. # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to # take as the pool fills). Do NOT cut the declaration to the observed # 40 — under-declaring a growing pool is how `ears` got OOM-killed. # # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, # whose entry warns about exactly this), SGLang does NOT 404 a request whose # `model` field it does not recognise — a request for `brain` was served # normally. So the gateway and the downstream agents need no alias and no change. # # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which # find :8001 the moment it is healthy. Client-side single-stream read 18.6 # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a # ceiling. To get a real c1 number, quiesce the agents first. name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, big pool, VL)" footprint_gb: 75 channel: experimental health: "http://localhost:8001/v1/models" # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form # made this entry report healthy whenever the dflash variant was serving :8001. # The Governor then double-counted both as running (147 GB committed) and blocked # scene switches with "is serving but is not identity-owned by Compute". health_contains: 'brain-qwen38-dflash-mem"' serve: kind: docker port: 8001 command: - "docker" - "run" - "--rm" - "--init" - "--name=brain-qwen38-dflash-mem" - "--gpus=all" - "--ipc=host" - "--shm-size=32g" - "-p" - "8001:30000" - "-v" - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" - "-v" - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build # and the cookbook numbers do not transfer to it. # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker # with "Cannot find model module". This image is that same digest plus the # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull # and to keep the A/B differing only in the DFlash2 code. # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh - "lmsysorg/sglang:qwen38-27b-dflash2-local" - "sglang" - "serve" - "--trust-remote-code" - "--model-path" - "/model" - "--served-model-name" - "brain-qwen38-dflash-mem" - "--host" - "0.0.0.0" - "--port" - "30000" # trtllm_mha is SM100-only; flashinfer is the GB10 path. - "--attention-backend" - "flashinfer" - "--kv-cache-dtype" - "auto" - "--chunked-prefill-size" - "8192" - "--max-prefill-tokens" - "8192" - "--context-length" - "262144" # 0.62 (~75 GB) instead of 0.46. See this entry's docstring: the point is # to separate "DFlash2 is not better" from "DFlash2 was starved". Mia's # profile wants 0.90; the Governor budget will not allow that with ears # and voice resident. - "--mem-fraction-static" - "0.62" - "--disable-prefill-cuda-graph" - "--reasoning-parser" - "qwen3" # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came # back "length" and content was EMPTY - so news/squawk narration silently # produced nothing at all. Every brain-nemotron* entry in this registry already # carries the equivalent default-chat-template-kwargs; this one was missing it. # Per-request chat_template_kwargs still takes precedence, so an agent that # wants thinking simply asks for it. # CANARY: a max_tokens:80 request must return NON-EMPTY content. - "--default-chat-template-kwargs" - '{"enable_thinking": false}' - "--tool-call-parser" - "qwen3_coder" - "--mamba-full-memory-ratio" - "4.59" - "--max-running-requests" - "8" # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). # DFlash2 instead of EAGLE. Our pinned image already lists DFLASH as a # builtin (`sglang serve --help`: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, # DFLASH, DSPARK), so unlike MiaAI-Lab's recipe we need NO derived image — # only the drafter (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB, at /draft). # # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: # probe EAGLE/MTP DSpark DFlash2 # code 34.5 51.5 50.9 # long essay 24.1 18.3 25.4 # short chat 21.0 23.2 66.6 # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats # MTP on essays, and is 3x on short chat. The downstream agents do short # tool-calling turns, so short chat is the regime that matters here. # # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: that # is ~110 GB and assumes the whole box. Lumbridge runs ears+voice+voice-vox # alongside, so we keep 0.46 and let the Governor hold the budget. - "--speculative-algorithm" - "DFLASH" - "--speculative-draft-model-path" - "/draft" - "--mamba-radix-cache-strategy" - "extra_buffer" # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). - "--speculative-num-draft-tokens" - "8"