Karti Tripathi
91a47fb42c
ci / rust (push) Successful in 2m26s
Governed compute for unified-memory AI hardware — the machines where CPU and GPU share one pool and there is no separate VRAM allocation to bounce off. Over-commit that pool and the box thrashes and wedges, SSH and ping included, before the OOM killer gets a turn. Compute does not run inference. It supervises the servers that do: - Admission control. A model starts only if committed + requested + margin fits the budget. The refusal is the feature. - A 1 Hz watchdog on MemAvailable that stops the newest model before thrash. - Scenes: named sets of models activated as one transactional unit, with pre-flight validation and rollback to the previously active Scene on failure. Scenes reference model ids, never weight paths or commands, so a Scene obtained from elsewhere cannot introduce code. - Process ownership bound to (boot_id, pid, start_time_ticks, pgid == pid), so a reused PID can never be group-killed. - A protocol-transparent TCP gateway, so clients keep one address while model runtimes move behind it. - An MCP server, so agents drive the node as tools rather than as a CLI. One binary, six direct dependencies, no async runtime outside the MCP surface. Published from the internal monorepo with a fresh history. The private development tree keeps its own history; nothing here carries it.
180 lines
6.3 KiB
YAML
180 lines
6.3 KiB
YAML
apiVersion: lumbridge/v1
|
|
kind: Registry
|
|
version: 1
|
|
|
|
# Local to this box. Ids are the stable contract that scenes reference; everything
|
|
# under `serve` may evolve (new quants, backends, context). Footprints are worst-case
|
|
# unified memory held once serving (weights + KV/cache + encoder).
|
|
|
|
models:
|
|
|
|
ears:
|
|
name: "Nemotron 3.5 ASR streaming (0.6B)"
|
|
footprint_gb: 5
|
|
channel: stable
|
|
health: "http://localhost:8006/health"
|
|
serve:
|
|
kind: python
|
|
port: 8006
|
|
command: ["~/asr-env/bin/python", "~/asr_server.py"]
|
|
env:
|
|
ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b"
|
|
|
|
brain:
|
|
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
|
|
footprint_gb: 66 # 26GB weights + ~30GB KV for 64k ctx (fp8)
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/vllm-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
served_name: # stable aliases so downstream agents survive a weight swap
|
|
- brain
|
|
- "local-moe"
|
|
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
|
- "unsloth/Qwen3.6-27B-NVFP4"
|
|
args:
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.55
|
|
enforce-eager: true
|
|
max-num-batched-tokens: 4096
|
|
limit-mm-per-prompt: '{"image": 0, "video": 0}'
|
|
enable-auto-tool-choice: true
|
|
tool-call-parser: qwen3_coder
|
|
reasoning-parser: qwen3
|
|
moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
|
|
voice:
|
|
name: "Chatterbox voice cloning (TTS)"
|
|
footprint_gb: 4
|
|
channel: stable
|
|
health: "http://localhost:8095/health"
|
|
serve:
|
|
kind: uvicorn
|
|
port: 8095
|
|
command:
|
|
- "~/cbx-env/bin/uvicorn"
|
|
- "voice_api:app"
|
|
- "--host"
|
|
- "0.0.0.0"
|
|
- "--port"
|
|
- "8095"
|
|
- "--app-dir"
|
|
- "~/chatterbox"
|
|
|
|
brain-laguna:
|
|
name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)"
|
|
# 71 GB main weights + draft, runtime/JIT overhead, and a conservative
|
|
# 64k FP8 KV allowance. This is a replacement for `brain`, never a peer.
|
|
footprint_gb: 86
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "poolside/Laguna-S-2.1-NVFP4"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/laguna-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Laguna-S-2.1-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
- "poolside/Laguna-S-2.1-NVFP4"
|
|
args:
|
|
default-chat-template-kwargs: '{"enable_thinking": false}'
|
|
speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}'
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.70
|
|
max-num-seqs: 8
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
enable-auto-tool-choice: true
|
|
tool-call-parser: poolside_v1
|
|
reasoning-parser: poolside_v1
|
|
override-generation-config: '{"temperature":0.7,"top_p":0.95}'
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
MAX_JOBS: "4"
|
|
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
|
|
|
brain-gemma:
|
|
name: "Gemma 4 31B-IT (NVFP4)"
|
|
# 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation
|
|
# headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window
|
|
# attention (10 of 60 layers are full-attention, rest capped at a 1024-token
|
|
# window) means its real KV cost at 64k context is far lighter than brain/
|
|
# brain-laguna's dense-attention KV — this cap is a conservative first pass,
|
|
# not yet confirmed by an actual load on the reference node.
|
|
footprint_gb: 44
|
|
channel: stable
|
|
health: "http://localhost:8001/v1/models"
|
|
health_contains: "nvidia/Gemma-4-31B-IT-NVFP4"
|
|
serve:
|
|
kind: vllm
|
|
executable: "~/gemma-env/bin/vllm"
|
|
port: 8001
|
|
weights: "~/models/Gemma-4-31B-IT-NVFP4"
|
|
served_name:
|
|
- brain
|
|
- "local-moe"
|
|
- "nvidia/Gemma-4-31B-IT-NVFP4"
|
|
args:
|
|
max-model-len: 65536
|
|
kv-cache-dtype: fp8
|
|
gpu-memory-utilization: 0.35
|
|
quantization: modelopt
|
|
max-num-seqs: 8
|
|
max-num-batched-tokens: 4096
|
|
enforce-eager: true
|
|
# Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM
|
|
# as of this writing — verify before relying on function-calling.
|
|
env:
|
|
CUTE_DSL_ARCH: sm_121a
|
|
|
|
music:
|
|
name: "ACE-Step 1.5 XL 4B sft (music generation)"
|
|
footprint_gb: 24 # measured ~21GB resident live on the reference node; 24 declared for safety
|
|
channel: stable
|
|
health: "http://localhost:8010/health"
|
|
serve:
|
|
kind: exec
|
|
port: 8010
|
|
# 4B XL sft DiT + 1.7B LM (pt backend). Custom FastAPI wrapper (POST /generate -> WAV).
|
|
# Runs as a systemd --user unit on the node today; this is the equivalent launch.
|
|
command: ["~/ACE-Step-1.5/.venv/bin/python", "~/ace_step_server.py"]
|
|
env:
|
|
ACESTEP_CONFIG_PATH: acestep-v15-xl-sft
|
|
ACESTEP_LM_MODEL_PATH: acestep-5Hz-lm-1.7B
|
|
ACESTEP_LM_BACKEND: pt
|
|
ACESTEP_API_PORT: "8010"
|
|
ACESTEP_API_HOST: 0.0.0.0
|
|
|
|
image:
|
|
name: "FLUX.2-klein-9B (BF16)"
|
|
# BFL's own card claims ~29GB VRAM, but an independent benchmark that
|
|
# actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident
|
|
# (95s/image) — trust the on-hardware number for admission control until
|
|
# we re-measure it ourselves. Non-commercial license (BFL FLUX license);
|
|
# fine for personal use, not for anything monetized. Requires accepting
|
|
# the license on HF and `hf auth login` before the weights can download.
|
|
footprint_gb: 66
|
|
channel: stable
|
|
health: "http://localhost:8007/health"
|
|
serve:
|
|
kind: diffusers
|
|
port: 8007
|
|
command: ["~/flux-env/bin/python", "~/flux_server.py"]
|
|
env:
|
|
FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B"
|
|
FLUX_API_PORT: "8007"
|
|
|
|
# Alternatives, ready to register when wanted:
|
|
# image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark
|
|
# image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering
|