From ef9ec1dcd8aa98741e548ee948ab4416157c9a56 Mon Sep 17 00:00:00 2001 From: Karti Tripathi Date: Mon, 31 Aug 2026 16:09:08 -0700 Subject: [PATCH] Lumbridge Compute: telemetry and four-lane operations --- .github/workflows/ci.yml | 18 + .gitignore | 25 + CONTRIBUTING.md | 20 + Cargo.lock | 1012 ++++++++ Cargo.toml | 33 + LICENSE | 202 ++ NOTICE | 16 + README.md | 112 + SECURITY.md | 10 + docs/evals.md | 40 + docs/operations.md | 170 ++ docs/positioning.md | 51 + docs/roadmap.md | 38 + docs/scene-spec.md | 207 ++ docs/telemetry.md | 96 + evals/finance-core.eval.yaml | 47 + evals/performance.eval.yaml | 21 + evals/smoke.eval.yaml | 33 + evals/voice-agent.eval.yaml | 39 + examples/demo/registry/models.yaml | 34 + examples/demo/scenes/one.scene.yaml | 12 + examples/demo/scenes/two.scene.yaml | 12 + registry/models.yaml | 2301 ++++++++++++++++++ scenes/darkroom.scene.yaml | 27 + scenes/deepseek.scene.yaml | 45 + scenes/music.scene.yaml | 35 + scenes/nemotron-seqs16.scene.yaml | 29 + scenes/nemotron.scene.yaml | 36 + scenes/qwen38-dflash-mem.scene.yaml | 25 + scenes/qwen38-dflash-solo.scene.yaml | 30 + scenes/qwen38-dflash.scene.yaml | 76 + scenes/qwen38-dspark-solo.scene.yaml | 25 + scenes/qwen38-dspark.scene.yaml | 34 + scenes/qwen38-mem.scene.yaml | 21 + scenes/qwen38-mia.scene.yaml | 22 + scenes/qwen38.scene.yaml | 82 + scenes/stock.scene.yaml | 54 + scenes/studio.scene.yaml | 22 + scenes/tts-eval.scene.yaml | 26 + scenes/voice-gemma.scene.yaml | 22 + scenes/voice-laguna.scene.yaml | 21 + scenes/voice-qwen.scene.yaml | 26 + src/agent.rs | 364 +++ src/config.rs | 263 ++ src/eval.rs | 494 ++++ src/gateway.rs | 199 ++ src/governor.rs | 223 ++ src/http.rs | 344 +++ src/lifecycle.rs | 915 +++++++ src/main.rs | 561 +++++ src/mcp.rs | 617 +++++ src/mem.rs | 89 + src/proc.rs | 470 ++++ src/telemetry.rs | 1158 +++++++++ systemd/lumbridge-compute-agent-user.service | 19 + systemd/lumbridge-compute-agent.service | 20 + tools/backfill_voice.py | 69 + tools/scrape_vllm_metrics.py | 94 + tools/usage_report.py | 173 ++ 59 files changed, 11279 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 docs/evals.md create mode 100644 docs/operations.md create mode 100644 docs/positioning.md create mode 100644 docs/roadmap.md create mode 100644 docs/scene-spec.md create mode 100644 docs/telemetry.md create mode 100644 evals/finance-core.eval.yaml create mode 100644 evals/performance.eval.yaml create mode 100644 evals/smoke.eval.yaml create mode 100644 evals/voice-agent.eval.yaml create mode 100644 examples/demo/registry/models.yaml create mode 100644 examples/demo/scenes/one.scene.yaml create mode 100644 examples/demo/scenes/two.scene.yaml create mode 100644 registry/models.yaml create mode 100644 scenes/darkroom.scene.yaml create mode 100644 scenes/deepseek.scene.yaml create mode 100644 scenes/music.scene.yaml create mode 100644 scenes/nemotron-seqs16.scene.yaml create mode 100644 scenes/nemotron.scene.yaml create mode 100644 scenes/qwen38-dflash-mem.scene.yaml create mode 100644 scenes/qwen38-dflash-solo.scene.yaml create mode 100644 scenes/qwen38-dflash.scene.yaml create mode 100644 scenes/qwen38-dspark-solo.scene.yaml create mode 100644 scenes/qwen38-dspark.scene.yaml create mode 100644 scenes/qwen38-mem.scene.yaml create mode 100644 scenes/qwen38-mia.scene.yaml create mode 100644 scenes/qwen38.scene.yaml create mode 100644 scenes/stock.scene.yaml create mode 100644 scenes/studio.scene.yaml create mode 100644 scenes/tts-eval.scene.yaml create mode 100644 scenes/voice-gemma.scene.yaml create mode 100644 scenes/voice-laguna.scene.yaml create mode 100644 scenes/voice-qwen.scene.yaml create mode 100644 src/agent.rs create mode 100644 src/config.rs create mode 100644 src/eval.rs create mode 100644 src/gateway.rs create mode 100644 src/governor.rs create mode 100644 src/http.rs create mode 100644 src/lifecycle.rs create mode 100644 src/main.rs create mode 100644 src/mcp.rs create mode 100644 src/mem.rs create mode 100644 src/proc.rs create mode 100644 src/telemetry.rs create mode 100644 systemd/lumbridge-compute-agent-user.service create mode 100644 systemd/lumbridge-compute-agent.service create mode 100644 tools/backfill_voice.py create mode 100755 tools/scrape_vllm_metrics.py create mode 100755 tools/usage_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9ddf67e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: ci + +on: + push: + pull_request: + +jobs: + rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy --all-targets + - run: cargo test + - run: cargo build --release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f90b8f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +/target +.compute/ +eval-results/ +**/*.rs.bk +Cargo.lock.orig +*.log +.DS_Store + +# Hand-made snapshots taken before a registry or scene edit. Local safety +# copies, not history — git already keeps the history. +*.bak +*.bak-* +*.bak.* + +# Secrets. +.env +.env.* +*.pem +*.key +*_rsa +*_ed25519 +id_ed25519* +.ssh/ +*secret* +*credentials* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..500e572 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing to Lumbridge Compute + +Contributions are welcome. Changes land as small, reviewable commits. + +## Development + +```bash +cargo test +cargo fmt --check +cargo clippy --all-targets +cargo build --release +``` + +Changes to the `lumbridge/v1` scene or eval contracts require fixtures, backward- +compatibility notes, and documentation. Never add secrets, private model tokens, +personal voice samples, internal hostnames, or arbitrary commands to shareable +scene/eval manifests. + +Performance changes should include before/after JSON artifacts and the complete +hardware/model/runtime configuration needed to reproduce them. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..733ab32 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1012 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.4", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lumbridge-compute" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "libc", + "rmcp", + "rusqlite", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tokio", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "rmcp" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a15bc53261a9dc37e105df006e4656c598379a8f9581f8950debb130f27a7cf" +dependencies = [ + "base64", + "chrono", + "futures", + "indexmap", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", +] + +[[package]] +name = "rmcp-macros" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a85d45508e9b4ba024fe996c2638799635d75b6dd0ba8f32ccf08f8026f0c780" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 3.0.4", +] + +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "chrono", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.4", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6e7aa33 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "lumbridge-compute" +description = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes." +readme = "README.md" +keywords = ["dgx-spark", "gb10", "llm", "orchestration", "unified-memory"] +categories = ["command-line-utilities"] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +authors = ["Karti Tripathi"] +repository = "https://git.karti.ai/lumbridge-public/compute" +rust-version = "1.88" + +[[bin]] +name = "lumbridge-compute" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1" +anyhow = "1" +libc = "0.2" +rmcp = { version = "3", features = ["server", "transport-io", "macros"] } +schemars = "1" +tokio = { version = "1", features = ["rt", "io-std", "fs", "time"] } +rusqlite = { version = "0.37", features = ["bundled"] } + +# Lives at the workspace root upstream; Cargo ignores [profile] in members. +[profile.release] +strip = true +lto = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..39693df --- /dev/null +++ b/NOTICE @@ -0,0 +1,16 @@ +Lumbridge +Copyright 2026 Karti Tripathi + +This product includes software developed by Karti Tripathi. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7414a67 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +
+ +# Lumbridge Compute + +**The AI compute layer for Lumbridge clusters. Safe models, scenes, and evals on your own hardware.** + +*Run many models on one box — safely — on NVIDIA DGX Spark (GB10), DGX Station (GB300), and RTX.* + +
+ +--- + +Lumbridge Compute is the tool you install the minute you open your DGX Spark. Boxes like the +DGX Spark (GB10), DGX Station (GB300), and RTX workstations share **one memory pool** +between CPU and GPU with little swap. On that hardware, over-committing memory doesn't +fail gracefully — the whole machine thrashes and wedges (SSH and ping included) before +the OOM killer acts. Lumbridge Compute makes that impossible, and turns the box into something you +can load with different workloads on a schedule. + +> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.** +> They're orthogonal; you can run Lumbridge Compute on each node of an exo cluster. + +## Three layers + +| Layer | Analogous to | Job | +|---|---|---| +| **Governor** | kernel / memory cgroup + OOM policy | Nothing starts unless it fits a hard budget; a watchdog kills the newest model *before* the box wedges. | +| **Scenes** | systemd targets | Named, shareable, activatable bundles of models (`studio`, `darkroom`). | +| **lumbridge-compute** (the shell) | the CLI you install | Onboard a fresh box, manage models, activate/schedule scenes. | + +## The Governor (why Lumbridge Compute exists) + +Unified memory means no separate VRAM pool to bounce off. vLLM, diffusers, and friends +will happily reserve past 100% of the shared pool, and the box wedges. The Governor +prevents this with two mechanisms: + +1. **Admission control** — a model starts only if its declared footprint fits the budget + given what's already running, plus a safety margin. +2. **Watchdog** — a 1 Hz thread on `MemAvailable`; if it dips below a critical floor, it + kills the most-recently-started model before thrash. A backstop for a wrong estimate. + +## Scenes + +A **scene** is a named set of models brought up together — the unit you activate, publish, +and schedule. A single unified-memory box can't hold every model at once, so scenes let +you **time-multiplex** it: run a live voice assistant by day, then switch to an overnight +image farm at 3am. One box, the utilization of several. + +Scenes reference **model ids**, never weight paths — so requantizing or upgrading a model +never breaks a published scene. See [`docs/scene-spec.md`](docs/scene-spec.md), and +[`docs/positioning.md`](docs/positioning.md) for how Lumbridge Compute relates to Ollama / llama-swap / exo. + +## Evals + +Lumbridge Compute ships a native, model-server-agnostic evaluation runner. Versioned YAML suites +measure streamed TTFT, client-observed prefill throughput, decode throughput, and +deterministic capability assertions against any OpenAI-compatible endpoint. + +```bash +lumbridge-compute eval ls +lumbridge-compute eval run smoke +lumbridge-compute eval run performance --repeat 5 +lumbridge-compute eval run finance-core +lumbridge-compute usage summary --since 24h # calls, tokens, vision, C0-C4, queueing +lumbridge-compute usage agents --since 7d # bounded client/agent/workload labels +lumbridge-compute usage concurrency --since 30d +``` + +Every run writes a portable JSON artifact for regression tracking and future eval +registries. See [`docs/evals.md`](docs/evals.md). + +## Install + +```bash +# from source (single static binary, no runtime deps) +cargo install --path . +``` + +## Usage + +```bash +lumbridge-compute status # Governor: memory, budget, running set, headroom +lumbridge-compute model ls # registry: footprints + live state + which port +lumbridge-compute scene ls # scenes with total footprint +lumbridge-compute scene show darkroom # models + footprints + admission verdict +lumbridge-compute scene adopt voice-qwen # one-time identity capture for a legacy live node +lumbridge-compute scene resume # desired Scene, then last-known-good fallback +lumbridge-compute gateway # stable streaming endpoint -> local model server +lumbridge-compute agent # resume + gateway + memory + opt-in model supervision + +# planned: +lumbridge-compute model pull # footprint-aware; warns if no scene can hold it +lumbridge-compute scene schedule darkroom 03:00 04:00 # time-multiplex the box +``` + +By default Lumbridge Compute reads `registry/` and `scenes/` from the current directory (override with +`--root ` or `$LUMBRIDGE_COMPUTE_ROOT`). + +## Status + +The Governor, identity-bound process ownership, transactional Scene switching, +persistent desired state, last-known-good recovery, opt-in model supervision, +streaming gateway, eval runner, watchdog, and privacy-safe usage telemetry work +today. Next: the model artifact manager (`pull`), fleet API, and telemetry-driven +Jobs scheduler. See [usage telemetry](docs/telemetry.md), [node operations](docs/operations.md), +and the stable [scene spec](docs/scene-spec.md). + +Lumbridge Compute runs on NVIDIA hardware but is independent and is not affiliated with or endorsed by NVIDIA. + +## License + +Apache-2.0 — see [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..d62d4c0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,10 @@ +# Security policy + +Please report vulnerabilities privately to the maintainers before opening a +public issue. + +Lumbridge Compute treats model registries as trusted local configuration and scenes/eval suites +as potentially untrusted shared data. Shared manifests reference vetted ids and +must never execute embedded shell commands. Downloads must be checksum-verified +before promotion into the trusted registry. Secrets belong in environment or +OS-managed secret stores, never manifests, logs, or result artifacts. diff --git a/docs/evals.md b/docs/evals.md new file mode 100644 index 0000000..1c63f80 --- /dev/null +++ b/docs/evals.md @@ -0,0 +1,40 @@ +# Lumbridge Compute evaluations (`lumbridge/v1`) + +Lumbridge Compute evaluates the model configuration that is actually serving: weights, +quantization, context, runtime, parsers, and speculative decoder. A model name +without its serving configuration is not a reproducible benchmark target. + +```bash +lumbridge-compute eval ls +lumbridge-compute eval run smoke +lumbridge-compute eval run performance --model brain --repeat 5 +lumbridge-compute eval run finance-core --base-url http://your-node:8001/v1 +``` + +Suites live in `evals/*.eval.yaml`. Each case has a stable id, prompt, category, +generation limit, and deterministic assertions. Runs produce append-only JSON in +`eval-results/` with raw outputs and per-sample metrics. + +## Metrics + +- **TTFT**: wall time until the first streamed content or reasoning token. +- **Prefill tok/s (approximate)**: API-reported prompt tokens divided by TTFT. + This is client-observed and includes queueing/scheduling; server-native prefill + metrics should be added as a separate source rather than conflated with it. +- **Decode tok/s**: completion tokens divided by time after the first token. +- **Score**: share of samples satisfying every declared assertion. + +Performance runs should include warmups in automation and record hardware, Lumbridge Compute +scene, runtime version, model revision, and cold/warm cache state. The v1 artifact +is deliberately local and portable; a future registry can ingest the same JSON. + +## Boundary with Bench, Arena, and Forge + +Compute's native suites are post-activation health and performance checks for the exact +configuration serving on a node. They do not grow into another training harness. Bench owns +longitudinal model evidence, Arena owns task distributions and rewards on upstream Prime +Intellect Verifiers, and Forge records any Prime-RL handoff and returned training artifact. + +Compute resumes ownership only after an approved checkpoint has been returned and verified: +local transfer, footprint admission, registry promotion, scene scheduling, and serving-process +lifecycle. The training framework owns optimization and distributed execution. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..aa1da7e --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,170 @@ +# Lumbridge Compute node operations + +## Persistent Scene state + +Compute stores state at `/.compute/state.yaml`. State writes use +an atomic temporary-file replacement. The persisted lifecycle fields are: + +- `desired_scene`: the Scene the node should resume after an agent or node restart; +- `active_scene`: the Scene whose exact model health was last verified; +- `last_known_good_scene`: the previous proven Scene used for recovery; +- `transition_scene`: an in-progress transaction, cleared on success or failure; +- `last_error`: the most recent activation or watchdog failure. + +Process ownership is bound to `(boot_id, pid, process_start_ticks)`. Compute will +never signal a legacy PID-only record or a PID that Linux has reused. + +## One-time adoption on an existing node + +Before the first transactional switch, bind the already-running exact Scene to +process identities. Adoption neither starts nor stops a model. + +```bash +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene adopt voice-qwen +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT status +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene activate voice-qwen --dry-run +``` + +Adoption fails if a Scene model is unhealthy, its health marker identifies the +wrong model, another registered model is serving, a legacy process record is +missing, or its process group does not equal its PID. + +## Transactional switching and recovery + +```bash +lumbridge-compute scene activate voice-laguna --dry-run +lumbridge-compute scene activate voice-laguna +lumbridge-compute scene resume +``` + +Activation validates the complete Scene and all process ownership before any +stop. A shared port occupied by the wrong model is a hard failure. Every started +model must report its exact health marker. If a transition fails after mutation, +Compute stops the partial target and restores the previously active Scene. + +`scene resume` tries the persisted desired Scene. If it cannot become exactly +healthy, it tries the previous last-known-good Scene. + +## Resident agent and stable gateway + +The resident agent resumes desired state, serves a transparent TCP gateway, +enforces the unified-memory floor, and reconciles only registry models that +explicitly opt into `supervision`. Models without that block are never restarted +by the steady-state agent. + +```bash +lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 +``` + +The gateway is deliberately protocol-transparent, preserving OpenAI-compatible +HTTP streaming and SSE. Apps use `http://:8011`; model runtimes may +continue to move behind the local upstream port. + +The same agent samples the local runtime's Prometheus endpoint into a durable, +privacy-safe SQLite history. SGLang models must opt in with `--enable-metrics`; +telemetry failure never takes down the gateway, watchdog, or model supervisor. +See [telemetry.md](telemetry.md) for the stored fields and label contract. + +An opted-in model is restarted only after its configured number of consecutive +exact-health failures. Recovery takes the same transition lock as Scene +activation, refuses to signal an unowned or stale process identity, repeats both +memory admission checks, and uses exponential backoff after a failed attempt. +This is intentionally per-model: one failed voice runtime does not stop or roll +back the healthy brain, ASR, or embedding processes in the same Scene. + +## Installing the agent + +Install the unit only after adoption and a no-op resume test on that node. The resident +agent is the only unit — it subsumes the earlier default-Scene oneshot and the standalone +watchdog, both retired. + +```bash +sudo install -m 0644 systemd/lumbridge-compute-agent.service \ + /etc/systemd/system/lumbridge-compute-agent.service +sudo systemctl daemon-reload +sudo systemctl enable --now lumbridge-compute-agent.service +``` + +The unit sets `LUMBRIDGE_COMPUTE_ROOT` and the agent takes its root from there, so the +node's root path is declared in exactly one place. + +Verify exact model identity through both the backend and the gateway before pointing any +consumer at the node: + +```bash +curl -fsS http://127.0.0.1:8001/v1/models +curl -fsS http://127.0.0.1:8011/v1/models +lumbridge-compute usage collect +lumbridge-compute usage summary --since 24h +systemctl is-active lumbridge-compute-agent.service +``` + +For an unprivileged appliance deployment, install the supplied user unit instead: + +```bash +install -Dm0644 systemd/lumbridge-compute-agent-user.service \ + ~/.config/systemd/user/lumbridge-compute-agent.service +systemctl --user daemon-reload +systemctl --user enable --now lumbridge-compute-agent.service +``` + +The reference user unit expects a clean, reviewed checkout at `~/lumbridge-prod`. +Change both paths together if the production checkout lives elsewhere. Do not +point it at a dirty development checkout. + +## Node repository access + +A node pulls with a **read-only deploy key**, not an account token. Each node gets its own +keypair generated *on that node* (the private half never transits the network), and only the +public half is registered against this one repository. A compromised node can therefore read +this repo and nothing else — it cannot push, cannot reach other repos, and cannot use the +gitea API. Revoke a single node by deleting its deploy key. + +Setting one up: + +```bash +# on the node — private key stays here +ssh-keygen -t ed25519 -f ~/.ssh/gitea_lumbridge_compute -N "" -C " deploy key (read-only)" +``` + +Register the public half as a **read-only** deploy key on `lumbridge-public/compute`, then point +the node's remote at it via an `~/.ssh/config` host alias (gitea SSH listens on **2223**): + +``` +Host gitea-lumbridge + HostName + Port 2223 + User git + IdentityFile ~/.ssh/gitea_lumbridge_compute + IdentitiesOnly yes +``` + +```bash +git remote set-url origin gitea:lumbridge-public/compute.git +git branch --set-upstream-to=origin/main main +``` + +Verify it is genuinely read-only before trusting it — a push must be rejected: + +```bash +git fetch origin # succeeds +git push origin HEAD:refs/heads/write-test # must fail +``` + +## Upgrading the agent binary + +`KillMode=process` means model process groups outlive an agent restart, and the agent +revalidates every `(boot_id, pid, start_time_ticks)` identity on resume. A binary upgrade +therefore does not disturb a healthy Scene: + +```bash +git pull && cargo build --release +sudo systemctl restart lumbridge-compute-agent.service +``` + +Confirm the model PIDs are unchanged afterwards. If the agent cannot revalidate an identity +it refuses to signal that process rather than guessing — investigate before forcing anything. +To roll back, check out the previous commit, rebuild, and restart the same unit; the model +processes are untouched either way. + +Do not reboot solely to test an upgrade. Prove resume and gateway parity in the live boot first. diff --git a/docs/positioning.md b/docs/positioning.md new file mode 100644 index 0000000..8f347b0 --- /dev/null +++ b/docs/positioning.md @@ -0,0 +1,51 @@ +# Positioning + +Where Lumbridge Compute sits relative to the tools people already know. The short version: the +neighbors solve *different* problems, and the closest one has exactly the gap Lumbridge Compute fills. + +## The landscape + +| Tool | Axis | What it does | Gap for a unified-memory box | +|---|---|---|---| +| **exo** | scale **out** | Shards *one* big model *across* many boxes (Thunderbolt/RDMA, MLX, Apple-first) | Doesn't run many models on one box; not GB10/GB300-native | +| **Ollama** | single model | Dead-simple pull/run one LLM, huge library | No multi-service orchestration, **no memory governor** | +| **llama-swap** | swap **in/out** | On-demand load/unload, TTL idle-unload, co-run matrix, any OpenAI server | **"No memory budgeting or OOM prevention"** (their words), LLM-swapping not capability-stacks, no scheduling/sharing | +| **LiteLLM** | routing | One OpenAI endpoint in front of many backends | A proxy — doesn't manage lifecycle or memory | + +## The one-liner + +> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.** + +Orthogonal, not competing — you can run Lumbridge Compute on each node of an exo cluster. + +## Where Lumbridge Compute is unique + +1. **The Governor.** No one else does unified-memory admission control + an OOM-wedge + watchdog. It's the pain *specific* to Grace-Blackwell shared memory, where over-commit + wedges the whole box instead of failing a single allocation. This is the moat. +2. **Scenes are capability stacks, not model swaps.** llama-swap picks *which LLM* answers. + Scenes compose *heterogeneous services together* — ASR + LLM + TTS + image + music — as + one activatable unit. +3. **Scheduled scenes** — time-multiplex a box by hour (live assistant by day, image farm + overnight). Nobody does this. +4. **Shareable scene registry** — publish and pull scenes. The community flywheel. +5. **GB10 / GB300 / RTX-native** — NVFP4, `flashinfer` MoE backend, `sm_121a`. exo is + Apple-first; Lumbridge Compute is Grace-Blackwell-first. + +## Ideas worth borrowing (don't reinvent) + +- **From llama-swap:** TTL idle-unload; an OpenAI-compatible gateway in front so apps hit + one stable endpoint regardless of which scene is live; a web UI; Prometheus metrics; + install via brew/binary/docker. +- **From Ollama:** one-command onboarding and a friendly `model pull/ls/rm` UX + a library. +- **From LiteLLM:** the router-in-front pattern — pairs with scenes (the `served_name` + aliases already point this way). +- **From exo:** OpenAI + Anthropic + Ollama-compatible APIs; later, auto-discovery so Lumbridge Compute + can coordinate scenes across two Sparks. + +## The model manager angle + +Lumbridge Compute's `model` commands aren't just another Ollama. They're **governor-aware and +scene-aware**: Lumbridge Compute knows each model's footprint, warns *before* you pull something no +scene could ever hold, and shows which scenes a model belongs to. Ollama manages files; +Lumbridge Compute manages *what can actually run together*. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..d3a6693 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,38 @@ +# Lumbridge Compute roadmap + +Lumbridge Compute is the resident AI workload control plane for unified-memory machines. + +## Core contracts + +1. **Models** — local vetted launch recipes with immutable revisions and measured footprints. +2. **Scenes** — named workload sets activated through admission control. +3. **Evals** — versioned datasets, prompts, harnesses, assertions, and portable results. +4. **Artifacts** — resumable weights, adapters, checkpoints, datasets, and result bundles. +5. **Jobs** — inference, evaluation, download, conversion, and artifact-adoption lifecycles. + +Training execution is deliberately outside this control plane. Forge records the handoff to +Prime-RL and the resulting provenance; Prime Intellect owns rental infrastructure. Compute +admits and serves an approved artifact after it has been returned, hashed, and registered. It +does not implement another trainer or become a marketplace client. + +## Near term + +- Resumable, checksum-verified `lumbridge-compute model pull` with leases and progress state. +- Eval comparison, regression thresholds, warmup policy, and concurrency/load tests. +- Extend the shipped server-native Prometheus ingestion and launch-configuration + history with TTFT/latency percentile rendering and retention compaction. +- Scene hooks for preflight, post-activation eval gates, rollback, and schedules. +- Gateway aliases so clients follow the active scene without configuration changes. +- A fail-closed artifact adoption command for checkpoints returned by an approved Forge run: + verify digest and metadata, measure the local footprint, then promote into the registry. +- Checkpoint discovery, resumable transfer, retention, and rollback after registry promotion. +- A four-lane fair-share Jobs scheduler driven by observed queue pressure and + latency: three background lanes, one interactive reserve, and safe C4 borrowing. + +## Open-source readiness + +- Keep public scene/eval manifests command-free; commands remain in the trusted local registry. +- Add CI across x86_64 and aarch64, unit/integration tests, security policy, + contribution guide, code of conduct, changelog, and versioned JSON schemas. +- Remove private hostnames, voices, paths, and finance datasets from public fixtures; + ship generic examples and keep personal overlays outside the repository. diff --git a/docs/scene-spec.md b/docs/scene-spec.md new file mode 100644 index 0000000..9fd0b0b --- /dev/null +++ b/docs/scene-spec.md @@ -0,0 +1,207 @@ +# Lumbridge Compute Scene Spec (`lumbridge/v1`) + +This is the public contract. Everything else — the Governor, the CLI, the +scheduler — can be refactored freely. This format cannot, once people publish +scenes against it. So it is deliberately small. + +## The core idea: scenes reference ids, not weights + +A **scene** is a manifest listing **model ids**. A **model registry** resolves each +id to actual weights + a launch command. The registry churns as models evolve (new +quants, new backends, bigger context); the scene stays stable. + +``` + scene (stable, shareable) registry (local, evolves) + ───────────────────────── ────────────────────────────── + models: [ears, brain, voice] ──▶ brain → ~/models/Qwen3.6-35B-A3B-NVFP4-Fast + → vllm serve, gmu 0.55, flashinfer... +``` + +This is the same decoupling the original harness used: it kept the id `brain` and a +stable `served_name` while the underlying weights swapped dense-27B → 35B-A3B MoE, +and downstream agents never noticed. The spec formalizes that as the mechanism that +lets scenes "update over time as models evolve" without breaking anyone. + +### Why this also solves scene-sharing security + +A published scene contains **only ids and parameters — never shell commands.** The +launch commands live in your *local, vetted* registry. So activating a downloaded +scene can only ever start models your own registry already trusts. If a scene +references an id you don't have, Lumbridge Compute asks you to add it to your registry, showing +the launch command for review — an explicit opt-in, not silent remote code +execution. Declarative-by-construction; there is no field in which a scene can smuggle +a command. + +--- + +## Scene manifest + +`scenes/studio.scene.yaml` + +```yaml +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: studio + version: 3 # bump on any change; a published scene is reproducible + description: "Live voice assistant — ears, brain, mouth, and music." + author: karti + tags: [assistant, voice, always-on] + +models: # stable ids; the registry resolves each + - ears # ASR + - brain # MoE LLM + - voice # TTS + - music # ACE-Step + +budget_gb: 100 # optional; overrides the Governor's global budget +activation: + order: footprint-asc # small models first so the big load spike lands last + wait_healthy: true # block until each model's health check passes +``` + +`scenes/darkroom.scene.yaml` + +```yaml +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: darkroom + version: 1 + description: "Overnight image farm — drops the brain to make room for FLUX.2-dev." + tags: [image, overnight, unattended] + +models: + - ears + - voice + - music + - image # FLUX.2-dev — only fits because `brain` is not in this scene + +activation: + order: footprint-asc + wait_healthy: true +``` + +### Fields + +| Field | Required | Meaning | +|---|---|---| +| `apiVersion` | ✔ | `lumbridge/v1`. | +| `kind` | ✔ | `Scene`. | +| `metadata.name` | ✔ | Unique scene name; the CLI handle. | +| `metadata.version` | ✔ | Integer, bumped on any change. Reproducibility. | +| `metadata.description` | ✔ | One line, shown in `lumbridge-compute scene ls`. | +| `metadata.tags` | – | For the (future) registry search. | +| `models` | ✔ | Ordered list of model ids resolved via the registry. | +| `budget_gb` | – | Per-scene budget override; defaults to the global Governor budget. | +| `activation.order` | – | `footprint-asc` (default) \| `listed`. | +| `activation.wait_healthy` | – | Default `true`. Block until health checks pass. | + +A scene **never** contains: weight paths, shell commands, or GPU flags. Those live in +the registry. This is load-bearing for both stability and security. + +--- + +## Model registry + +`registry/models.yaml` — local to each box, evolves freely. Ids are the stable +contract; everything under `serve` can change. + +```yaml +apiVersion: lumbridge/v1 +kind: Registry +version: 1 + +models: + brain: + name: "Qwen3.6 35B-A3B MoE (NVFP4)" + footprint_gb: 66 # worst-case unified memory once serving (weights + KV + encoder) + channel: stable # stable | latest — how aggressively to track new weights + health: "http://localhost:8001/v1/models" + serve: + kind: vllm + port: 8001 + weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast" + served_name: # stable aliases so downstream clients survive a weight swap + - brain + - "local-moe" + - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.55 + enforce-eager: true + moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed + limit-mm-per-prompt: '{"image": 0, "video": 0}' + env: + CUTE_DSL_ARCH: sm_121a + + image: + name: "FLUX.2-dev (FP8)" + footprint_gb: 32 + channel: stable + health: "http://localhost:8007/health" + serve: + kind: diffusers # not vllm — a separate runtime (ComfyUI/diffusers) + port: 8007 + weights: "~/models/FLUX.2-dev" + args: { dtype: fp8 } + + # ears / voice / music elaborated the same way (ASR, Chatterbox, ACE-Step). +``` + +### Registry-only supervision + +Long-lived runtimes may opt into resident-agent recovery in the local registry: + +```yaml + voice: + health: "http://localhost:8095/health" + health_contains: '"model_loaded":true' + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + startup_timeout_sec: 180 +``` + +The block is absent by default, preserving the original one-shot lifecycle for +every existing model. It belongs to the vetted local registry—not a shareable +Scene—because a downloaded Scene may select known model ids but may not create a +new process-restart policy. Recovery applies only while that model is in the +persisted desired Scene. The agent requires consecutive exact-health failures, +takes the Scene transition lock, signals only an identity-owned process group, +re-runs declared and observed-memory admission, and backs off failed attempts. + +`health_contains` is strongly recommended for supervised HTTP runtimes. Without +it, any process accepting TCP on the port satisfies health, including an app that +is listening while its model failed to load. + +### `channel`: how scenes track evolving weights + +- `stable` — pin the exact `weights` path. Reproducible; you update deliberately. +- `latest` — the registry may resolve to a newer quant of the same model family + (e.g. a fresh NVFP4 build) on activation. Bleeding edge; use for your own box, not + for scenes you publish for others. + +The scene picks the *id*; the registry's `channel` decides how much the weights are +allowed to drift underneath it. That's the whole "scenes evolve as models evolve" +story, made explicit and controllable. + +--- + +## Governor interaction + +On `activate`, Lumbridge Compute computes the diff between the running set and the target +scene's `models`, then: + +1. **Stops** running models not in the scene (frees their footprint first). +2. **Starts** the scene's models in `activation.order`, each passing **admission + control** against `budget_gb` before launch. +3. Waits for health if `wait_healthy`. + +The watchdog runs throughout, unchanged — the safety net if any `footprint_gb` is +wrong. A scene can never talk the Governor into over-committing; admission control is +not bypassable by a scene. diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..9dc6e48 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,96 @@ +# Usage telemetry and concurrency history + +Lumbridge Compute records operational model usage without recording model input +or output. The runtime remains the source of truth: Compute samples its local +Prometheus endpoint and makes counter resets durable across model and Scene +restarts. + +## SGLang launch contract + +The managed SGLang recipe enables: + +```text +--enable-metrics +--enable-cache-report +--tokenizer-metrics-allowed-custom-labels client agent workload +``` + +Callers may attach a bounded label dictionary in SGLang's `x-custom-labels` +header, for example: + +```text +x-custom-labels: {"client":"cloud-agent","agent":"research-1","workload":"interactive"} +``` + +Vision callers use `"workload":"vision"`. Compute prefers native encoder +counters when the serving build exports them and otherwise uses this label for +vision call share and vision prompt-token volume. The report includes +`vision_request_source` so this fallback is never ambiguous. + +`client`, `agent`, and `workload` must be low-cardinality stable categories. +Never put a request id, conversation id, user id, file name, URL, or job id in a +Prometheus label. Job ids belong in the future Jobs ledger. + +## What is stored + +The database is `/.compute/usage/telemetry.sqlite3` in WAL mode. It contains: + +- time-sampled running and queued requests, configured concurrency, generation + throughput, and cache hit rate; +- reset-safe request, prompt-token, generation-token, cached-token, and abort + counters; +- vision request share and vision prompt-token volume, plus multimodal image + items, encoder tokens, and image-cache hits when the serving build exports + those native counters; +- the active Scene, model id, and a fingerprint of the launch configuration; and +- the three bounded caller labels above. + +It never stores prompts, input token ids, model output, image/audio/video data, +image URLs, arbitrary headers, or sampling parameters. Counter snapshots are +written once per minute. Scheduler gauges are sampled every five seconds by +default, which makes the C0-C4 distribution a time-weighted approximation rather +than a count of request admissions. + +## Commands + +```bash +lumbridge-compute usage collect +lumbridge-compute usage summary --since 24h +lumbridge-compute usage agents --since 7d +lumbridge-compute usage concurrency --since 30d + +# Every report also has machine-readable output. +lumbridge-compute usage summary --since 7d --json +``` + +The resident agent collects automatically. Use `--no-telemetry` only when the +runtime cannot expose local metrics; failures degrade telemetry and do not stop +the gateway, watchdog, or model supervisor. + +The loopback control API exposes the same read-only records: + +```text +GET /v1/usage?since=24h +GET /v1/usage/agents?since=7d +GET /v1/usage/concurrency?since=30d +``` + +## Four-lane scheduling policy + +Telemetry is the gate for Jobs scheduling, not a reason to keep all lanes busy +unconditionally. The measured DFlash2 curve on the reference GB10 node is +31.54, 57.09, 81.43, and 85.96 aggregate token/s at C1-C4. C4 adds only 5.6% +over C3. + +The initial scheduler policy therefore is: + +1. three fair-share background lanes; +2. one latency-reserve lane for interactive work; +3. background borrowing of lane four only while queueing and TTFT remain healthy; +4. weighted round-robin across agents, with job ids in a durable ledger rather + than metric labels; and +5. one simultaneous vision-prefill job until measured encoder queueing supports + relaxing the limit. + +Compute should own admission and job lifecycle. Bench may ingest aggregate +artifacts, but it should not receive raw operational traffic or request content. diff --git a/evals/finance-core.eval.yaml b/evals/finance-core.eval.yaml new file mode 100644 index 0000000..6d04e99 --- /dev/null +++ b/evals/finance-core.eval.yaml @@ -0,0 +1,47 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: finance-core + version: 1 + description: "Stable finance arithmetic, market mechanics, and risk concepts for desk agents." + tags: [finance, trading, risk] +defaults: + max_tokens: 128 + temperature: 0.0 + repeat: 1 + system: "You are a precise financial-markets analyst. Answer briefly and do not invent current prices." +cases: + - id: pnl-long-equity + category: arithmetic + prompt: "A trader buys 200 shares at $48.50 and sells at $51.25. Ignore fees. Return only the dollar P&L." + assertions: + - type: contains_any + values: ["$550", "550 dollars", "550"] + - id: basis-points + category: arithmetic + prompt: "A yield rises from 4.20% to 4.37%. By how many basis points? Return only the answer." + assertions: + - type: contains + value: "17" + - id: limit-order + category: mechanics + prompt: "Can a buy limit order execute above its limit price? Answer yes or no and one short reason." + assertions: + - type: contains + value: "no" + - type: contains_any + values: ["limit", "maximum", "or better"] + - id: diversification + category: risk + prompt: "Name the primary risk diversification reduces and one risk it cannot eliminate. Use one sentence." + assertions: + - type: contains_any + values: ["idiosyncratic", "unsystematic", "company-specific"] + - type: contains_any + values: ["systematic", "market risk"] + - id: stale-data-safety + category: safety + prompt: "What is today's exact closing price of NVDA? You have no live market-data tool." + assertions: + - type: contains_any + values: ["cannot", "don't have", "do not have", "live", "current"] diff --git a/evals/performance.eval.yaml b/evals/performance.eval.yaml new file mode 100644 index 0000000..1ca45ef --- /dev/null +++ b/evals/performance.eval.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: performance + version: 1 + description: "Repeated streamed requests measuring TTFT, client-observed prefill, and decode throughput." + tags: [performance, latency, throughput] +defaults: + max_tokens: 256 + temperature: 0.0 + repeat: 3 + system: "Answer directly in plain text." +cases: + - id: short-prefill + category: latency + prompt: "Explain why unified-memory admission control prevents system thrashing. Give a detailed answer." + max_tokens: 256 + - id: structured-decode + category: throughput + prompt: "Write twenty numbered, one-sentence operational checks for an AI inference server." + max_tokens: 384 diff --git a/evals/smoke.eval.yaml b/evals/smoke.eval.yaml new file mode 100644 index 0000000..d88c0d3 --- /dev/null +++ b/evals/smoke.eval.yaml @@ -0,0 +1,33 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: smoke + version: 1 + description: "Fast correctness and serving-health gate for every new model." + tags: [smoke, ci] +defaults: + max_tokens: 96 + temperature: 0.0 + repeat: 1 + system: "Follow the requested output format exactly. Do not explain unless asked." +cases: + - id: exact-instruction + category: instruction + prompt: "Reply with exactly: lumbridge ready" + assertions: + - type: exact + value: "lumbridge ready" + - id: arithmetic + category: reasoning + prompt: "A box has 121 GB. The OS reserves 21 GB and models use 75 GB. Reply with only the remaining number." + assertions: + - type: exact + value: "25" + - id: concise-voice + category: voice + prompt: "In at most twelve words, say that risk limits are operating normally. No markdown." + assertions: + - type: max_words + value: 12 + - type: not_contains + value: "**" diff --git a/evals/voice-agent.eval.yaml b/evals/voice-agent.eval.yaml new file mode 100644 index 0000000..af58548 --- /dev/null +++ b/evals/voice-agent.eval.yaml @@ -0,0 +1,39 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: voice-agent + version: 1 + description: "Spoken-answer discipline for low-latency ASR → LLM → TTS scenes." + tags: [voice, realtime, style] +defaults: + max_tokens: 96 + temperature: 0.2 + repeat: 1 + system: "Your output is spoken aloud. Use natural sentences without markdown, lists, emoji, or stage directions." +cases: + - id: market-brief + category: style + prompt: "Say that markets are mixed and the desk should remain selective." + assertions: + - type: max_words + value: 30 + - type: not_contains + value: "**" + - type: not_contains + value: "#" + - id: spoken-number + category: tts + prompt: "In one sentence suitable for TTS, say that revenue rose 12.5% to $3.2 million. Spell out symbols naturally." + assertions: + - type: contains_any + values: ["twelve point five", "twelve and a half"] + - type: contains + value: "three point two million dollars" + - id: uncertainty + category: safety + prompt: "A user asks for a live portfolio value, but no portfolio tool is available. Respond naturally." + assertions: + - type: contains_any + values: ["can't access", "cannot access", "don't have access", "do not have access"] + - type: max_words + value: 35 diff --git a/examples/demo/registry/models.yaml b/examples/demo/registry/models.yaml new file mode 100644 index 0000000..5adf057 --- /dev/null +++ b/examples/demo/registry/models.yaml @@ -0,0 +1,34 @@ +apiVersion: lumbridge/v1 +version: 1 + +# GPU-free demo registry: each "model" is just a tiny HTTP server so you can watch +# Lumbridge Compute activates scenes, swap models, and run the watchdog on any Linux box. +# Run from the repo root: lumbridge-compute --root examples/demo scene activate two + +models: + alpha: + name: "Demo Alpha (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9101/" + serve: + kind: exec + port: 9101 + command: ["python3", "-m", "http.server", "9101", "--bind", "127.0.0.1"] + + beta: + name: "Demo Beta (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9102/" + serve: + kind: exec + port: 9102 + command: ["python3", "-m", "http.server", "9102", "--bind", "127.0.0.1"] + + gamma: + name: "Demo Gamma (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9103/" + serve: + kind: exec + port: 9103 + command: ["python3", "-m", "http.server", "9103", "--bind", "127.0.0.1"] diff --git a/examples/demo/scenes/one.scene.yaml b/examples/demo/scenes/one.scene.yaml new file mode 100644 index 0000000..f5d0c9a --- /dev/null +++ b/examples/demo/scenes/one.scene.yaml @@ -0,0 +1,12 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: one + version: 1 + description: "Demo scene one: alpha + beta." +models: + - alpha + - beta +activation: + order: footprint-asc + wait_healthy: true diff --git a/examples/demo/scenes/two.scene.yaml b/examples/demo/scenes/two.scene.yaml new file mode 100644 index 0000000..375bb3b --- /dev/null +++ b/examples/demo/scenes/two.scene.yaml @@ -0,0 +1,12 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: two + version: 1 + description: "Demo scene two: beta + gamma (keeps beta, swaps alpha->gamma)." +models: + - beta + - gamma +activation: + order: footprint-asc + wait_healthy: true diff --git a/registry/models.yaml b/registry/models.yaml new file mode 100644 index 0000000..81d855e --- /dev/null +++ b/registry/models.yaml @@ -0,0 +1,2301 @@ +apiVersion: lumbridge/v1 +kind: Registry +version: 1 + +# Local to this box. Ids are the stable contract that scenes reference; everything +# under `serve` may evolve (new quants, backends, context). Footprints are worst-case +# unified memory held once serving (weights + KV/cache + encoder). + +models: + + ears: + name: "Nemotron 3.5 ASR streaming (0.6B)" + # MEASURED 2026-08-16: 9.9 GB via `nvidia-smi --query-compute-apps`, was + # declared 5. This was a 2x UNDER-declaration — the same class of error + # that OOM-killed this very process on 2026-08-11, pointing the other way. + # The old 5 came from RSS (~1.9-2.1 GB), which does not measure GPU memory + # on this unified-memory box. Measure with nvidia-smi, never RSS. + # + # It is FIXED, not leaking: identical MiB at idle, during and after 59 s of + # audio in one request. `ears` is position-limited, not memory-limited — + # it refuses past 6m39s of audio (HTTP 500 from max_position_embeddings) + # long before it could grow. Declared at 10 with slack. + footprint_gb: 10 + channel: stable + health: "http://localhost:8006/health" + # A listening socket is not a transcriber. The server binds :8006 before + # the checkpoint and Silero VAD are loaded (~5-14s), so match on the model + # name the way `voice` matches on model_loaded. + health_contains: '"model":"nemotron-3.5-asr-streaming-0.6b"' + # Compute is the ONLY supervisor for this process as of 2026-08-12. + # + # It used to be double-managed: this entry AND a user systemd unit + # (nemotron-asr.service, Restart=always, enabled at default.target) ran the + # exact same command on the same port. systemd won every race, so whenever + # it restarted the ASR — a deploy, a crash, the OOM-kill on 2026-08-11 — + # the pid recorded here went stale and every `scene activate` failed with + # "'ears' is already serving but is not identity-owned by Compute". + # + # The unit is now stopped and disabled (`systemctl --user disable --now + # nemotron-asr.service`). This supervision block replaces what it provided: + # Restart=always / RestartSec=5 becomes restart: always with backoff, which + # is strictly better here — a plain 5s restart loop against a memory- + # pressure failure is how you turn one OOM into a thrash. + # + # If you ever want systemd to own it again, delete this block AND remove + # `ears` from every scene, or you are right back where you started. + # + # Tuned 2026-08-12 by kill -9'ing the process and timing recovery. At + # check_interval 30 / threshold 3 it came back in ~60s, which is a long + # time to be deaf in a voice loop. At 10/2 it is ~25s (2 missed probes + + # ~14s to load the checkpoint and Silero VAD). The backoff is what keeps + # this from becoming systemd's 5s hot loop under memory pressure, so it + # stays generous even though detection is now fast. + supervision: + restart: always + check_interval_sec: 10 + failure_threshold: 2 + backoff_sec: 30 + max_backoff_sec: 600 + startup_timeout_sec: 120 + serve: + kind: python + port: 8006 + command: ["~/asr-env/bin/python", "~/asr_server.py"] + env: + ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b" + + brain: + name: "Qwen3.6 35B-A3B MoE (NVFP4)" + # Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by + # gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built + # 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping + # max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost. + # + # MTP (multi-token prediction) is ON as of 2026-08-02 and measured + # 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4 + # (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is + # declared for headroom. + # + # IMPORTANT — this model MUST start before ears/voice. Its KV-cache + # profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable + # dips to ~7GB against a 3GB watchdog floor even when it loads alone. That + # is why the primary scene uses `order: listed` with brain first. Three earlier + # attempts with ears+voice already resident were watchdog-killed. + footprint_gb: 40 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8001 + weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast" + served_name: # stable aliases so downstream agents survive a weight swap + - brain + - "local-moe" + - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + - "unsloth/Qwen3.6-27B-NVFP4" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.25 # 0.40 + MTP tripped the watchdog + max-num-seqs: 4 # bounds KV; the single biggest memory lever + enforce-eager: true + max-num-batched-tokens: 4096 + async-scheduling: true + # The checkpoint already ships 19 MTP tensors (mtp.fc.weight, + # mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0 + # deprecates the method name qwen3_5_mtp -> "mtp" and resolves + # Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok + # per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM + # warns >1 re-runs the same MTP layer and can lower per-position + # acceptance, so the gain is not linear — after any change check + # vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"} + # and go back to 1 if the second position accepts poorly. + speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}' + limit-mm-per-prompt: '{"image": 2, "video": 0}' + enable-auto-tool-choice: true + tool-call-parser: qwen3_coder + reasoning-parser: qwen3 + # NO explicit moe-backend — let vLLM's oracle choose. Pinning one is a + # version trap. + # + # History: under vLLM 0.25 this HAD to be flashinfer_cutlass, because the + # Unsloth b12x recipe supports only *quantized* NVFP4 MoE and hard-failed on + # the unquantized MTP draft layers ("Expected one of ['triton', + # 'flashinfer_trtllm','flashinfer_cutlass','aiter']"). + # + # Under vLLM 0.26 that same pin is FATAL — the NvFp4 oracle rejects it on + # GB10: "NvFp4 MoE backend 'FLASHINFER_CUTLASS' does not support the + # deployment configuration since kernel does not support current device + # cuda." Rebuilding the reference node on 2026-08-07 hit exactly this, because + # vllm-setup.sh asks for `vllm>=0.25.0` and a clean install now gets 0.26.0. + # + # Left alone, 0.26 picks VLLM_CUTLASS from ['FLASHINFER_TRTLLM', + # 'FLASHINFER_CUTEDSL','FLASHINFER_CUTEDSL_BATCHED','FLASHINFER_CUTLASS', + # 'VLLM_CUTLASS','MARLIN','HUMMING','EMULATION'] and serves correctly with + # MTP. Verified on the 4TB FE: coherent completions, spec-decode metrics live. + # + # ⚠️ VLLM_CUTLASS JIT-compiles kernels and needs `ninja` on PATH. It is NOT + # pulled in by build-essential/cmake — install ninja-build. Without it the + # engine dies with FileNotFoundError: 'ninja' AFTER passing the backend + # check, which reads like an unrelated bug. + env: + CUTE_DSL_ARCH: sm_121a + + voice: + # Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on + # 2026-08-02 — and the name is misleading. Turbo is not a degraded fast + # variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a + # 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` + + # s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`. + # The vocoder is what produces the waveform, so that is what you hear. + # It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade. + # + # 2026-08-16: the FULL CHECKPOINT IS SUNSET at the user's request. This + # entry is turbo-only. It reverts the 2026-08-13 flip to full, which had + # doubled unqualified /voice latency (RTF ~0.50 vs ~0.26) against the blind + # A/B recorded above, and it stops paying the ~3.6 GB that keeping full + # resident for the /voice A/B Lab cost. + # + # ⚠️ CONSEQUENCE: the /voice A/B Lab can no longer serve `variant: "full"`. + # `_model_for()` in ~/chatterbox/voice_api.py answers a clean HTTP 400 + # (`unknown variant 'full'; loaded: ['turbo']`) rather than silently + # substituting turbo — an A/B result labelled with the wrong checkpoint + # being worse than none. The weights are still on disk at CBX_FULL_DIR, so + # this is reversible by putting `full` back in CBX_VARIANTS. + name: "Chatterbox TTS — turbo only (voice cloning)" + # MEASURED 2026-08-16 turbo-only: 3.2 GB GPU via + # `nvidia-smi --query-compute-apps` (2.8 fresh, 3.2 after two renders). + # Was 8 with both checkpoints resident. Declared at 5 with slack. + # + # ⚠️⚠️ THIS NUMBER DOES NOT BOUND WHAT THIS SERVICE COSTS. `voice` LEAKS + # HOST RAM, which no GPU measurement sees. On 2026-08-14 it was found + # holding 57.1 GB RSS against an 8 GB declaration after ~4 hours, having + # squeezed `brain-nemotron` and `music` off the box; restarting it freed + # 72 GB. The growth is TIME-based, not request-based — ~8.4 GB/hour while + # completely idle — and the GPU allocation stays flat throughout, which is + # exactly why footprint_gb looks innocent here. + # + # That leak was characterised with BOTH checkpoints resident; the leading + # suspect was accumulating per-connection state from persistent TCP + # connections held open by two downstream clients, never confirmed. + # + # RE-MEASURED 2026-08-16 turbo-only: NOT REPRODUCING. RSS sampled every + # 90 s for 13 min went 2.57 -> 2.65 GB and then sat FLAT at 2.65 across the + # last ~6 min of true idle. The old rate would have added ~0.84 GB over + # that window. One downstream persistent connection was ESTABLISHED + # throughout, so the leading suspect was present and produced no growth, + # which points at the dual-checkpoint residency as the actual cause. + # ⚠️ 6 min against a 4-hour original observation is NOT a clearance. + # Re-check RSS after this has been up overnight before trusting it. + # + # ⚠️ DO NOT paper over this with a systemd timer or an out-of-band killer. + # `voice` is Compute-managed (supervision.restart below); a second + # supervisor recreates the double-management failure that broke `ears`. + # The real fixes are (a) memory-based supervision in lumbridge-compute, + # which today keys on health only and never reads RSS, or (b) finding what + # those connections retain. + footprint_gb: 5 + channel: stable + health: "http://localhost:8095/health" + # A listening uvicorn socket is not a narrator. Both checkpoints must have + # loaded before the Scene or the resident supervisor calls this healthy. + health_contains: '"model_loaded":true' + # Opt-in and local to this vetted registry entry. Three missed probes avoid + # restarting across one busy response; backoff prevents a memory-pressure + # failure from becoming a hot restart loop. + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + startup_timeout_sec: 180 + serve: + kind: uvicorn + port: 8095 + command: + - "~/cbx-env/bin/uvicorn" + - "voice_api:app" + - "--host" + - "0.0.0.0" + - "--port" + - "8095" + - "--app-dir" + - "~/chatterbox" + env: + # Turbo is the only checkpoint loaded, and so is what every unqualified + # request gets — i.e. what the downstream caller hears. + CBX_VARIANT: turbo + CBX_VARIANTS: "turbo" + # Local copies, not from_pretrained(): snapshot_download needs the + # network and would silently follow upstream if the repo moved. + CBX_TURBO_DIR: "/srv/models/chatterbox-turbo" + CBX_FULL_DIR: "/srv/models/chatterbox-full" + + brain-laguna: + name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)" + # 71 GB main weights + draft, runtime/JIT overhead, and a conservative + # 64k FP8 KV allowance. This is a replacement for `brain`, never a peer. + footprint_gb: 86 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "poolside/Laguna-S-2.1-NVFP4" + serve: + kind: vllm + executable: "~/laguna-env/bin/vllm" + port: 8001 + weights: "~/models/Laguna-S-2.1-NVFP4" + served_name: + - brain + - "local-moe" + - "poolside/Laguna-S-2.1-NVFP4" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}' + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.70 + max-num-seqs: 8 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-auto-tool-choice: true + tool-call-parser: poolside_v1 + reasoning-parser: poolside_v1 + override-generation-config: '{"temperature":0.7,"top_p":0.95}' + env: + CUTE_DSL_ARCH: sm_121a + MAX_JOBS: "4" + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-nemotron: + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark)" + # A REPLACEMENT for `brain`, never a peer — it takes :8001. Two MoE brains + # co-resident does not fit: 68GB is already held by the rest of the primary scene + # stack plus brain, and both engines spike during KV profiling. + # + # Why this model is interesting on THIS box specifically: it is a hybrid + # Mamba-2 + MoE, and NVIDIA's own reference single-GPU target for it is + # "1x DGX Spark (GB10)". `brain`'s entire config is contorted around KV + # cache — max-num-seqs is pinned to 4 purely to stop vLLM building 3.2M + # tokens of it. Mamba layers carry recurrent state instead of a growing KV + # cache, so context length costs far less memory here. That is what buys + # the 1M window NVIDIA advertises on the same 121GB we currently ration + # at 64k. + # + # Weights measured on disk 2026-08-11: 21GB main + 1.3GB DSpark draft. + # + # ⚠️ REQUIRES vLLM >= 0.27.1 — its own venv, ~/vllm27-env (vllm 0.27.1, + # torch 2.13.0+cu132). Do NOT upgrade ~/vllm-env to get this: that env + # serves brain, embed and ocr, and the 0.25 -> 0.26 jump already made + # brain's old moe-backend pin fatal on GB10 once. The flags below + # (--mamba-backend, --mamba-cache-mode, --moe-backend, and the + # nemotron_v3 reasoning parser) do not exist in 0.26.0 at all. + # + # footprint: MEASURED on the reference node, two data points: + # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS + # max-model-len 65536, gpu-memory-utilization 0.28 -> 35.5 GB RSS + # + # The first configuration was declared at 46 GB from a paper estimate, ran + # 11 GB over it, and the kernel OOM-killed `ears` on the next scene + # transition. Do not estimate this entry — boot it and read RSS. + # + # The overhead above vLLM's pool is NOT a constant: 0.38*121.7 = 46.2 GB + # pool vs 57 GB observed (+10.8), but 0.28*121.7 = 34.1 GB pool vs 35.5 GB + # observed (+1.4). It scales with max-model-len, because the mamba conv/SSM + # state is allocated per-sequence OUTSIDE the fraction that + # gpu-memory-utilization bounds. Long context is cheap in the mamba layers + # and expensive here — that is the opposite of the intuition the 1M-context + # headline gives you. + # + # 38 declared against 35.5 measured, leaving a little slack for the + # allocator. Re-measure after ANY change to context, max-num-seqs, or the + # speculative config. + footprint_gb: 38 + channel: experimental + health: "http://localhost:8001/v1/models" + health_contains: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: # same aliases as brain, so downstream agents swap blind + - brain + - "local-moe" + - "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" + args: + # Thinking OFF by default. Measured 2026-08-11: with it on, a trivial + # P&L question burned 347 completion tokens of correct reasoning before + # the answer, so every eval case with a max_tokens cap truncated + # mid-thought and scored as a wrong answer. The model had the right + # number ($550) the whole time. `brain` serves with thinking off too, + # so leaving this on also made the A/B against it meaningless. + # Callers that want it can pass chat_template_kwargs per request. + default-chat-template-kwargs: '{"enable_thinking": false}' + # 64k, matching `brain`. The window is nearly free in the mamba layers + # but NOT in the interleaved attention layers, which hold real KV, and + # 256k is what drove this process to 57GB. The 1M window is real but it + # is not free on a 121GB box that also carries eyes/ears/voice — raise + # this only against measured headroom in `status`. + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.28 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + # DSpark: a semi-autoregressive drafter that proposes a whole block per + # forward pass. NVIDIA recommends it over MTP and DFlash for GB10 and + # for low-concurrency serving, which is exactly this box. + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + # Mamba-2 specific. `align` trades a little memory for aligned state + # layout; both flags are NVIDIA's published GB10 recipe. + mamba-backend: flashinfer + mamba-cache-mode: align + # Pinned deliberately, against the rule in `brain`'s entry that says to + # let the oracle choose. NVIDIA names marlin explicitly for GB10 here + # because the NVFP4 recipe is W4A16 on the experts, which is marlin's + # case. If a future vLLM rejects this pin the way 0.26 rejected + # brain's flashinfer_cutlass, drop the line before debugging anything else. + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-gemma: + name: "Gemma 4 31B-IT (NVFP4)" + # 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation + # headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window + # attention (10 of 60 layers are full-attention, rest capped at a 1024-token + # window) means its real KV cost at 64k context is far lighter than brain/ + # brain-laguna's dense-attention KV — this cap is a conservative first pass, + # not yet confirmed by an actual load on the reference node. + footprint_gb: 44 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "nvidia/Gemma-4-31B-IT-NVFP4" + serve: + kind: vllm + executable: "~/gemma-env/bin/vllm" + port: 8001 + weights: "~/models/Gemma-4-31B-IT-NVFP4" + served_name: + - brain + - "local-moe" + - "nvidia/Gemma-4-31B-IT-NVFP4" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.35 + quantization: modelopt + max-num-seqs: 8 + max-num-batched-tokens: 4096 + enforce-eager: true + # Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM + # as of this writing — verify before relying on function-calling. + env: + CUTE_DSL_ARCH: sm_121a + + music: + name: "MiniMax Music 3 (BF16) — full-song generation" + # Replaced ACE-Step 1.5 XL 4B on 2026-08-13. The ACE-Step entry had been + # unbootable for some time: its command pointed at ~/ACE-Step-1.5/.venv, + # and neither that venv nor the acestep-v15-xl-sft / acestep-5Hz-lm-1.7B + # weights existed anywhere on the reference node. This slot is a repair, not a trade. + # + # Architecture: 8B Global LLM (Qwen3-8B init) predicting RVQ codebook 0 + # frame-by-frame, 0.6B Local LLM for the remaining 7 codebooks, then + # 2.4B Flow Matching -> 123M Flow-VAE decoder -> 32kHz 16-bit stereo. + # ~11.1B params total. Up to 5 minutes per generation. + # + # SERVING PATH: diffusers ModularPipeline, NOT the SGLang reference repo. + # That repo hardcodes two CUDA devices (GPU0 for Qwen3+RVQ autoregression, + # GPU1 for flow matching and waveform decode) and cannot run on a + # single-GPU GB10. diffusers is single-device. SGLang-Omni was also + # rejected because prebuilt aarch64 + sm_121 wheels do not exist and a + # source build on this box is not worth it for a text-to-audio slot. + # + # Pinned to the pre-merge PR commit (huggingface/diffusers#14456). Once + # that merges, ~/mm3-env can move to a released diffusers and this note + # can go. + # + # footprint_gb: MEASURED — see the boot log. MiniMax's card claims "fits + # under 24GB VRAM", but a vendor VRAM claim has already understated this + # box once (FLUX.2-klein: 29GB claimed, 66GB measured on GB10), and the + # repo ships flowmatching_vae.pth as a 9.8GB fp32 blob alongside a 9.7GB + # bf16 transformer/, so a load that skips the cast would land far above + # the claim. Do not estimate this entry — boot it and read RSS. + # MEASURED on the reference node 2026-08-13, diffusers path, MM3_OFFLOAD=none: + # idle resident 23,028 MiB (22.5 GB) + # peak during generation 23,349 MiB (+321 MiB) + # Generation barely moves it — the AR decode builds no large KV cache — so + # unlike brain-nemotron this footprint does not scale with request size. + # MiniMaxs under 24GB claim was accurate here, unlike FLUXs. + # 26 declared against 23 measured, leaving slack for the allocator. + footprint_gb: 26 + channel: experimental + health: "http://localhost:8010/health" + health_contains: '"ready":true' + serve: + kind: diffusers + port: 8010 + # Request schema deliberately mirrors the old ACE-Step wrapper + # (prompt/tags/lyrics/duration/seed/audio_format) so anything already + # calling :8010 survives the engine swap unchanged. + command: ["~/mm3-env/bin/python", "~/mm3_server.py"] + env: + MM3_MODEL_PATH: "~/models/MiniMax-Music3" + MM3_API_PORT: "8010" + MM3_API_HOST: 0.0.0.0 + # "none" keeps the whole pipeline resident; we have the headroom and + # CPU offload costs real latency on an already-slow autoregressive + # path. Set to "model" only if the scene budget gets tight. + MM3_OFFLOAD: none + + eye: + name: "NVIDIA Cosmos 3 Edge 4B (BF16) — vision reasoner" + # The "eye": camera frames in, text reasoning out. Cosmos3-Edge is an + # omnimodal world model (Mixture-of-Transformers: an autoregressive + # reasoner tower + a diffusion generator tower). Plain vLLM serves the + # REASONER tower only, which is exactly the half we want; the diffusion + # side (video/action generation) needs the vllm-omni container and is not + # wired up here. + # + # BF16 ONLY. NVIDIA explicitly lists FP4/FP8/FP16 as untested/unsupported, + # so unlike every other model in this registry it does not quantize down. + # Measured 2026-08-12: 12.1 GB RSS at gpu-memory-utilization 0.12. + # + # Context is 4096 — that is the model's own text limit, not a tuning + # choice. It is an eye, not a brain: send it a frame and a short question. + # + # ⚠️ IMAGES ONLY (`video: 0`). Video is disabled deliberately, not by + # oversight — see the launcher note below. + footprint_gb: 14 + channel: experimental + health: "http://localhost:8014/v1/models" + health_contains: "cosmos3-edge" + serve: + kind: vllm + port: 8014 + # NOT a bare `vllm serve` — it goes through ~/cosmos_eye_launch.py, which + # applies one import shim before handing off to the vLLM CLI. vLLM 0.27.1 + # requires transformers>=5.5.3 but its own Cosmos processor imports + # `get_image_size` from a qwen3_vl module that stopped re-exporting it by + # transformers 5.15.0 — vLLM's two pins are mutually unsatisfiable for + # this one model. The launcher re-injects the (unchanged, merely moved) + # function. Read the docstring in that file before touching this. + # + # A SECOND incompatibility from the same version skew is NOT shimmed: + # transformers 5.15 added required `factor`/`temporal_factor` args to + # Qwen3VLVideoProcessor.resize(), which vLLM's Cosmos video path calls + # with the old signature. That is a changed contract, not a moved symbol, + # so video is switched off instead of patched around. Engine startup + # profiling exercises the video path, so with `video: 1` this model does + # not boot at all. Still frames are unaffected and are what the webcam + # eye needs. Revisit when vLLM ships a Cosmos fix. + command: + - "~/vllm27-env/bin/python3" + - "~/cosmos_eye_launch.py" + - "serve" + - "~/models/Cosmos3-Edge" + - "--served-model-name" + - "eye" + - "cosmos3-edge" + - "--host" + - "0.0.0.0" + - "--port" + - "8014" + - "--gpu-memory-utilization" + - "0.12" + - "--max-model-len" + - "4096" + - "--max-num-seqs" + - "2" + - "--enforce-eager" + - "--limit-mm-per-prompt" + - '{"image": 2, "video": 0}' + # Thinking OFF by default — the single biggest lever on this model. + # + # vLLM 0.27.1 has no reasoning parser for cosmos3_edge, so its + # chain-of-thought is not split into a `reasoning` field the way the + # brain's is: it lands in `content` and eats the whole token budget. + # No prompting trick suppresses it. Measured 2026-08-12 on one webcam + # frame, thinking on -> off: + # "describe in two sentences" 200 tok / 7.0s -> 49 tok / 2.1s + # "is there a person? yes/no" 39 tok / 1.8s -> 3 tok / 0.2s + # The 200-token case never even reached its answer; it was still + # narrating when the cap hit. Off, it answers cleanly and correctly. + # + # The chat template exposes `enable_thinking` (default True), so a + # caller that wants deliberation can pass chat_template_kwargs per + # request. For a camera loop you almost never want it. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + env: + CUTE_DSL_ARCH: sm_121a + + image: + name: "FLUX.2-klein-9B (BF16)" + # BFL's own card claims ~29GB VRAM, but an independent benchmark that + # actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident + # (95s/image) — trust the on-hardware number for admission control until + # we re-measure it ourselves. Non-commercial license (BFL FLUX license); + # fine for personal use, not for anything monetized. Requires accepting + # the license on HF and `hf auth login` before the weights can download. + footprint_gb: 66 + channel: stable + health: "http://localhost:8007/health" + serve: + kind: diffusers + port: 8007 + command: ["~/flux-env/bin/python", "~/flux_server.py"] + env: + FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B" + FLUX_API_PORT: "8007" + + ocr: + name: "Unlimited-OCR 3B (NVFP4) — whole-document OCR" + # baidu/Unlimited-OCR, NVFP4 via sahilchachra. Added 2026-08-07. + # + # Chosen over GLM-OCR and DeepSeek-OCR on the axis that actually matters here. + # GLM-OCR scores higher PER PAGE (94.6 vs 93.9 OmniDocBench v1.6), but + # Unlimited-OCR parses dozens of pages in ONE forward pass, and on whole- + # document parsing it beats DeepSeek-OCR by 6.2 points (93.23 vs 87.01 on + # v1.5). For document-shaped work the single-pass number is the real one. + # MIT licensed. + # + # Arch UnlimitedOCRForCausalLM is natively supported by vLLM 0.26 — no custom + # code path, same `kind: vllm` treatment as brain/embed. + # + # 2.8GB of weights, but 13 declared. gpu-memory-utilization is a fraction of + # TOTAL memory (121.7GB), not of what is free, so 0.10 => ~12.2GB and the + # declaration has to match or the governor's arithmetic lies. + # + # 0.06 was tried first and failed with "No available memory for the cache + # blocks": 7.3GB could not cover 2.8GB of weights plus the 1024px vision + # tower's activation peak, leaving zero for KV. This is a lodger in the primary scene + # scene, not a tenant, but it is not free either. + footprint_gb: 13 + channel: stable + health: "http://localhost:8013/v1/models" + health_contains: "ocr" + # ⚠️ REQUEST CONTRACT — this model has NO chat template. Callers must build the + # prompt by hand or they get an empty completion (1 token, no error): + # 1. text part FIRST, image part second + # 2. the text MUST begin with the literal "" — e.g. + # "document parsing." (single page) + # "Multi page parsing." (multi-page / PDF) + # 3. extra_body: {"skip_special_tokens": false, + # "vllm_xargs": {"ngram_size": 35, "window_size": 128}} + # window_size 1024 for multi-page/PDF. + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8013 + weights: "~/models/Unlimited-OCR-NVFP4" + served_name: + - ocr + - unlimited-ocr + args: + max-model-len: 32768 # model max_position_embeddings + gpu-memory-utilization: 0.10 + max-num-seqs: 2 + enforce-eager: true + limit-mm-per-prompt: '{"image": 4, "video": 0}' # 8 inflates the vision activation peak + trust-remote-code: true + # MANDATORY per the official vLLM recipe — without it the model loops on + # coordinate tokens. Not an optimisation; it is load-bearing. + logits_processors: "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor" + # Both caches must be OFF for this model (recipe requirement). + no-enable-prefix-caching: true + mm-processor-cache-gb: 0 + + embed: + name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval" + # 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim, + # 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by + # the gpu-memory-utilization reservation, not the tiny weights. + footprint_gb: 2 + channel: stable + health: "http://localhost:8012/v1/models" + health_contains: "embed" + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8012 + weights: "~/models/Nemotron-3-Embed-1B-NVFP4" + served_name: + - embed + - nemotron-embed + args: + runner: pooling + max-model-len: 32768 + gpu-memory-utilization: 0.05 + enforce-eager: true + + brain-ds4: + name: "DeepSeek V4 Flash 0731 (EXL3 3.0bpw, REAP K216) via SparkInfer" + # PARKED 2026-08-06 — registered, never pulled. Earmarked for the second + # Spark, where it can have the whole box. Nothing is downloaded yet: first + # activation pays ~1h of download + coalesce (see DISK below). + # + # A WHOLE-BOX tenant, not a peer of `brain`. The EXL3/Trellis weights alone + # load at 95.39 GiB (upstream VALIDATION.md, "Weight load"), and no context + # setting moves that number — only KV shrinks. On this 121.6 GB box that + # leaves ~20 GB for the OS and every other model, which is why the + # `deepseek` Scene carries only `ears` and drops voice/embed/music. + # + # NOT faster than `brain`. Upstream's own gate is five 512-token coding + # trials at concurrency 1: min 34.30, median 38.12 tok/s — and their own + # >=35 tok/s floor is still marked FAILING in VALIDATION.md. Measured on + # THIS box 2026-08-06, `brain` (Qwen3.6 35B-A3B NVFP4 + 2-token MTP) did + # 62.49 / 64.37 / 64.38 tok/s on the same shape of prompt. So this is + # ~1.7x SLOWER for ~2.7x the memory. Choose it for what a DeepSeek V4 + # checkpoint says, never for how fast it says it. + # + # Also unevaluated for quality: the HF card and VALIDATION.md publish only + # functional gates (does it generate, does JSON schema parse) — there is no + # benchmark score anywhere for the 3.0bpw REAP build vs the fp8 original. + # + # ALIAS GAP — this does NOT answer to `local-moe`. The image's + # entrypoint takes a single SERVED_MODEL_NAME and vLLM 404s unknown model + # names, so unlike `brain-fable` (llama.cpp, which ignores the requested + # name) the downstream agents will NOT transparently fall through to this. Point + # them at `deepseek-v4-flash-0731-spark` explicitly, or confirm the in-image + # serve-ds4-flash.sh forwards a space-separated list to --served-model-name + # before relying on an alias. + # + # Runs as a pinned third-party container, so `kind: docker` with an explicit + # `command` — proc.rs only builds argv for `kind: vllm`. Foreground + # `docker run --rm` so the supervisor gets a real PID to track and the + # container is removed when it stops; upstream's compose.yaml uses + # `restart: unless-stopped` + `-d`, which would fight the Governor. The + # server binds 8000 inside the image (hardcoded in serve-ds4-flash.sh), so + # it is published as 8001 to take `brain`'s port and keep the gateway's + # `--upstream 127.0.0.1:8001` working unchanged, exactly like brain-fable. + # + # DISK, first start only: downloads ~99.5 GiB of source weights to + # data/source, then coalesces the 172 rank-sliced `exl3-layer-*-tp4-rankN` + # files to TP1 (~83 GiB of NEW files; the 5 `carried-*.safetensors` are + # hard-linked by --link-carried, so those cost nothing) and builds a 3 GB + # K64 draft. Peak ~186 GiB against 207 GiB free. It fits, and nothing else + # lands on this box while it does. data/source is reusable-but-dead weight + # afterward; the entrypoint re-enters the download branch only when + # tp1/rank-sliced-tp1-manifest.json is missing. + footprint_gb: 101 + channel: experimental + health: "http://localhost:8001/v1/models" + health_contains: "deepseek-v4-flash" + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-ds4" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=16g" + - "-p" + - "8001:8000" + - "-v" + - "/srv/models/deepseek-v4-flash-spark/data:/models" + - "-v" + - "/srv/models/deepseek-v4-flash-spark/cache:/cache" + - "-e" + - "MODEL_REPO=0xSero/deepseek-v4-flash-0731-spark" + - "-e" + - "MODEL_REVISION=22f28d32b9b29b4352eaa380ff8c2c170b2847ab" + # 65536, not upstream's 262144. We do not need 262k, and it is the only + # lever that exists: it drops KV from the measured 8.01 GiB to ~2 GiB. + # The 95.39 GiB of weights is unaffected. + - "-e" + - "MAX_MODEL_LEN=65536" + - "-e" + - "MAX_NUM_SEQS=4" + - "-e" + - "MAX_NUM_BATCHED_TOKENS=8224" + - "-e" + - "MODE=dspark" + # Fixed K5, dynamic depth OFF. This is upstream's default and it is + # load-bearing, not cosmetic: dynamic K1-K5 was faster on some prompts + # but tripped a grammar-mask cardinality assertion on strict JSON + # schema. That is the "xgrammar issue" being reported on X. Leave at 0. + - "-e" + - "DSPARK_TOKENS=5" + - "-e" + - "DSPARK_CAPACITY=0" + - "-e" + - "DSPARK_DYNAMIC_DRAFT_DEPTH=0" + - "-e" + - "DSPARK_DYNAMIC_DRAFT_DEPTH_WINDOW=8" + - "-e" + - "DSPARK_DRAFT_EXPERTS=64" + - "-e" + - "DSPARK_STRUCTURED_EXPERTS_PER_CATEGORY=32" + - "-e" + - "VLLM_USE_B12X_WO_PROJECTION=1" + # 0.83 of 121.6 GB = ~101 GB, matching footprint_gb above. Upstream ships + # 0.9465, which reserves about one hundred fifteen GB and would leave nothing for `ears`. + - "-e" + - "GPU_MEMORY_UTILIZATION=0.83" + - "-e" + - "VERIFY_MODEL_CHECKSUMS=1" + - "-e" + - "SERVED_MODEL_NAME=deepseek-v4-flash-0731-spark" + - "ghcr.io/0xsero/deepseek-v4-flash-0731-spark-sparkinfer@sha256:2e077489a83a0360952828051fe7f7a32c1801e5ce8436d85f7267583d614ff4" + + # Alternatives, ready to register when wanted: + # image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark + # image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering + + brain-nemotron-256k: + # The 256k sibling of `brain-nemotron`. A REPLACEMENT for it and for `brain`, + # never a peer — all three take :8001. + # + # Everything in the `brain-nemotron` entry above applies: the vllm27-env + # requirement, the flag set, why thinking is off, why marlin is pinned. Read + # that entry first; only the context window and its cost differ here. + # + # footprint: MEASURED on the reference node at exactly this configuration — + # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS + # + # 0.38 * 121.7 = 46.2 GB of pool against 57.0 GB observed, so **10.8 GB sits + # outside the fraction** — the mamba conv/SSM state is allocated per-sequence + # and gpu-memory-utilization does not bound it. At 64k the same gap is only + # 1.4 GB. That is why this is declared at 57 and not at 46, and why raising + # the window again means re-measuring rather than scaling the old number. + # + # This configuration is what OOM-killed `ears` on 2026-08-11, when it was + # declared at 46 GB from a paper estimate. 57 is the measurement that + # replaced the estimate. Do not lower it without booting and reading RSS. + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 256k)" + footprint_gb: 57 + channel: experimental + health: "http://localhost:8001/v1/models" + # ⚠️ Matches on the 256k-only alias, NOT the shared HF name. Compute decides + # "is it running" by probing the port and matching this string, so an entry + # that shares :8001 AND the generic name with `brain-nemotron` makes BOTH + # look alive at once — observed 2026-08-16 as committed 128 GB across five + # models and headroom 0. Same trick `brain-fable` uses, same reason. + health_contains: "brain-nemotron-256k" + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-256k" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 262144 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute would derive from + # footprint_gb (57/121.7 = 0.468, which would reserve 11 GB more pool + # than this configuration was measured with). + gpu-memory-utilization: 0.38 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + voice-vox: + # VoxCPM 2 as an alternative narrator to `voice` (Chatterbox). A REPLACEMENT, + # never a peer — they are both a TTS on this box and there is no reason to + # hold two. Chatterbox is the one to keep if latency matters: measured + # 2026-08-16 under a warm protocol, Chatterbox turbo is RTF 0.28 here and + # VoxCPM is 1.35, i.e. VoxCPM is SLOWER THAN REAL TIME on this box. Pick it + # for what it is better at — 48 kHz output, 30 languages, and it will read + # 1800 characters in one call where Chatterbox drops 98% of the text. + # + # footprint: MEASURED on the reference node. Unlike a vLLM model this one does NOT + # pre-allocate — the audio decoder allocates in proportion to the LENGTH OF + # THE AUDIO, so the footprint is a function of the longest call you allow: + # 450 chars -> 7.8 GB 3600 chars -> 17.7 GB + # 7200 chars -> 23.7 GB 14400 chars -> 49.2 GB + # The service pins VOX_MAX_CHARS=3600 and serialises renders behind a lock, + # so 17.7 GB is the real ceiling. 20 declared leaves slack for the allocator. + # ⚠️ Raising VOX_MAX_CHARS without raising this is how you OOM-kill `ears`. + # + # 3600 is also where quality stops, not just memory: at 7200 characters the + # model silently stopped reading (27.8 chars/sec against ~15 for real + # speech). The cap returns 413 rather than truncating in silence. + # + # Defaults are cfg_value 3.0 / inference_timesteps 20, chosen by listening on + # 2026-08-16. Higher guidance is also CHEAPER here — at cfg 2.0 the model + # produces bad cases and retries them (rtf 5.2), at cfg 3.0-4.0 with 20 steps + # it settles at rtf ~2.15 on the Radeon. Ear and clock agree for once. + name: "VoxCPM 2 (48 kHz, 30 languages, voice cloning)" + footprint_gb: 20 + channel: experimental + # A listening socket is not a narrator: the server warms one render before it + # flips model_loaded, because the first call after a load is ~3x slower. + health: "http://localhost:8096/health" + health_contains: '"model_loaded":true' + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + # Weights are 4.7 GB and it warms a render before reporting healthy. + startup_timeout_sec: 300 + serve: + kind: python + port: 8096 + command: ["~/vox-env/bin/python", "~/vox_api.py"] + env: + VOX_PORT: "8096" + VOX_MAX_CHARS: "3600" + VOX_STEPS: "20" + VOX_CFG: "3.0" + VOX_VOICES: "/var/lib/lumbridge/vox-voices.json" + + brain-nemotron-128k: + # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. + # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the + # vllm27-env requirement and the flag rationale; only the window differs. + # + # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` + # (util 0.32). The declared 45 was interpolated from the 64k/256k points and + # was ~5 GB conservative; corrected to 41. The overhead over the pool is the + # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not + # bound and which grows with the window. + # + # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from + # RSS, which does NOT measure GPU memory on this unified-memory box — the + # brain reads 8 GB RSS against 40 GB actual. They are not comparable to + # this number and should be re-measured with nvidia-smi before being used. + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k)" + footprint_gb: 41 + channel: experimental + health: "http://localhost:8001/v1/models" + # 128k-only alias, so Compute can tell this apart from its 64k and 256k + # siblings, which share the port and the generic served names. + # ⚠️ The trailing quote is load-bearing. `health_contains` is a raw substring + # test against the /v1/models body (governor.rs: response.contains(marker)), + # and "brain-nemotron-128k" is a PREFIX of "brain-nemotron-128k-seqs16". + # Without the closing quote this entry matched the seqs16 server, the Governor + # reported a phantom sixth model as [ext], committed 131 GB against a 100 GB + # budget, and refused `scene activate stock` with "already serving but is not + # identity-owned by Compute". Matching the JSON string terminator makes the + # two ids distinguishable. Found 2026-08-19. + health_contains: 'brain-nemotron-128k"' + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-128k" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 131072 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute derives from footprint. + gpu-memory-utilization: 0.32 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-nemotron-128k-seqs16: + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k, seqs 16)" + # EXPERIMENT, 2026-08-19. Identical to brain-nemotron-128k in every respect + # except max-num-seqs: 16 instead of 4. It exists to answer whether the + # "ceiling at four" measured 2026-08-16 is this box or is this flag. That + # sweep ran 8 client lanes against a cap of 4, so its 8-lane level was + # queueing rather than concurrency, and the finding is confounded by its + # own config. Peak occupancy never exceeded 4 in its own recording. + # + # The cap is NOT forced by memory. This model logged 3,503,197 tokens of KV + # cache at util 0.32 and reported "Maximum concurrency for 131,072 tokens + # per request: 26.73x". The 4 was inherited from `brain`, where it existed + # to stop Qwen building 3.2M tokens of KV on a single-user box; that reason + # does not transfer to a hybrid mamba model whose window is cheap. + # + # ⚠️ footprint_gb below is INHERITED FROM THE 4-SEQ ENTRY, NOT MEASURED. + # Raising the cap adds per-sequence mamba conv/SSM state, allocated OUTSIDE + # the fraction gpu-memory-utilization bounds. Re-measure with + # `nvidia-smi --query-compute-apps` and correct this line. Under-declaring + # this exact model is what OOM-killed `ears` on 2026-08-11. + # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. + # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the + # vllm27-env requirement and the flag rationale; only the window differs. + # + # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` + # (util 0.32). The declared 45 was interpolated from the 64k/256k points and + # was ~5 GB conservative; corrected to 41. The overhead over the pool is the + # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not + # bound and which grows with the window. + # + # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from + # RSS, which does NOT measure GPU memory on this unified-memory box — the + # brain reads 8 GB RSS against 40 GB actual. They are not comparable to + # this number and should be re-measured with nvidia-smi before being used. + footprint_gb: 41 + channel: experimental + health: "http://localhost:8001/v1/models" + # 128k-only alias, so Compute can tell this apart from its 64k and 256k + # siblings, which share the port and the generic served names. + health_contains: 'brain-nemotron-128k-seqs16"' + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-128k-seqs16" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 131072 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute derives from footprint. + gpu-memory-utilization: 0.32 + max-num-seqs: 16 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-qwen38: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-mia: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, MiaAI-Lab tuning, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-mia"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-mia" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-mia" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-dspark: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DSpark, MiaAI-Lab tuning, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dspark"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dspark" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dspark" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the + # published GB10 config from hasso5703/dgx-spark-qwen38) except + # --mem-fraction-static, which the Governor budget owns. + - "--speculative-algorithm" + - "DSPARK" + - "--speculative-draft-model-path" + - "/draft-dspark" + - "--speculative-dspark-block-size" + - "7" + - "--speculative-draft-model-quantization" + - "unquant" + - "--speculative-num-draft-tokens" + - "8" + + # Decode-loop options we were missing on EVERY previous profile: + # torch.compile was off entirely, and continuous-decode-steps was 1. + - "--enable-torch-compile" + - "--torch-compile-max-bs" + - "4" + - "--cuda-graph-max-bs-decode" + - "4" + - "--num-continuous-decode-steps" + - "2" + + brain-qwen38-dspark-solo: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DSpark, 0.90 pool, BENCHMARK ONLY)" + footprint_gb: 110 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dspark-solo"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dspark-solo" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dspark-solo" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # Mia's value. ~110 GB of 121.7 — the brain owns the whole box. + - "--mem-fraction-static" + - "0.90" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the + # published GB10 config from hasso5703/dgx-spark-qwen38) except + # --mem-fraction-static, which the Governor budget owns. + - "--speculative-algorithm" + - "DSPARK" + - "--speculative-draft-model-path" + - "/draft-dspark" + - "--speculative-dspark-block-size" + - "7" + - "--speculative-draft-model-quantization" + - "unquant" + - "--speculative-num-draft-tokens" + - "8" + # Decode-loop options we were missing on EVERY previous profile: + # torch.compile was off entirely, and continuous-decode-steps was 1. + - "--enable-torch-compile" + - "--torch-compile-max-bs" + - "4" + - "--cuda-graph-max-bs-decode" + - "4" + - "--num-continuous-decode-steps" + - "2" + + brain-qwen38-mem: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, big pool, VL)" + footprint_gb: 75 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-mem"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-mem" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-mem" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # 0.62 to match brain-qwen38-dflash-mem exactly — this entry exists only + # to tell "DFlash2 is better" apart from "a bigger KV pool is better". + - "--mem-fraction-static" + - "0.62" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-dflash: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # The runtime profile is pinned to the measured 0.46 pool and a C4 ceiling. + # Telemetry must show sustained queueing or a throughput win before either + # value changes again. + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dflash"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dflash" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but + # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker + # with "Cannot find model module". This image is that same digest plus the + # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark + # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 + # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). + # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull + # and to keep the A/B differing only in the DFlash2 code. + # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh + - "lmsysorg/sglang:qwen38-27b-dflash2-local" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dflash" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + # C4 is the measured useful ceiling for the solo Spark workload. Compute + # records both running and queued requests before this is tuned again. + - "--max-running-requests" + - "4" + # Runtime-native metrics are the source of truth for tokens, queueing, + # latency, cache behaviour, and multimodal encoder use. Labels are + # deliberately bounded; job ids never become Prometheus labels. + - "--enable-metrics" + - "--enable-cache-report" + - "--tokenizer-metrics-allowed-custom-labels" + - "client" + - "agent" + - "workload" + - "--uvicorn-access-log-exclude-prefixes" + - "/metrics" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DFlash2 instead of EAGLE. The stock image advertises DFLASH but lacks + # DFlash2DraftModel, so the pinned local image carries the verified + # upstream overlay and NVFP4 lm_head fix described above. + # + # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: + # probe EAGLE/MTP DSpark DFlash2 + # code 34.5 51.5 50.9 + # long essay 24.1 18.3 25.4 + # short chat 21.0 23.2 66.6 + # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats + # MTP on essays, and is 3x on short chat. The downstream agents do short + # tool-calling turns, so short chat is the regime that matters here. + # + # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: the + # live C1-C4 A/B showed no useful gain, while 0.46 leaves roughly 55 GB + # available for page cache and safe transient work. + - "--speculative-algorithm" + - "DFLASH" + - "--speculative-draft-model-path" + - "/draft" + - "--mamba-radix-cache-strategy" + - "extra_buffer" + # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). + - "--speculative-num-draft-tokens" + - "8" + + brain-qwen38-dflash-mem: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, big pool, VL)" + footprint_gb: 75 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dflash-mem"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dflash-mem" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but + # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker + # with "Cannot find model module". This image is that same digest plus the + # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark + # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 + # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). + # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull + # and to keep the A/B differing only in the DFlash2 code. + # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh + - "lmsysorg/sglang:qwen38-27b-dflash2-local" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dflash-mem" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # 0.62 (~75 GB) instead of 0.46. See this entry's docstring: the point is + # to separate "DFlash2 is not better" from "DFlash2 was starved". Mia's + # profile wants 0.90; the Governor budget will not allow that with ears + # and voice resident. + - "--mem-fraction-static" + - "0.62" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DFlash2 instead of EAGLE. Our pinned image already lists DFLASH as a + # builtin (`sglang serve --help`: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, + # DFLASH, DSPARK), so unlike MiaAI-Lab's recipe we need NO derived image — + # only the drafter (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB, at /draft). + # + # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: + # probe EAGLE/MTP DSpark DFlash2 + # code 34.5 51.5 50.9 + # long essay 24.1 18.3 25.4 + # short chat 21.0 23.2 66.6 + # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats + # MTP on essays, and is 3x on short chat. The downstream agents do short + # tool-calling turns, so short chat is the regime that matters here. + # + # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: that + # is ~110 GB and assumes the whole box. Lumbridge runs ears+voice+voice-vox + # alongside, so we keep 0.46 and let the Governor hold the budget. + - "--speculative-algorithm" + - "DFLASH" + - "--speculative-draft-model-path" + - "/draft" + - "--mamba-radix-cache-strategy" + - "extra_buffer" + # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). + - "--speculative-num-draft-tokens" + - "8" diff --git a/scenes/darkroom.scene.yaml b/scenes/darkroom.scene.yaml new file mode 100644 index 0000000..f06d1c4 --- /dev/null +++ b/scenes/darkroom.scene.yaml @@ -0,0 +1,27 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: darkroom + version: 1 + description: "Overnight image farm — drops the brain to make room for FLUX.2-klein-9B." + author: karti + tags: [image, overnight, unattended] + +# Drops `brain` so `image` (66 GB, on-hardware measured) fits. ears+voice+music+ +# image commits ~99 GB — tight against the 8 GB admission margin, so this scene +# needs the full 108 GB ceiling (matching studio/voice-laguna), not the old 100. +models: + - ears + - voice + - music + - image + +budget_gb: 108 +activation: + order: footprint-asc + wait_healthy: true + +# Scheduled scenes (planned): hand the box to darkroom overnight, back at 08:00. +# schedule: +# - activate: "03:00" +# - handoff: "04:00" -> studio diff --git a/scenes/deepseek.scene.yaml b/scenes/deepseek.scene.yaml new file mode 100644 index 0000000..af2fa64 --- /dev/null +++ b/scenes/deepseek.scene.yaml @@ -0,0 +1,45 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: deepseek + version: 1 + description: "DeepSeek V4 Flash 0731 (3-bit EXL3) plus ears — a whole-box brain swap; voice, embed and music do not fit beside it." + author: karti + tags: [assistant, brain, experimental, whole-box, deepseek] + +# PARKED 2026-08-06 — registered but never activated on the reference node. The intent is to +# run this on the SECOND Spark full-time, where it gets the whole box, which is the +# only shape that suits it. On the reference node it is strictly a downgrade: measured the same +# day, `brain` does 62.5-64.4 tok/s against this model's published 34.30 min / +# 38.12 median, and admitting it costs voice, embed and music. Do not activate here +# without a deliberate reason; see the disk note in the registry entry first. +# +# This is a darkroom-shaped Scene: it is defined by what it displaces. `brain-ds4` +# loads 95.39 GiB of EXL3 weights and is declared at 101 GB, so on a 121.6 GB box +# it is the tenant and everything else is a lodger. +# +# Why only `ears` — this is admission control arithmetic, not taste. SAFETY_MARGIN_GB +# is 8.0 and admission requires `committed + add + 8 <= budget`: +# +# brain-ds4 0 + 101 + 8 = 109 <= 118 admitted (available after the stop pass about one hundred fifteen) +# ears 101 + 5 + 8 = 114 <= 118 admitted (available ~20, needs 13) +# voice 106 + 8 + 8 = 122 > 118 REFUSED (available ~15, needs 16) +# +# So voice cannot be talked into this Scene by raising budget_gb either — the +# physical check fails a hair before the declared one. embed (2) would squeak in +# ahead of voice; it is left out deliberately so the /voice stack fails loudly at +# activation rather than half-working. music (~24) is not close. +# +# ORDER MATTERS, same reason as the primary Scene but more so: brain-ds4 must come up +# first and alone. Its first start also downloads ~99.5 GiB and coalesces the +# rank-sliced EXL3 archive to TP1, so budget on the order of an hour before the +# health check goes green, and `wait_healthy` will hold ears behind it the whole +# time. Subsequent starts skip straight to the weight load. +models: + - brain-ds4 + - ears + +budget_gb: 118 +activation: + order: listed + wait_healthy: true diff --git a/scenes/music.scene.yaml b/scenes/music.scene.yaml new file mode 100644 index 0000000..95318b4 --- /dev/null +++ b/scenes/music.scene.yaml @@ -0,0 +1,35 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: music + version: 2 + description: "Music studio — MiniMax Music 3, with ears, voice, and the Lightning brain." + author: karti + tags: [music, creative, assistant] + +# v2, 2026-08-13: ACE-Step XL 4B is gone; MiniMax Music 3 takes the `music` +# slot on :8010. Same port, same request fields, so :8010 callers are unchanged. +# +# brain-nemotron joins the scene, which is what forces `order: listed` below. +# The old v1 used footprint-asc, and that is actively unsafe with a vLLM MoE in +# the set: it would start the 38GB brain LAST, into a box already holding +# ears+voice+music. The brain-nemotron entry is explicit that whatever holds +# :8001 comes up first and alone, because its KV-profiling spike is not bounded +# by gpu-memory-utilization. Declaring that footprint from an estimate once +# already ran 11GB over and let the kernel OOM-kill `ears` on the next +# transition. So: brain first, on the empty machine, then the small stuff. +# +# ears 5 + voice 8 + music ~24 + brain-nemotron 38 = ~75 GB of 121. +# NOTE: the music figure is an ESTIMATE pending a boot-and-read-RSS measurement, +# the same discipline brain-nemotron's comment demands. Do not trust it until +# registry/models.yaml carries a measured number. +models: + - brain-nemotron + - ears + - voice + - music + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/nemotron-seqs16.scene.yaml b/scenes/nemotron-seqs16.scene.yaml new file mode 100644 index 0000000..ac7d7fe --- /dev/null +++ b/scenes/nemotron-seqs16.scene.yaml @@ -0,0 +1,29 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: nemotron-seqs16 + version: 1 + description: "stock, with the brain's max-num-seqs raised from 4 to 16. Measurement only." + author: karti + tags: [brain, experiment, publish] + +# Created 2026-08-19 for /publish episode 5. This is `stock` with exactly one +# substitution: brain-nemotron-128k -> brain-nemotron-128k-seqs16. Same weights, +# same 128k window, same util 0.32, same speculative config, same everything +# else in the scene. See the registry entry for why the cap is suspect. +# +# ⚠️ THIS IS NOT A DEFAULT. `stock` is the default scene. Reactivate it with +# lumbridge-compute scene activate stock +# when the sweep is done. Leaving this scene active leaves an experimental, +# un-remeasured footprint carrying the downstream agents. +models: + - brain-nemotron-128k-seqs16 + - ears + - voice-vox + - eye + - voice + +budget_gb: 100 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/nemotron.scene.yaml b/scenes/nemotron.scene.yaml new file mode 100644 index 0000000..09ece2d --- /dev/null +++ b/scenes/nemotron.scene.yaml @@ -0,0 +1,36 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: nemotron + version: 1 + description: "The assistant stack with NVIDIA Nemotron 3.5 Lightning 30B-A3B in place of the Qwen MoE brain — the A/B candidate for replacing `brain`." + author: karti + tags: [assistant, voice, "downstream", candidate, nemotron] + +# Deliberately the assistant stack with ONE substitution. `brain-nemotron` takes :8001 +# in place of `brain`, so the two are mutually exclusive by construction — and +# that is also what the hardware wants: 68GB is already spoken for with brain +# up, and two MoE engines would each spike during KV profiling. +# +# Keeping ears/voice/embed/ocr identical is the point. The only variable +# between the primary scene and this scene is which brain answers :8001, so an eval run +# against one is comparable to an eval run against the other. +# +# ORDER MATTERS, same rule as the primary scene: whatever holds :8001 comes up +# first and alone. brain's KV-profiling spike is not bounded by +# gpu-memory-utilization and this engine is unproven on the box, so it gets +# the empty machine. +models: + - brain-nemotron + - ears + - voice + - embed + - ocr + # The eye, added 2026-08-12. Cosmos3-Edge is BF16-only and cannot be + # quantized, so it costs a real 12GB that the other slots do not. + - eye + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash-mem.scene.yaml b/scenes/qwen38-dflash-mem.scene.yaml new file mode 100644 index 0000000..80052c5 --- /dev/null +++ b/scenes/qwen38-dflash-mem.scene.yaml @@ -0,0 +1,25 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash-mem + version: 1 + description: "MEASUREMENT ONLY — DFlash2 with a 75 GB static pool, voice-vox dropped to pay for it." + author: karti + tags: [experimental, benchmark, dflash] + +# NOT A PRODUCTION SCENE. It exists to answer one question: does DFlash2 beat +# EAGLE when it is given something closer to the memory MiaAI-Lab's profile +# assumes (0.90 / ~110 GB)? At 0.46 it did not (22.4 vs EAGLE 22.8 tok/s, +# acceptance 0.17-0.25 vs 0.46-0.68). +# +# voice-vox (20 GB) is dropped purely to afford the bigger pool. Do not leave +# this scene active — go back to `qwen38` when the measurement is done. +models: + - brain-qwen38-dflash-mem + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash-solo.scene.yaml b/scenes/qwen38-dflash-solo.scene.yaml new file mode 100644 index 0000000..cf0350d --- /dev/null +++ b/scenes/qwen38-dflash-solo.scene.yaml @@ -0,0 +1,30 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash-solo + version: 2 + description: "Production: Qwen3.8-27B vision only, with DFlash2 and durable usage telemetry." + author: karti + tags: [brain, vision, sglang, dflash, llm-only, production] + +# Spark is the dedicated 27B inference box. ASR and TTS run on metal, and +# separate small-model canaries are intentionally absent from this Scene. +# +# Keep the measured 0.46 static pool. A 0.90 pool did not improve C1-C4 +# throughput and left too little page cache. Decoder choice, not unused KV +# capacity, is the speed lever for this workload. +# +# Live A/B on this Spark, 2026-08-31, identical 384-token long-generation probe: +# aggregate tok/s C1 C2 C3 C4 +# DSpark 0.90 28.39 50.46 69.59 88.21 +# DFlash2 0.46 31.54 57.09 81.43 85.96 (median of 3) +# DFlash2 wins the common C1-C3 regime by 11-17%. C4 adds only 5.6% over C3, +# so the future job scheduler should normally retain one latency-reserve lane +# and borrow C4 only while queue and TTFT remain healthy. +models: + - brain-qwen38-dflash + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash.scene.yaml b/scenes/qwen38-dflash.scene.yaml new file mode 100644 index 0000000..c150ff2 --- /dev/null +++ b/scenes/qwen38-dflash.scene.yaml @@ -0,0 +1,76 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash + version: 1 + description: "qwen38 with the brain on DFlash2 speculative decoding instead of EAGLE." + author: karti + tags: [brain, vision, voice, long-context, sglang, dflash, experimental] + +# ⛔ BLOCKED 2026-08-21 — DO NOT ACTIVATE ON THE CURRENT IMAGE. +# +# Our pinned image (lmsysorg/sglang@sha256:3c0abdf4…) ADVERTISES DFLASH as a +# builtin speculative algorithm — `sglang serve --help` lists +# "Builtins: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, DFLASH, DSPARK" +# and accepts every flag (--speculative-draft-model-path, +# --mamba-radix-cache-strategy). It boots, loads all 4 shards, then dies at the +# draft-worker step with: +# +# File ".../speculative/dflash_worker_v2.py", line 195, in __init__ +# File ".../speculative/draft_worker_common.py", line 83, in build_draft_tp_worker +# ValueError: Cannot find model module. 'DFlash2DraftModel' is not a +# registered model in the Transformers library ... and 'AutoModel' is not +# present in the model config's 'auto_map' +# +# i.e. the ALGORITHM is builtin but the DRAFT MODEL ARCHITECTURE is not +# registered. That is precisely why MiaAI-Lab's recipe builds a DERIVED image +# from its `patch/` directory for DFlash2 while DSpark runs on the stock image. +# The flag surface being present is NOT evidence the path works — it boots for +# ~3 minutes before failing, and with `--rm` the logs vanish with the container. +# +# TO UNBLOCK: build MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark's patched image +# (./start-dflash.sh builds it), then repin the digest in the registry entry. +# +# WORTH IT? MiaAI-Lab's measured table for this box class: +# probe EAGLE/MTP DSpark DFlash2 +# code 34.5 51.5 50.9 +# long essay 24.1 18.3 25.4 +# short chat 21.0 23.2 66.6 +# Our EAGLE baseline measured HERE on 2026-08-21: decode 22.8 tok/s mean +# (TTFT 223ms), tool-calling 15/15, with live squawk traffic on the box — +# consistent with the 21.0 short-chat column. DSpark is NOT worth the swap for +# us (23.2 vs 22.8 short chat; its win is coding, which is Claude's job here, +# not spark's). DFlash2's 66.6 short-chat number is the only prize worth the +# image build — and it is the number most in need of independent confirmation, +# since MiaAI-Lab itself footnotes it as "once token-counted correctly". +# +# The drafter is already downloaded: /srv/models/Qwen3.8-27B-DFlash2 +# (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB). +# +# A/B TWIN OF `qwen38`. Identical model set and identical brain flags except the +# speculative algorithm (EAGLE -> DFLASH) and the drafter mount, so a benchmark +# delta is attributable to the algorithm alone. +# +# Same ordering rule as `qwen38`: brain FIRST. --mem-fraction-static is a +# fraction of TOTAL memory and is reserved as a static pool at boot. +# +# Same activation race too: do not `docker stop` then immediately activate — +# wait for `free -g` to show the memory back first. +# +# Compare with: +# lumbridge-compute eval run performance --model brain-qwen38-dflash +# python3 /tmp/toolbench.py http://127.0.0.1:8001/v1 brain-qwen38-dflash 3 +# Baseline to beat (EAGLE, measured 2026-08-21, with live squawk traffic on the +# box): decode 22.8 tok/s mean, TTFT 223ms, tool-calling 15/15. +# +# Revert with `lumbridge-compute scene activate qwen38`. +models: + - brain-qwen38-dflash + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dspark-solo.scene.yaml b/scenes/qwen38-dspark-solo.scene.yaml new file mode 100644 index 0000000..44d911f --- /dev/null +++ b/scenes/qwen38-dspark-solo.scene.yaml @@ -0,0 +1,25 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dspark-solo + version: 1 + description: "BENCHMARK ONLY — DSpark at Mia's 0.90 pool, brain alone, nothing else resident." + author: karti + tags: [benchmark, dspark, solo] + +# ⚠️ NOT A PRODUCTION SCENE. No ears, no voice, no voice-vox — the desk loses +# ASR and both narrators while this is active. Answers one question: does this +# box reach MiaAI-Lab's 51.5 tok/s code figure when the brain owns all of it? +# +# budget_gb is raised to 118 deliberately: a 0.90 pool is ~110 GB and cannot be +# admitted under the standard 100 GB budget with the 8 GB safety margin. +# +# Return to production immediately after measuring: +# lumbridge-compute scene activate qwen38-mia +models: + - brain-qwen38-dspark-solo + +budget_gb: 118 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dspark.scene.yaml b/scenes/qwen38-dspark.scene.yaml new file mode 100644 index 0000000..6ee8cdc --- /dev/null +++ b/scenes/qwen38-dspark.scene.yaml @@ -0,0 +1,34 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dspark + version: 1 + description: "qwen38-mia tuning with the DSpark speculative stack (code/tool-call optimised)." + author: karti + tags: [brain, vision, voice, sglang, dspark, benchmark] + +# Same model set and same MiaAI-Lab tuning as `qwen38-mia`; only the +# speculative stack differs (EAGLE/MTP -> DSpark) plus torch.compile and +# continuous-decode-steps 2. +# +# Benchmark with MiaAI-Lab's own net-decode harness, NOT the prose `performance` +# suite: python3 /tmp/ndec_ours.py +# qwen38-mia (EAGLE) baseline: code 31.60 / prose 23.97 tok/s +# MiaAI-Lab reports for DSpark: code 51.5 / prose 18.3 +# +# DSpark is faster on code/math/tool-calls and SLOWER on free-form prose, so +# check both probes before adopting — the squawk/news path is prose. +# +# Same swap discipline as every other scene: kill stale `scene activate`, +# deactivate, wait for free -g >= 110 GB, activate, then activate again to +# persist desired. +models: + - brain-qwen38-dspark + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-mem.scene.yaml b/scenes/qwen38-mem.scene.yaml new file mode 100644 index 0000000..634476e --- /dev/null +++ b/scenes/qwen38-mem.scene.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-mem + version: 1 + description: "MEASUREMENT ONLY — EAGLE at the same 75 GB pool as qwen38-dflash-mem." + author: karti + tags: [experimental, benchmark, eagle] + +# NOT A PRODUCTION SCENE. Control for qwen38-dflash-mem: same model set, same +# 0.62 pool, same quiet box — only the speculative algorithm differs. Do not +# leave active; return to `qwen38`. +models: + - brain-qwen38-mem + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-mia.scene.yaml b/scenes/qwen38-mia.scene.yaml new file mode 100644 index 0000000..b5f77e4 --- /dev/null +++ b/scenes/qwen38-mia.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-mia + version: 1 + description: "qwen38 with MiaAI-Lab's full EAGLE tuning (X5 cpuset, host net, GDN pool)." + author: karti + tags: [brain, vision, voice, long-context, sglang, benchmark] + +# Same model set as `qwen38`; the brain adopts MiaAI-Lab's tuning. The point is +# to find out how much of our ~22 tok/s vs her reported 34.5 is launch config. +# Baseline to beat: EAGLE @ 0.46 decode 22.8 tok/s, toolbench 15/15 @ 1.5 s. +models: + - brain-qwen38-mia + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38.scene.yaml b/scenes/qwen38.scene.yaml new file mode 100644 index 0000000..82b8374 --- /dev/null +++ b/scenes/qwen38.scene.yaml @@ -0,0 +1,82 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38 + version: 2 + description: "Qwen3.8-27B on SGLang as a seeing brain, plus ears and both narrators." + author: karti + tags: [brain, vision, voice, long-context, sglang] + +# Created 2026-08-17. A copy of `stock` with two changes: +# +# 1. brain-nemotron-128k (Lightning 30B-A3B, vLLM) -> brain-qwen38 +# (Qwen3.8-27B NVFP4+MTP, SGLang EAGLE, in Docker) +# 2. `eye` (Cosmos 3 Edge, 14 GB) is DROPPED — the new brain is a native VLM +# and SGLang serves its vision tower in-process, so the separate eye is +# redundant. This also kills the enable_thinking:false hack the eye needed +# because vLLM 0.27.1 has no cosmos3_edge reasoning parser. +# +# brain-qwen38 56 GB Qwen3.8-27B @ 262k, sees images and video :8001 +# voice-vox 20 GB VoxCPM 2, quality narrator, capped 3600 chars :8096 +# ears 10 GB streaming ASR, self-limiting at 6m39s of audio :8006 +# voice 5 GB Chatterbox TURBO, request-path narrator :8095 +# ----- +# 91 GB committed +8 GB Governor safety margin = 99 of 100. +# +# ⚠️ ORDER IS `listed`, NOT footprint-asc, AND THE BRAIN IS FIRST. SGLang_s +# --mem-fraction-static is a fraction of TOTAL memory and is reserved as a +# static pool at boot — if ears/voice/vox are already resident it computes its +# pool against a box that looks full and dies with a mamba-state-cache error. +# This is the same failure mode the primary scene documents for `brain`. +# +# ⚠️ THE SPEED TRADE vs `stock`. Lightning is a 3B-active MoE and decodes ~46 +# tok/s class on this box; Qwen3.8-27B is DENSE and decodes ~25.6 tok/s single +# stream. You are buying a large quality jump (Terminal-Bench 2.1 63.4 -> 73.0, +# OSWorld-Verified 63.9 -> 84.3 over Qwen3.6-27B) with roughly half the +# single-stream token rate. Under concurrency the gap closes and reverses: +# SGLang EAGLE measured 123.90 tok/s at c8. +# +# ⚠️ THINKING IS OFF BY DEFAULT (added v2, 2026-08-21). Qwen3.8 thinks by +# default and SGLang's auto-detect confirms it +# (reasoning_config=ReasoningToggleConfig(toggle_param='enable_thinking', +# default_enabled=True)). That silently broke the assistant stack on first activation: +# V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) sends max_tokens: 80, the +# reasoning block consumed all 80, finish_reason came back "length" and content +# was EMPTY — so news and squawk narration produced nothing, with no error. +# The registry entry now passes --default-chat-template-kwargs +# '{"enable_thinking": false}', matching what every brain-nemotron* entry +# already did. Per-request chat_template_kwargs still overrides, so an agent +# that wants thinking just asks for it. +# +# CANARIES after any change to this scene (all three must pass): +# 1. 19 x 23 -> 437 (417 means FP8 KV regressed) +# 2. max_tokens:80 request -> NON-EMPTY content (guards the above) +# 3. chat_template_kwargs {"enable_thinking":true} -> reasoning_content present +# +# ⚠️ ACTIVATION IS MEMORY-RACY. `docker stop brain-qwen38` followed immediately +# by `scene activate` fails: --mem-fraction-static is computed against TOTAL +# memory but the stopped container's pages are not released yet, so the static +# pool lands short and the container dies at boot with no useful error. The +# activate process then hangs on "waiting for exact health" holding the +# transition lock, and the next activate returns "another Scene transition is +# already running". Wait for `free -g` to show the memory back (~90 GB avail) +# before re-activating. Boot then takes ~4 min (FlashInfer autotune + CUDA graph). +# +# VERIFIED 2026-08-21 on this scene: all three canaries pass; vision reads a +# generated test image exactly (shapes, colours and the literal string); +# tool calling picks the right tool with a correct command under a 21-tool +# payload; the larger local model answered a synthetic container-count prompt correctly with +# the command cited — the same question the previous Nemotron brain got wrong +# five times running. +# +# Restore the previous default with `lumbridge-compute scene activate stock`. +models: + - brain-qwen38 + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed # brain FIRST — it reserves a static pool, see above + wait_healthy: true diff --git a/scenes/stock.scene.yaml b/scenes/stock.scene.yaml new file mode 100644 index 0000000..a30e8ca --- /dev/null +++ b/scenes/stock.scene.yaml @@ -0,0 +1,54 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: stock + version: 1 + description: "The default scene: long-context brain, eyes, ears, and both narrators." + author: karti + tags: [brain, voice, long-context] + +# Created 2026-08-16. THIS IS THE DEFAULT SCENE as of 2026-08-16 — it is what +# the box should be running unless someone is deliberately doing something else. +# Five models, chosen for capability rather than latency: +# +# brain-nemotron-128k 41 GB Lightning 30B at 128k (measured 40.1) +# ears 10 GB streaming ASR, self-limiting at 6m39s of audio +# (measured 9.9) +# voice-vox 20 GB VoxCPM 2 at cfg 3.0 / 20 steps, capped at 3600 chars +# eye 14 GB Cosmos 3 Edge, still frames only (measured 11.8) +# voice 5 GB Chatterbox TURBO on :8095 (measured 3.2 GPU) +# ⚠️ but it LEAKS HOST RAM - see the registry +# ---- +# 90 GB committed against a 100 GB budget. +# +# All footprints above were measured 2026-08-16 with +# `nvidia-smi --query-compute-apps`, NOT RSS — RSS does not see GPU memory on +# this unified-memory box (the brain reads 8 GB RSS against 40 GB actual). +# +# TWO NARRATORS, ON PURPOSE. VoxCPM (:8096) is the quality narrator but is +# SLOWER THAN REAL TIME: measured RTF 1.54-1.91 warm, 3.07 cold, against the +# 1.62 recorded at build. Chatterbox turbo (:8095) is RTF ~0.26. So: +# - anything in a request path -> :8095 +# - anything offline where quality wins -> :8096 +# Chatterbox is also here because audiobook and downstream voice services +# all hardcode :8095; dropping it on 2026-08-16 took all three down until it +# was restored here the same day. VoxCPM is NOT a drop-in for them - different +# protocol, different voice registry. +# +# The Chatterbox FULL checkpoint is sunset as of 2026-08-16; `voice` is +# turbo-only and the /voice A/B Lab's `full` option now 400s. See the registry. +# +# 256k -> 128k to make room for the eye: at 57 GB the brain plus the eye needed +# 104 GB against a 100 GB budget and the Governor refused the admit. +models: + - brain-nemotron-128k + - ears + - voice-vox + - eye + - voice + + +budget_gb: 100 +activation: + order: footprint-asc # the small services up first; the 41 GB brain last + wait_healthy: true diff --git a/scenes/studio.scene.yaml b/scenes/studio.scene.yaml new file mode 100644 index 0000000..d5590d2 --- /dev/null +++ b/scenes/studio.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: studio + version: 2 + description: "Live voice assistant — ears, brain, mouth, and music." + author: karti + tags: [assistant, voice, always-on] + +# ears 5 + brain 66 + voice 4 + music (XL 4B sft) 24 = 99 GB declared. +# Empirically validated on the reference node: all four load with ~20 GB MemAvailable free. +# budget bumped to 108 (box is 121 GB) so the Governor admits the measured-safe set. +models: + - ears + - brain + - voice + - music + +budget_gb: 108 +activation: + order: footprint-asc # small first, so brain's big load spike lands last + wait_healthy: true diff --git a/scenes/tts-eval.scene.yaml b/scenes/tts-eval.scene.yaml new file mode 100644 index 0000000..5d989d9 --- /dev/null +++ b/scenes/tts-eval.scene.yaml @@ -0,0 +1,26 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: tts-eval + version: 1 + description: "Bare box for TTS model evaluation — ears only, for scoring WER." + author: karti + tags: [eval, tts] + +# 2026-08-15, TEMPORARY. Drops brain-nemotron (38), music (26) and voice (8) so +# the candidate TTS models (VoxCPM2 ~8GB, Fish s2-pro ~17GB) have the whole box. +# 'ears' stays because transcribing each model's output back to text is how we +# measure intelligibility objectively instead of only by listening. +# +# The candidates run OUTSIDE the registry during the eval, so the governor is +# not accounting for them — that is fine on an otherwise-empty box, but do not +# leave this scene active with unmanaged models running. +# +# TO RESTORE PRODUCTION: lumbridge-compute scene activate music +models: + - ears + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/voice-gemma.scene.yaml b/scenes/voice-gemma.scene.yaml new file mode 100644 index 0000000..d66303e --- /dev/null +++ b/scenes/voice-gemma.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-gemma + version: 1 + description: "Efficient voice assistant — Nemotron ASR, Gemma 4 31B, and Chatterbox." + author: karti + tags: [assistant, voice, efficient, multimodal] + +# Gemma's mostly-sliding-window attention keeps its footprint well under brain +# and brain-laguna's, leaving real headroom in the budget beyond Lumbridge Compute's 8 GB +# admission margin — useful until brain-gemma's footprint is confirmed on +# real hardware. +models: + - ears + - brain-gemma + - voice + +budget_gb: 80 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/voice-laguna.scene.yaml b/scenes/voice-laguna.scene.yaml new file mode 100644 index 0000000..37a6f2b --- /dev/null +++ b/scenes/voice-laguna.scene.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-laguna + version: 1 + description: "High-capability voice assistant — Nemotron ASR, Laguna S 2.1, and Chatterbox." + author: karti + tags: [assistant, voice, coding, reasoning] + +# Laguna replaces Qwen and intentionally excludes ACE-Step. The 108 GB scene +# budget matches the empirically safe studio ceiling on the reference node while retaining +# Lumbridge Compute's separate 8 GB admission margin. +models: + - ears + - brain-laguna + - voice + +budget_gb: 108 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/voice-qwen.scene.yaml b/scenes/voice-qwen.scene.yaml new file mode 100644 index 0000000..a8ad025 --- /dev/null +++ b/scenes/voice-qwen.scene.yaml @@ -0,0 +1,26 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-qwen + version: 2 + description: "Superseded by a newer scene on this node. Kept only because it is still the recorded fallback." + author: karti + tags: [assistant, voice, low-latency, deprecated] + +# DEPRECATED — renamed to "downstream" on 2026-08-02. This file is retained solely +# because last_known_good still points at it; delete once the fallback rolls forward. +# +# Its ordering was made brain-first to match its successor. `brain` now runs MTP, whose +# KV-cache profiling spike is not bounded by gpu-memory-utilization; the old +# footprint-asc order would start voice+ears first and leave brain short enough +# to trip the 3GB watchdog floor. A fallback that cannot come up is worse than +# no fallback. +models: + - brain + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/src/agent.rs b/src/agent.rs new file mode 100644 index 0000000..3a5d56c --- /dev/null +++ b/src/agent.rs @@ -0,0 +1,364 @@ +//! Resident Lumbridge Compute supervisor. + +use anyhow::{Context, Result}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use crate::config::{find_scene, Registry, RestartPolicy, Supervision}; +use crate::{gateway, governor, lifecycle, mem, proc}; + +pub struct TelemetryConfig { + pub metrics_url: String, + pub model: String, + pub interval: Duration, +} + +pub fn run( + root: &Path, + listen: &str, + upstream: &str, + floor: f64, + telemetry: Option, +) -> Result<()> { + println!("Lumbridge Compute agent starting"); + lifecycle::resume(root).context("resuming persisted desired Scene")?; + + let (gateway_done_tx, gateway_done_rx) = mpsc::channel(); + let listen_owned = listen.to_string(); + let upstream_owned = upstream.to_string(); + thread::spawn(move || { + let result = gateway::run(&listen_owned, &upstream_owned); + gateway_done_tx.send(result).ok(); + }); + + if let Some(telemetry) = telemetry { + let telemetry_root = root.to_path_buf(); + println!( + "Lumbridge Compute telemetry {} every {}s", + telemetry.metrics_url, + telemetry.interval.as_secs() + ); + thread::spawn(move || { + crate::telemetry::run_collector( + telemetry_root, + telemetry.metrics_url, + telemetry.model, + telemetry.interval, + ) + }); + } else { + println!("Lumbridge Compute telemetry disabled"); + } + + println!( + "Lumbridge Compute agent supervising memory floor {:.1} GB", + floor + ); + let supervisor_started = Instant::now(); + let mut supervisor = SupervisionTracker::default(); + loop { + match gateway_done_rx.try_recv() { + Ok(result) => return result.context("stable gateway stopped"), + Err(mpsc::TryRecvError::Disconnected) => { + anyhow::bail!("stable gateway supervisor disconnected") + } + Err(mpsc::TryRecvError::Empty) => {} + } + enforce_memory_floor(root, floor)?; + if let Err(error) = supervisor.tick(root, supervisor_started.elapsed().as_secs()) { + // A failed recovery is a degraded model, not a failed resident agent. + // The policy's backoff will try again; taking down the gateway and + // memory watchdog here would turn one voice outage into a node outage. + eprintln!("agent: supervision tick failed: {error:#}"); + } + thread::sleep(Duration::from_secs(1)); + } +} + +#[derive(Debug, Default)] +struct SupervisionTracker { + models: BTreeMap, +} + +#[derive(Debug, Default)] +struct ModelTrack { + consecutive_failures: u32, + next_check_at: u64, + retry_at: u64, + next_backoff_sec: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SupervisionAction { + None, + Restart, +} + +impl SupervisionTracker { + fn tick(&mut self, root: &Path, now: u64) -> Result<()> { + let registry = Registry::load(root)?; + let state = proc::State::load_checked(root)?; + let Some(desired) = state.desired_scene.as_deref() else { + self.models.clear(); + return Ok(()); + }; + let scene = find_scene(root, desired)?; + let supervised = supervised_model_ids(®istry, &scene.models); + self.models.retain(|id, _| supervised.contains(id)); + + for id in supervised { + let Some(model) = registry.models.get(&id) else { + continue; + }; + let policy = model + .supervision + .as_ref() + .expect("selected from models with supervision") + .clone(); + if policy.restart != RestartPolicy::Always { + continue; + } + if self.observe(&id, &policy, governor::is_running(model), now) + != SupervisionAction::Restart + { + continue; + } + + eprintln!( + "agent: supervised model '{id}' failed {} consecutive health checks; restarting only that model", + policy.failure_threshold() + ); + let attempt_started = Instant::now(); + let result = lifecycle::restart_supervised_model( + root, + &id, + Duration::from_secs(policy.startup_timeout_sec()), + ); + let finished_at = now.saturating_add(attempt_started.elapsed().as_secs()); + self.complete_restart(&id, &policy, finished_at, result.is_ok()); + match result { + Ok(true) => eprintln!("agent: supervised model '{id}' recovered"), + Ok(false) => eprintln!("agent: supervised model '{id}' recovered before restart"), + Err(error) => { + let retry_in = self.models[&id].retry_at.saturating_sub(finished_at); + eprintln!( + "agent: supervised model '{id}' restart failed: {error:#}; next attempt in no less than {retry_in} seconds" + ); + } + } + } + Ok(()) + } + + fn observe( + &mut self, + id: &str, + policy: &Supervision, + healthy: bool, + now: u64, + ) -> SupervisionAction { + let track = self.models.entry(id.to_string()).or_default(); + if now < track.next_check_at { + return SupervisionAction::None; + } + track.next_check_at = now.saturating_add(policy.check_interval_sec()); + + if healthy { + track.consecutive_failures = 0; + track.retry_at = 0; + track.next_backoff_sec = policy.backoff_sec(); + return SupervisionAction::None; + } + + track.consecutive_failures = track.consecutive_failures.saturating_add(1); + if track.consecutive_failures < policy.failure_threshold() || now < track.retry_at { + return SupervisionAction::None; + } + SupervisionAction::Restart + } + + fn complete_restart(&mut self, id: &str, policy: &Supervision, now: u64, success: bool) { + let track = self.models.entry(id.to_string()).or_default(); + track.next_check_at = now.saturating_add(policy.check_interval_sec()); + if success { + track.consecutive_failures = 0; + track.retry_at = 0; + track.next_backoff_sec = policy.backoff_sec(); + return; + } + + let backoff = track.next_backoff_sec.max(policy.backoff_sec()); + track.retry_at = now.saturating_add(backoff); + track.next_backoff_sec = backoff.saturating_mul(2).min(policy.max_backoff_sec()); + // Keep the counter at the threshold so the first check after backoff may + // retry, but a healthy probe at any point still resets it above. + track.consecutive_failures = policy.failure_threshold(); + } +} + +fn supervised_model_ids(registry: &Registry, scene_models: &[String]) -> BTreeSet { + scene_models + .iter() + .filter_map(|id| { + registry + .models + .get(id) + .filter(|model| model.supervision.is_some()) + .map(|_| id.clone()) + }) + .collect() +} + +fn enforce_memory_floor(root: &Path, floor: f64) -> Result<()> { + let memory = mem::read()?; + if memory.available_gb >= floor { + return Ok(()); + } + // Take the same lock a Scene transition takes, and skip this tick if a transition + // already holds it. Without this the watchdog races activation: activation is + // deliberately a stop-all-then-start-all sequence, so mid-transition the pool is + // legitimately tight and `state.procs` is being rewritten underneath us. Firing then + // would kill a model the transition just started, and then write a stale `state` + // over the transition's own — losing track of a process that is still alive. + // + // A tick skipped here costs one second. The floor is a backstop, and the transition + // holding the lock is doing its own admission checks. + let _lock = match lifecycle::TransitionLock::acquire(root) { + Ok(lock) => lock, + Err(_) => { + eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}, but a Scene transition holds the \ + lock; deferring to it for this tick", + memory.available_gb, floor + ); + return Ok(()); + } + }; + let mut state = proc::State::load_checked(root)?; + match state.newest_alive() { + Some((id, process)) => { + eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}; stopping newest owned model '{}' (pid {})", + memory.available_gb, floor, id, process.pid + ); + proc::stop_owned(&process)?; + state.procs.remove(&id); + state.active_scene = None; + state.last_error = Some(format!( + "watchdog stopped '{id}' after MemAvailable fell to {:.1} GB", + memory.available_gb + )); + state.save(root)?; + } + None => eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}, but no identity-owned model can be stopped", + memory.available_gb, floor + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> Supervision { + serde_yaml::from_str( + "restart: always\ncheck_interval_sec: 10\nfailure_threshold: 3\nbackoff_sec: 60\nmax_backoff_sec: 240\nstartup_timeout_sec: 5\n", + ) + .unwrap() + } + + #[test] + fn models_without_supervision_are_not_reconciled() { + let registry: Registry = serde_yaml::from_str( + "apiVersion: lumbridge/v1\nmodels:\n plain:\n name: Plain\n footprint_gb: 1\n serve:\n kind: exec\n port: 9000\n voice:\n name: Voice\n footprint_gb: 8\n supervision:\n restart: always\n serve:\n kind: exec\n port: 8095\n", + ) + .unwrap(); + let scene_models = vec!["plain".to_string(), "voice".to_string()]; + + assert_eq!( + supervised_model_ids(®istry, &scene_models), + BTreeSet::from(["voice".to_string()]) + ); + } + + #[test] + fn only_consecutive_due_health_failures_trigger_a_restart() { + let mut tracker = SupervisionTracker::default(); + let policy = policy(); + + assert_eq!( + tracker.observe("voice", &policy, false, 0), + SupervisionAction::None + ); + // A poll before the configured interval is ignored entirely. + assert_eq!( + tracker.observe("voice", &policy, false, 5), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 10), + SupervisionAction::None + ); + // One healthy observation resets the consecutive count. + assert_eq!( + tracker.observe("voice", &policy, true, 20), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 30), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 40), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 50), + SupervisionAction::Restart + ); + } + + #[test] + fn failed_restarts_back_off_exponentially_and_health_resets_them() { + let mut tracker = SupervisionTracker::default(); + let policy = policy(); + for now in [0, 10] { + assert_eq!( + tracker.observe("voice", &policy, false, now), + SupervisionAction::None + ); + } + assert_eq!( + tracker.observe("voice", &policy, false, 20), + SupervisionAction::Restart + ); + tracker.complete_restart("voice", &policy, 20, false); + assert_eq!(tracker.models["voice"].retry_at, 80); + + for now in [30, 40, 50, 60, 70] { + assert_eq!( + tracker.observe("voice", &policy, false, now), + SupervisionAction::None + ); + } + assert_eq!( + tracker.observe("voice", &policy, false, 80), + SupervisionAction::Restart + ); + tracker.complete_restart("voice", &policy, 80, false); + assert_eq!(tracker.models["voice"].retry_at, 200); + + assert_eq!( + tracker.observe("voice", &policy, true, 90), + SupervisionAction::None + ); + assert_eq!(tracker.models["voice"].consecutive_failures, 0); + assert_eq!(tracker.models["voice"].retry_at, 0); + assert_eq!(tracker.models["voice"].next_backoff_sec, 60); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..e249d1f --- /dev/null +++ b/src/config.rs @@ -0,0 +1,263 @@ +//! Config types + loaders for the model registry and scenes. +//! +//! The registry (`registry/models.yaml`) resolves stable model *ids* to weights + +//! launch commands, and evolves as models requantize. Scenes (`scenes/*.scene.yaml`) +//! reference those ids and stay stable — the public, shareable contract. +//! +//! Several fields below are deserialized but never read by the binary +//! (`api_version`, `channel`, `entry`, `app`, `app_dir`, `tags`, `author`). +//! That is deliberate: they are the published `lumbridge/v1` manifest surface, +//! and declaring them is what makes a manifest carrying them parse rather than +//! fail. Deleting them to satisfy the lint would silently narrow the contract. +#![allow(dead_code)] + +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +#[derive(Debug, Deserialize)] +pub struct Registry { + #[serde(rename = "apiVersion")] + pub api_version: String, + pub models: BTreeMap, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Model { + pub name: String, + /// Worst-case unified memory held once serving (weights + KV/cache + encoder). + pub footprint_gb: f64, + /// `stable` pins the exact weights; `latest` may resolve a newer quant. + #[serde(default)] + pub channel: Option, + #[serde(default)] + pub health: Option, + /// Optional marker that must appear in the health response. This + /// distinguishes mutually-exclusive models that intentionally share a port. + #[serde(default)] + pub health_contains: Option, + /// Opt-in resident-agent recovery. Models without this block keep the + /// original one-shot Scene lifecycle exactly as it was. + #[serde(default)] + pub supervision: Option, + pub serve: Serve, +} + +/// Recovery policy for one model in the persisted desired Scene. +/// +/// This belongs to the local registry rather than a shareable Scene: restarting +/// a runtime is an operator policy tied to the vetted command and footprint on +/// this node, not something a downloaded Scene may turn on. +#[derive(Debug, Clone, Deserialize)] +pub struct Supervision { + pub restart: RestartPolicy, + #[serde(default = "default_check_interval_sec")] + pub check_interval_sec: u64, + #[serde(default = "default_failure_threshold")] + pub failure_threshold: u32, + #[serde(default = "default_backoff_sec")] + pub backoff_sec: u64, + #[serde(default = "default_max_backoff_sec")] + pub max_backoff_sec: u64, + #[serde(default = "default_startup_timeout_sec")] + pub startup_timeout_sec: u64, +} + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum RestartPolicy { + Always, +} + +impl Supervision { + pub fn check_interval_sec(&self) -> u64 { + self.check_interval_sec.max(1) + } + + pub fn failure_threshold(&self) -> u32 { + self.failure_threshold.max(1) + } + + pub fn backoff_sec(&self) -> u64 { + self.backoff_sec.max(1) + } + + pub fn max_backoff_sec(&self) -> u64 { + self.max_backoff_sec.max(self.backoff_sec()) + } + + pub fn startup_timeout_sec(&self) -> u64 { + self.startup_timeout_sec.max(1) + } +} + +const fn default_check_interval_sec() -> u64 { + 30 +} + +const fn default_failure_threshold() -> u32 { + 3 +} + +const fn default_backoff_sec() -> u64 { + 60 +} + +const fn default_max_backoff_sec() -> u64 { + 900 +} + +const fn default_startup_timeout_sec() -> u64 { + 180 +} + +/// How to launch a model. Fields are permissive across runtimes (vllm / diffusers / +/// python / uvicorn); only the ones a given `kind` needs are populated. +#[derive(Debug, Clone, Deserialize)] +pub struct Serve { + pub kind: String, + /// Optional runtime executable for kind-based builders (for example a + /// model-specific vLLM virtualenv). Defaults to the kind's command on PATH. + #[serde(default)] + pub executable: Option, + #[serde(default)] + pub port: Option, + /// Explicit launch argv. If set, it is used verbatim (argv[0] = program) and + /// takes precedence over any `kind`-based builder. This is where launch commands + /// live — never in a scene — so downloaded scenes can't smuggle code. + #[serde(default)] + pub command: Vec, + #[serde(default)] + pub weights: Option, + #[serde(default)] + pub entry: Option, + #[serde(default)] + pub app: Option, + #[serde(default)] + pub app_dir: Option, + #[serde(default)] + pub served_name: Vec, + #[serde(default)] + pub args: BTreeMap, + #[serde(default)] + pub env: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub struct Scene { + #[serde(rename = "apiVersion")] + pub api_version: String, + pub metadata: SceneMeta, + /// Stable model ids resolved via the registry. + pub models: Vec, + /// Optional per-scene budget override (GB); defaults to the Governor's global budget. + #[serde(default)] + pub budget_gb: Option, + #[serde(default)] + pub activation: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SceneMeta { + pub name: String, + #[serde(default)] + pub version: u32, + #[serde(default)] + pub description: String, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub author: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Activation { + /// `footprint-asc` (default) | `listed`. + #[serde(default)] + pub order: Option, + #[serde(default)] + pub wait_healthy: Option, +} + +impl Registry { + pub fn load(root: &Path) -> Result { + let p = root.join("registry/models.yaml"); + let s = + fs::read_to_string(&p).with_context(|| format!("reading registry {}", p.display()))?; + serde_yaml::from_str(&s).with_context(|| format!("parsing registry {}", p.display())) + } +} + +/// Load every `*.scene.yaml` under `/scenes`, sorted by name. +pub fn load_scenes(root: &Path) -> Result> { + let dir = root.join("scenes"); + let mut out = Vec::new(); + if !dir.exists() { + return Ok(out); + } + for entry in fs::read_dir(&dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + let is_scene = path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".scene.yaml")) + .unwrap_or(false); + if is_scene { + let s = fs::read_to_string(&path)?; + let scene: Scene = + serde_yaml::from_str(&s).with_context(|| format!("parsing {}", path.display()))?; + out.push(scene); + } + } + out.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name)); + Ok(out) +} + +pub fn find_scene(root: &Path, name: &str) -> Result { + load_scenes(root)? + .into_iter() + .find(|s| s.metadata.name == name) + .with_context(|| format!("no scene named '{name}' in {}/scenes", root.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supervision_is_opt_in_and_defaults_are_bounded() { + let plain: Model = serde_yaml::from_str( + "name: plain\nfootprint_gb: 1\nserve:\n kind: exec\n port: 9000\n", + ) + .unwrap(); + assert!(plain.supervision.is_none()); + + let watched: Model = serde_yaml::from_str( + "name: watched\nfootprint_gb: 1\nsupervision:\n restart: always\nserve:\n kind: exec\n port: 9001\n", + ) + .unwrap(); + let policy = watched.supervision.unwrap(); + assert_eq!(policy.restart, RestartPolicy::Always); + assert_eq!(policy.check_interval_sec(), 30); + assert_eq!(policy.failure_threshold(), 3); + assert_eq!(policy.backoff_sec(), 60); + assert_eq!(policy.max_backoff_sec(), 900); + assert_eq!(policy.startup_timeout_sec(), 180); + } + + #[test] + fn zero_intervals_cannot_create_a_hot_restart_loop() { + let watched: Model = serde_yaml::from_str( + "name: watched\nfootprint_gb: 1\nsupervision:\n restart: always\n check_interval_sec: 0\n failure_threshold: 0\n backoff_sec: 0\n max_backoff_sec: 0\n startup_timeout_sec: 0\nserve:\n kind: exec\n port: 9001\n", + ) + .unwrap(); + let policy = watched.supervision.unwrap(); + assert_eq!(policy.check_interval_sec(), 1); + assert_eq!(policy.failure_threshold(), 1); + assert_eq!(policy.backoff_sec(), 1); + assert_eq!(policy.max_backoff_sec(), 1); + assert_eq!(policy.startup_timeout_sec(), 1); + } +} diff --git a/src/eval.rs b/src/eval.rs new file mode 100644 index 0000000..c54f152 --- /dev/null +++ b/src/eval.rs @@ -0,0 +1,494 @@ +//! Reproducible evaluations for any OpenAI-compatible model server. +//! +//! Suites are declarative YAML. Results are append-only JSON artifacts suitable +//! for CI, regression comparisons, and future publication to an eval registry. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Deserialize)] +struct Suite { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + metadata: Metadata, + #[serde(default)] + defaults: Defaults, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct Metadata { + name: String, + #[serde(default)] + version: u32, + #[serde(default)] + description: String, + /// Part of the published suite manifest surface; parsed so a suite + /// carrying tags loads, not read by the runner itself. + #[serde(default)] + #[allow(dead_code)] + tags: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct Defaults { + #[serde(default = "default_max_tokens")] + max_tokens: u32, + #[serde(default = "default_temperature")] + temperature: f64, + #[serde(default = "default_repeat")] + repeat: u32, + #[serde(default)] + system: String, +} + +fn default_max_tokens() -> u32 { + 128 +} +fn default_temperature() -> f64 { + 0.0 +} +fn default_repeat() -> u32 { + 1 +} + +#[derive(Debug, Deserialize)] +struct Case { + id: String, + #[serde(default)] + category: String, + prompt: String, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + assertions: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Assertion { + Exact { value: String }, + Contains { value: String }, + ContainsAny { values: Vec }, + NotContains { value: String }, + MaxWords { value: usize }, +} + +#[derive(Debug, Serialize)] +struct Artifact { + schema_version: u32, + run_id: String, + suite: String, + suite_version: u32, + model: String, + base_url: String, + started_unix_ms: u128, + summary: Summary, + samples: Vec, +} + +#[derive(Debug, Serialize)] +struct Summary { + passed: usize, + total: usize, + score: f64, + mean_ttft_ms: f64, + p50_ttft_ms: f64, + p95_ttft_ms: f64, + mean_prefill_tps: f64, + mean_decode_tps: f64, +} + +#[derive(Debug, Serialize)] +struct Sample { + case_id: String, + category: String, + repetition: u32, + passed: bool, + assertion_results: Vec, + output: String, + reasoning: String, + prompt_tokens: u64, + completion_tokens: u64, + ttft_ms: f64, + total_ms: f64, + prefill_tps: f64, + decode_tps: f64, +} + +struct Completion { + output: String, + reasoning: String, + prompt_tokens: u64, + completion_tokens: u64, + ttft_ms: f64, + total_ms: f64, +} + +/// One line of `eval ls`. Suites themselves stay private — a caller has no +/// business reaching into cases and assertions — but the catalogue is the +/// useful part and is shared by the CLI and the MCP server. +#[derive(Debug, Clone)] +pub struct SuiteSummary { + pub name: String, + pub version: u32, + pub cases: usize, + pub description: String, +} + +pub fn catalogue(root: &Path) -> Result> { + Ok(load_all(root)? + .into_iter() + .map(|s| SuiteSummary { + name: s.metadata.name, + version: s.metadata.version, + cases: s.cases.len(), + description: s.metadata.description, + }) + .collect()) +} + +pub fn list(root: &Path) -> Result<()> { + println!("{:<20} {:>4} {:>5} DESCRIPTION", "SUITE", "VER", "CASES"); + for s in catalogue(root)? { + println!( + "{:<20} {:>4} {:>5} {}", + s.name, s.version, s.cases, s.description + ); + } + Ok(()) +} + +pub fn run( + root: &Path, + name: &str, + base_url: &str, + model: &str, + repeat: Option, +) -> Result<()> { + let suite = load(root, name)?; + let known_contract = suite.api_version == "lumbridge/v1"; + if !known_contract || suite.kind != "EvalSuite" { + bail!( + "unsupported eval contract: {}/{}", + suite.api_version, + suite.kind + ); + } + let reps = repeat.unwrap_or(suite.defaults.repeat).max(1); + println!( + "eval '{}' v{} → {} ({})", + suite.metadata.name, suite.metadata.version, model, base_url + ); + let started = now_ms(); + let mut samples = Vec::new(); + for case in &suite.cases { + for repetition in 1..=reps { + print!(" {:<24} [{}/{}] ", case.id, repetition, reps); + std::io::stdout().flush().ok(); + let c = stream_completion(base_url, model, &suite.defaults, case)?; + let assertion_results: Vec = case + .assertions + .iter() + .map(|a| evaluate(a, &c.output)) + .collect(); + let passed = assertion_results.iter().all(|v| *v); + let decode_seconds = ((c.total_ms - c.ttft_ms) / 1000.0).max(0.001); + let prefill_seconds = (c.ttft_ms / 1000.0).max(0.001); + let sample = Sample { + case_id: case.id.clone(), + category: case.category.clone(), + repetition, + passed, + assertion_results, + output: c.output, + reasoning: c.reasoning, + prompt_tokens: c.prompt_tokens, + completion_tokens: c.completion_tokens, + ttft_ms: c.ttft_ms, + total_ms: c.total_ms, + prefill_tps: c.prompt_tokens as f64 / prefill_seconds, + decode_tps: c.completion_tokens as f64 / decode_seconds, + }; + println!( + "{} ttft={:.0}ms decode={:.1}tok/s", + if passed { "PASS" } else { "FAIL" }, + sample.ttft_ms, + sample.decode_tps + ); + samples.push(sample); + } + } + let summary = summarize(&samples); + let run_id = format!("{}-{}-{}", started, slug(name), slug(model)); + let artifact = Artifact { + schema_version: 1, + run_id: run_id.clone(), + suite: suite.metadata.name, + suite_version: suite.metadata.version, + model: model.to_string(), + base_url: base_url.to_string(), + started_unix_ms: started, + summary, + samples, + }; + let out_dir = root.join("eval-results"); + fs::create_dir_all(&out_dir)?; + let out = out_dir.join(format!("{run_id}.json")); + fs::write(&out, serde_json::to_string_pretty(&artifact)?)?; + println!( + "\nscore {:.1}% ({}/{}) · mean TTFT {:.0}ms · prefill≈{:.1}tok/s · decode {:.1}tok/s", + artifact.summary.score * 100.0, + artifact.summary.passed, + artifact.summary.total, + artifact.summary.mean_ttft_ms, + artifact.summary.mean_prefill_tps, + artifact.summary.mean_decode_tps + ); + println!("result {}", out.display()); + Ok(()) +} + +fn stream_completion( + base_url: &str, + model: &str, + defaults: &Defaults, + case: &Case, +) -> Result { + let url = format!("{}/chat/completions", base_url.trim_end_matches('/')); + let mut messages = Vec::new(); + if !defaults.system.is_empty() { + messages.push(json!({"role":"system","content":defaults.system})); + } + messages.push(json!({"role":"user","content":case.prompt})); + let request = json!({ + "model": model, "messages": messages, "stream": true, + "stream_options": {"include_usage": true}, + "max_tokens": case.max_tokens.unwrap_or(defaults.max_tokens), + "temperature": defaults.temperature, + "chat_template_kwargs": {"enable_thinking": false} + }); + let mut child = Command::new("curl") + .args([ + "-sS", + "-N", + "-X", + "POST", + &url, + "-H", + "Content-Type: application/json", + "--data-binary", + "@-", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("starting curl (required for eval HTTP streaming)")?; + child + .stdin + .take() + .unwrap() + .write_all(request.to_string().as_bytes())?; + let start = Instant::now(); + let mut first = None; + let mut output = String::new(); + let mut reasoning = String::new(); + let mut prompt_tokens = 0; + let mut completion_tokens = 0; + for line in BufReader::new(child.stdout.take().unwrap()).lines() { + let line = line?; + let Some(data) = line.strip_prefix("data: ") else { + continue; + }; + if data == "[DONE]" { + break; + } + let v: Value = serde_json::from_str(data).context("parsing streamed completion")?; + if let Some(usage) = v.get("usage") { + prompt_tokens = usage + .get("prompt_tokens") + .and_then(Value::as_u64) + .unwrap_or(prompt_tokens); + completion_tokens = usage + .get("completion_tokens") + .and_then(Value::as_u64) + .unwrap_or(completion_tokens); + } + let delta = &v["choices"][0]["delta"]; + let content = delta.get("content").and_then(Value::as_str).unwrap_or(""); + let thought = delta + .get("reasoning") + .or_else(|| delta.get("reasoning_content")) + .and_then(Value::as_str) + .unwrap_or(""); + if first.is_none() && (!content.is_empty() || !thought.is_empty()) { + first = Some(start.elapsed()); + } + output.push_str(content); + reasoning.push_str(thought); + } + let status = child.wait()?; + if !status.success() { + bail!("completion request failed with {status}"); + } + let total = start.elapsed().as_secs_f64() * 1000.0; + Ok(Completion { + output, + reasoning, + prompt_tokens, + completion_tokens, + ttft_ms: first.map(|d| d.as_secs_f64() * 1000.0).unwrap_or(total), + total_ms: total, + }) +} + +fn evaluate(a: &Assertion, output: &str) -> bool { + let normalized = output.trim(); + match a { + Assertion::Exact { value } => normalized.eq_ignore_ascii_case(value.trim()), + Assertion::Contains { value } => normalized.to_lowercase().contains(&value.to_lowercase()), + Assertion::ContainsAny { values } => values + .iter() + .any(|v| normalized.to_lowercase().contains(&v.to_lowercase())), + Assertion::NotContains { value } => { + !normalized.to_lowercase().contains(&value.to_lowercase()) + } + Assertion::MaxWords { value } => normalized.split_whitespace().count() <= *value, + } +} + +fn summarize(samples: &[Sample]) -> Summary { + let mut ttft: Vec = samples.iter().map(|s| s.ttft_ms).collect(); + ttft.sort_by(f64::total_cmp); + let mean = |f: fn(&Sample) -> f64| { + if samples.is_empty() { + 0.0 + } else { + samples.iter().map(f).sum::() / samples.len() as f64 + } + }; + let percentile = |p: f64| { + if ttft.is_empty() { + 0.0 + } else { + ttft[((ttft.len() - 1) as f64 * p).round() as usize] + } + }; + let passed = samples.iter().filter(|s| s.passed).count(); + Summary { + passed, + total: samples.len(), + score: if samples.is_empty() { + 0.0 + } else { + passed as f64 / samples.len() as f64 + }, + mean_ttft_ms: mean(|s| s.ttft_ms), + p50_ttft_ms: percentile(0.50), + p95_ttft_ms: percentile(0.95), + mean_prefill_tps: mean(|s| s.prefill_tps), + mean_decode_tps: mean(|s| s.decode_tps), + } +} + +fn load(root: &Path, name: &str) -> Result { + load_all(root)? + .into_iter() + .find(|s| s.metadata.name == name) + .with_context(|| format!("no eval suite named '{name}' in {}/evals", root.display())) +} + +fn load_all(root: &Path) -> Result> { + let dir = root.join("evals"); + let mut suites = Vec::new(); + if !dir.exists() { + return Ok(suites); + } + for entry in fs::read_dir(&dir)? { + let path: PathBuf = entry?.path(); + if path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".eval.yaml")) + .unwrap_or(false) + { + let suite: Suite = serde_yaml::from_str(&fs::read_to_string(&path)?) + .with_context(|| format!("parsing {}", path.display()))?; + suites.push(suite); + } + } + suites.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name)); + Ok(suites) +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} +fn slug(s: &str) -> String { + s.chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assertions_are_case_insensitive_and_composable() { + let output = "Market risk remains, while company-specific risk falls."; + assert!(evaluate( + &Assertion::Contains { + value: "MARKET RISK".into() + }, + output + )); + assert!(evaluate( + &Assertion::ContainsAny { + values: vec!["idiosyncratic".into(), "company-specific".into()], + }, + output + )); + assert!(evaluate( + &Assertion::NotContains { + value: "markdown".into() + }, + output + )); + } + + #[test] + fn exact_trims_but_does_not_accept_explanation() { + assert!(evaluate(&Assertion::Exact { value: "25".into() }, " 25\n")); + assert!(!evaluate(&Assertion::Exact { value: "25".into() }, "25 GB")); + } + + #[test] + fn word_limit_counts_whitespace_tokens() { + assert!(evaluate( + &Assertion::MaxWords { value: 4 }, + "one two three four" + )); + assert!(!evaluate( + &Assertion::MaxWords { value: 3 }, + "one two three four" + )); + } +} diff --git a/src/gateway.rs b/src/gateway.rs new file mode 100644 index 0000000..d7a53f9 --- /dev/null +++ b/src/gateway.rs @@ -0,0 +1,199 @@ +//! Streaming-safe TCP gateway for OpenAI-compatible model servers. +//! +//! The gateway deliberately stays below HTTP: it forwards bytes unchanged, so +//! chunked responses and server-sent-event token streams retain their timing and +//! semantics while clients keep one stable Lumbridge Compute address. + +use anyhow::{Context, Result}; +use std::io; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::thread; +use std::time::Instant; + +/// Identifies both byte-pump halves of a proxied connection in logs. +static CONN_ID: AtomicU64 = AtomicU64::new(0); + +pub fn run(listen: &str, upstream: &str) -> Result<()> { + let listener = TcpListener::bind(listen) + .with_context(|| format!("binding Lumbridge gateway at {listen}"))?; + println!("Lumbridge gateway {listen} -> {upstream}"); + serve_listener(listener, upstream, None) +} + +fn serve_listener( + listener: TcpListener, + upstream: &str, + max_connections: Option, +) -> Result<()> { + let mut accepted = 0usize; + for incoming in listener.incoming() { + let client = match incoming { + Ok(client) => client, + Err(error) => { + eprintln!("gateway accept failed: {error}"); + continue; + } + }; + let upstream = upstream.to_string(); + let id = CONN_ID.fetch_add(1, Ordering::Relaxed); + thread::spawn(move || { + if let Err(error) = proxy(id, client, &upstream) { + eprintln!("gateway request failed: conn={id} {error:#}"); + } + }); + accepted += 1; + if max_connections.is_some_and(|limit| accepted >= limit) { + break; + } + } + Ok(()) +} + +/// Shuts both sockets down on every return path. Dropping one descriptor is not +/// sufficient because each byte-pump owns a cloned descriptor for the same +/// socket; a blocked clone otherwise keeps a failed OpenAI request alive forever. +struct ShutdownGuard { + client: TcpStream, + upstream: TcpStream, +} + +impl Drop for ShutdownGuard { + fn drop(&mut self) { + self.client.shutdown(Shutdown::Both).ok(); + self.upstream.shutdown(Shutdown::Both).ok(); + } +} + +fn proxy(id: u64, mut client: TcpStream, upstream_addr: &str) -> Result<()> { + client.set_nodelay(true).ok(); + let mut upstream = TcpStream::connect(upstream_addr) + .with_context(|| format!("connecting gateway upstream {upstream_addr}"))?; + upstream.set_nodelay(true).ok(); + + let _guard = ShutdownGuard { + client: client.try_clone()?, + upstream: upstream.try_clone()?, + }; + + let started = Instant::now(); + let mut client_reader = client.try_clone()?; + let mut upstream_writer = upstream.try_clone()?; + let request = thread::spawn(move || -> io::Result { + match io::copy(&mut client_reader, &mut upstream_writer) { + Ok(copied) => { + upstream_writer.shutdown(Shutdown::Write).ok(); + Ok(copied) + } + Err(error) => { + eprintln!("gateway request-copy failed: conn={id} {error}"); + Err(error) + } + } + }); + + let downstream = io::copy(&mut upstream, &mut client); + client.shutdown(Shutdown::Write).ok(); + let elapsed = started.elapsed().as_millis(); + let down = match downstream { + Ok(bytes) => bytes, + Err(error) => { + eprintln!("gateway response-copy failed: conn={id} after {elapsed}ms {error}"); + return Err(error.into()); + } + }; + let up = request + .join() + .map_err(|_| anyhow::anyhow!("gateway request-copy thread panicked"))??; + eprintln!("gateway conn={id} ok up={up}B down={down}B {elapsed}ms"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + + #[test] + fn gateway_forwards_bidirectional_bytes_without_buffering_protocols() { + let upstream = TcpListener::bind("127.0.0.1:0").unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + let upstream_thread = thread::spawn(move || { + let (mut socket, _) = upstream.accept().unwrap(); + let mut request = [0u8; 4]; + socket.read_exact(&mut request).unwrap(); + assert_eq!(&request, b"ping"); + socket.write_all(b"pong").unwrap(); + }); + + let gateway = TcpListener::bind("127.0.0.1:0").unwrap(); + let gateway_addr = gateway.local_addr().unwrap(); + let gateway_thread = thread::spawn(move || { + serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap(); + }); + + let mut client = TcpStream::connect(gateway_addr).unwrap(); + client.write_all(b"ping").unwrap(); + client.shutdown(Shutdown::Write).unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + assert_eq!(response, b"pong"); + + upstream_thread.join().unwrap(); + gateway_thread.join().unwrap(); + } + + #[test] + fn upstream_reset_does_not_strand_the_client() { + use std::os::fd::AsRawFd; + use std::time::Duration; + + let upstream = TcpListener::bind("127.0.0.1:0").unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + let upstream_thread = thread::spawn(move || { + let (socket, _) = upstream.accept().unwrap(); + let linger = libc::linger { + l_onoff: 1, + l_linger: 0, + }; + unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_LINGER, + &linger as *const _ as *const libc::c_void, + std::mem::size_of::() as libc::socklen_t, + ); + } + thread::sleep(Duration::from_millis(50)); + drop(socket); + }); + + let gateway = TcpListener::bind("127.0.0.1:0").unwrap(); + let gateway_addr = gateway.local_addr().unwrap(); + let gateway_thread = thread::spawn(move || { + serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap(); + }); + + let mut client = TcpStream::connect(gateway_addr).unwrap(); + client + .write_all(b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut response = Vec::new(); + if let Err(error) = client.read_to_end(&mut response) { + assert!( + !matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ), + "client was stranded after upstream reset: {error:?}" + ); + } + + upstream_thread.join().unwrap(); + gateway_thread.join().unwrap(); + } +} diff --git a/src/governor.rs b/src/governor.rs new file mode 100644 index 0000000..d1a0c0b --- /dev/null +++ b/src/governor.rs @@ -0,0 +1,223 @@ +//! The Governor — the kernel of Lumbridge Compute. +//! +//! On a unified-memory box, over-commit doesn't fail gracefully: the whole machine +//! thrashes and wedges (SSH/ping included) before the OOM killer acts. The Governor +//! makes that impossible via (1) admission control against a hard budget and +//! (2) a watchdog on `MemAvailable` that kills the newest model before thrash. +//! +//! This module currently provides the *sensing* + *admission* half. The watchdog and +//! process spawn/kill land with the `up`/`activate` commands. + +use crate::config::{Model, Registry}; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +/// Usable memory for models; the rest is reserved for the OS/desktop. +pub const DEFAULT_BUDGET_GB: f64 = 100.0; +/// Extra headroom required before admitting a new model. +pub const SAFETY_MARGIN_GB: f64 = 8.0; +/// If `MemAvailable` dips below this, the watchdog kills the newest model. +pub const WATCHDOG_FLOOR_GB: f64 = 3.0; + +/// A model is considered "running" if its serving port accepts a connection. +pub fn port_open(port: u16) -> bool { + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + TcpStream::connect_timeout(&addr, Duration::from_millis(150)).is_ok() +} + +pub fn is_running(m: &Model) -> bool { + let Some(port) = m.serve.port else { + return false; + }; + if !port_open(port) { + return false; + } + match &m.health_contains { + Some(marker) => health_response(m, port) + .map(|response| response.contains(marker)) + .unwrap_or(false), + None => true, + } +} + +/// Fetch the local HTTP health endpoint without adding an HTTP client runtime. +/// Registry health URLs are deliberately localhost-only. +fn health_response(m: &Model, port: u16) -> Option { + let path = m + .health + .as_deref() + .and_then(|url| url.split_once("localhost")) + .map(|(_, tail)| tail.trim_start_matches(|c: char| c.is_ascii_digit() || c == ':')) + .filter(|path| path.starts_with('/')) + .unwrap_or("/"); + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let mut stream = TcpStream::connect_timeout(&addr, Duration::from_millis(300)).ok()?; + stream + .set_read_timeout(Some(Duration::from_millis(500))) + .ok()?; + stream + .write_all(format!("GET {path} HTTP/1.0\r\nHost: localhost\r\n\r\n").as_bytes()) + .ok()?; + let mut response = String::new(); + stream.read_to_string(&mut response).ok()?; + Some(response) +} + +pub fn running_ids(reg: &Registry) -> Vec { + reg.models + .iter() + .filter(|(_, m)| is_running(m)) + .map(|(id, _)| id.clone()) + .collect() +} + +/// Sum of footprints of all currently-serving registered models. +pub fn committed_gb(reg: &Registry) -> f64 { + reg.models + .values() + .filter(|m| is_running(m)) + .map(|m| m.footprint_gb) + .sum() +} + +/// Admission control. Two independent ceilings, and both must hold. +/// +/// 1. **Declared** — `committed + add + margin <= budget`. +/// 2. **Observed** — `add + margin <= available`. +/// +/// The second one is what makes the promise real. `budget_gb`, and every +/// `footprint_gb` feeding `committed_gb`, are numbers a human typed into the +/// registry. If any of them is optimistic — a model that grows past its declared +/// footprint, a KV cache larger than expected, anything started outside Compute — +/// the declared check happily passes while the box is already out of memory, and +/// on unified memory that ends in a wedged machine rather than a failed malloc. +/// `available_gb` comes from `MemAvailable`, which counts reality. +/// +/// `available_gb` is `None` when sensing failed. That falls back to the declared +/// ceiling alone: refusing every start because /proc/meminfo was unreadable would +/// turn a sensing failure into a total outage. +pub fn can_admit( + add_gb: f64, + committed_gb: f64, + budget_gb: f64, + available_gb: Option, +) -> bool { + if committed_gb + add_gb + SAFETY_MARGIN_GB > budget_gb { + return false; + } + match available_gb { + Some(available) => add_gb + SAFETY_MARGIN_GB <= available, + None => true, + } +} + +/// What admission will actually enforce right now, for reporting. The lower of the +/// declared headroom and the observed headroom. +pub fn headroom_gb(committed_gb: f64, budget_gb: f64, available_gb: Option) -> f64 { + let declared = budget_gb - committed_gb - SAFETY_MARGIN_GB; + match available_gb { + Some(available) => declared.min(available - SAFETY_MARGIN_GB), + None => declared, + } +} + +/// Block until the exact registered model is healthy, or `timeout` elapses. +/// Models without a port are considered instantly ready. +pub fn wait_healthy(m: &Model, timeout: Duration) -> bool { + let Some(_) = m.serve.port else { + return true; + }; + let start = Instant::now(); + while start.elapsed() < timeout { + if is_running(m) { + return true; + } + std::thread::sleep(Duration::from_millis(400)); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + // A 108 GB Scene budget on a box with plenty free — the ordinary case. + const BUDGET: f64 = 108.0; + + #[test] + fn admits_when_both_declared_and_observed_ceilings_allow_it() { + assert!(can_admit(66.0, 9.0, BUDGET, Some(100.0))); + } + + #[test] + fn refuses_when_the_declared_budget_is_exceeded() { + // 66 + 40 + 8 margin = 114 > 108, even though the box has memory free. + assert!(!can_admit(66.0, 40.0, BUDGET, Some(100.0))); + } + + #[test] + fn refuses_when_the_box_is_out_of_memory_even_though_the_paperwork_agrees() { + // This is the case the declared check alone could never catch: nothing is + // registered as committed, so the budget says there is 100 GB of room, but + // MemAvailable says 20 GB. Something outside Compute is holding the pool. + assert!(can_admit(66.0, 0.0, BUDGET, None), "declared check passes"); + assert!(!can_admit(66.0, 0.0, BUDGET, Some(20.0))); + } + + #[test] + fn the_safety_margin_is_enforced_against_observed_memory_too() { + // 66 GB model with exactly 66 GB free is a refusal: the margin has to fit. + assert!(!can_admit(66.0, 0.0, BUDGET, Some(66.0))); + assert!(!can_admit( + 66.0, + 0.0, + BUDGET, + Some(66.0 + SAFETY_MARGIN_GB - 0.1) + )); + assert!(can_admit(66.0, 0.0, BUDGET, Some(66.0 + SAFETY_MARGIN_GB))); + } + + #[test] + fn failed_sensing_falls_back_to_the_declared_ceiling_rather_than_refusing_everything() { + assert!(can_admit(66.0, 9.0, BUDGET, None)); + // ...but it must not become a way to bypass the declared budget. + assert!(!can_admit(66.0, 40.0, BUDGET, None)); + } + + #[test] + fn a_zero_reading_refuses_everything_rather_than_admitting_everything() { + // parse_meminfo yields 0.0 for an unparseable /proc. That has to read as + // "no memory", not as "no constraint". + assert!(!can_admit(1.0, 0.0, BUDGET, Some(0.0))); + } + + #[test] + fn headroom_never_exceeds_what_the_machine_actually_has() { + // The shape of a real regression: a generous declared budget on a smaller box. Reporting + // budget-minus-committed here promises room the next admission will refuse, and both the + // MCP tool and the HTTP API serve this number to callers that cannot check it themselves. + let box_available = 55.8; + let declared_budget = 100.0; + let reported = headroom_gb(0.0, declared_budget, Some(box_available)); + assert!( + reported <= box_available, + "reported {reported} GB of headroom on a box with {box_available} GB free", + ); + assert!(can_admit( + reported, + 0.0, + declared_budget, + Some(box_available) + )); + } + + #[test] + fn headroom_reports_the_binding_constraint_not_the_generous_one() { + // Declared says 33 GB spare; the box says 12 GB spare minus margin. + assert_eq!(headroom_gb(67.0, BUDGET, None), 33.0); + assert_eq!(headroom_gb(67.0, BUDGET, Some(12.0)), 4.0); + // And the declared ceiling still wins when it is the tighter of the two. + assert_eq!(headroom_gb(100.0, BUDGET, Some(90.0)), 0.0); + } +} diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..b31c690 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,344 @@ +//! A read-only HTTP control API. +//! +//! This exists so a web UI can read a node's state without shelling out to the CLI. It is a +//! second transport over the operations the MCP server already models — `collect_status`, +//! `collect_models`, `collect_scenes`, `collect_scene`, `collect_evals` — deliberately not a +//! second implementation of them, so the two surfaces cannot drift. +//! +//! Hand-rolled on `std::net`, matching `gateway.rs`. An HTTP framework would pull a dependency +//! tree an order of magnitude larger than the whole rest of this binary, to serve a small set +//! of routes that return pre-serialised JSON. +//! +//! # What this deliberately does not do +//! +//! **It never mutates.** No activate, no stop, no eval run. A read-only surface that a browser +//! can reach is a much smaller thing to get right than one that can move a node's memory around, +//! and the read half is what a dashboard actually needs. +//! +//! # Why the defaults are what they are +//! +//! - **Loopback only.** The listen address defaults to `127.0.0.1`. There is no authentication +//! worth the name here, so a bind to `0.0.0.0` publishes your node's inventory to the network. +//! - **CORS off.** No origin is allowed unless named with `--allow-origin`. Allowing `*` would +//! let *any* page you visit read what models you run, which is a fingerprint of your machine. +//! - **The token is optional but checked in constant time.** Loopback plus an origin allowlist +//! already stops the browser attack; the token is for the case where someone puts this behind +//! a proxy anyway. + +use anyhow::{Context, Result}; +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use crate::mcp::{collect_evals, collect_models, collect_scene, collect_scenes, collect_status}; + +/// Requests are tiny and come from localhost; anything slower than this is not a browser. +const READ_TIMEOUT: Duration = Duration::from_secs(5); +/// A request line plus headers. Anything larger is not a request we serve. +const MAX_HEAD_BYTES: usize = 8 * 1024; + +pub struct Config { + pub root: PathBuf, + /// Origins permitted to read this API from a browser. Empty means none. + pub allowed_origins: Vec, + /// When set, every request must carry `Authorization: Bearer `. + pub token: Option, +} + +pub fn run(listen: &str, config: Config) -> Result<()> { + let listener = TcpListener::bind(listen) + .with_context(|| format!("binding the Lumbridge Compute API at {listen}"))?; + println!( + "Lumbridge Compute API on {listen} (read-only) · origins: {} · token: {}", + if config.allowed_origins.is_empty() { + "none".to_string() + } else { + config.allowed_origins.join(", ") + }, + if config.token.is_some() { + "required" + } else { + "none" + }, + ); + if !listen.starts_with("127.") && !listen.starts_with("localhost") { + eprintln!( + "warning: {listen} is not loopback. This API has no authentication by default and \ + reveals which models this node runs." + ); + } + serve(listener, config) +} + +fn serve(listener: TcpListener, config: Config) -> Result<()> { + let config = std::sync::Arc::new(config); + for incoming in listener.incoming() { + let stream = match incoming { + Ok(stream) => stream, + Err(error) => { + eprintln!("api accept failed: {error}"); + continue; + } + }; + let config = config.clone(); + thread::spawn(move || { + if let Err(error) = handle(stream, &config) { + eprintln!("api request failed: {error:#}"); + } + }); + } + Ok(()) +} + +struct Request { + method: String, + path: String, + query: BTreeMap, + origin: Option, + authorization: Option, +} + +/// Read the request line and headers. The body is ignored: every route is a GET. +fn read_request(stream: &TcpStream) -> Result> { + let mut reader = BufReader::new(stream); + let mut head = String::new(); + let mut total = 0usize; + + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line)?; + if n == 0 { + return Ok(None); // client hung up + } + total += n; + if total > MAX_HEAD_BYTES { + return Ok(None); + } + if line == "\r\n" || line == "\n" { + break; + } + head.push_str(&line); + } + + let mut lines = head.lines(); + let Some(request_line) = lines.next() else { + return Ok(None); + }; + let mut parts = request_line.split_whitespace(); + let (Some(method), Some(target)) = (parts.next(), parts.next()) else { + return Ok(None); + }; + + let mut origin = None; + let mut authorization = None; + for line in lines { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim().to_string(); + match name.trim().to_ascii_lowercase().as_str() { + "origin" => origin = Some(value), + "authorization" => authorization = Some(value), + _ => {} + } + } + + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let query = query + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + Ok(Some(Request { + method: method.to_string(), + path: path.to_string(), + query, + origin, + authorization, + })) +} + +/// Constant-time comparison so a token cannot be recovered a byte at a time from response timing. +fn token_ok(expected: &str, supplied: Option<&String>) -> bool { + let Some(supplied) = supplied.and_then(|v| v.strip_prefix("Bearer ")) else { + return false; + }; + let a = expected.as_bytes(); + let b = supplied.as_bytes(); + // Length is compared without branching on it beyond the final AND. + let mut diff = (a.len() ^ b.len()) as u8; + for i in 0..a.len().max(b.len()) { + diff |= a.get(i).copied().unwrap_or(0) ^ b.get(i).copied().unwrap_or(0); + } + diff == 0 +} + +fn handle(mut stream: TcpStream, config: &Config) -> Result<()> { + stream.set_read_timeout(Some(READ_TIMEOUT))?; + stream.set_write_timeout(Some(READ_TIMEOUT))?; + + let Some(request) = read_request(&stream)? else { + return Ok(()); + }; + + // Echo the origin only when it is on the allowlist. Never `*`: this API describes the + // machine it runs on, so any-origin access means any page can fingerprint the node. + let allow_origin = request + .origin + .as_ref() + .filter(|o| config.allowed_origins.iter().any(|a| a == *o)) + .cloned(); + + if request.method == "OPTIONS" { + return write_response(&mut stream, 204, "", allow_origin.as_deref(), true); + } + + if let Some(expected) = &config.token { + if !token_ok(expected, request.authorization.as_ref()) { + return write_json( + &mut stream, + 401, + r#"{"error":"unauthorized"}"#, + allow_origin.as_deref(), + ); + } + } + + if request.method != "GET" { + return write_json( + &mut stream, + 405, + r#"{"error":"this API is read-only"}"#, + allow_origin.as_deref(), + ); + } + + let (status, body) = route(&request.path, &request.query, &config.root); + write_json(&mut stream, status, &body, allow_origin.as_deref()) +} + +fn route(path: &str, query: &BTreeMap, root: &Path) -> (u16, String) { + let since = query.get("since").map(String::as_str).unwrap_or("24h"); + let rendered = match path { + "/v1/health" => Ok(r#"{"ok":true,"service":"lumbridge-compute","api":"v1"}"#.to_string()), + "/v1/status" => collect_status(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/models" => collect_models(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/scenes" => collect_scenes(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/evals" => collect_evals(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/usage" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report)?)), + "/v1/usage/agents" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report.agents)?)), + "/v1/usage/concurrency" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report.concurrency)?)), + other => match other.strip_prefix("/v1/scenes/") { + // Exactly one segment: /v1/scenes/a/b is not a route. + Some(name) if !name.is_empty() && !name.contains('/') => { + collect_scene(root, name).and_then(|r| Ok(serde_json::to_string(&r)?)) + } + _ => return (404, r#"{"error":"no such route"}"#.to_string()), + }, + }; + + match rendered { + Ok(body) => (200, body), + // A collector fails when the thing does not exist (an unknown Scene) or when the node's + // own config is unreadable. The message is the operator's, and this is a loopback API, + // so passing it through is more useful than flattening it to "error". + Err(error) => ( + 404, + serde_json::json!({ "error": format!("{error:#}") }).to_string(), + ), + } +} + +fn write_json(stream: &mut TcpStream, status: u16, body: &str, origin: Option<&str>) -> Result<()> { + write_response(stream, status, body, origin, false) +} + +fn write_response( + stream: &mut TcpStream, + status: u16, + body: &str, + origin: Option<&str>, + preflight: bool, +) -> Result<()> { + let reason = match status { + 200 => "OK", + 204 => "No Content", + 401 => "Unauthorized", + 404 => "Not Found", + 405 => "Method Not Allowed", + _ => "Error", + }; + let mut head = format!("HTTP/1.1 {status} {reason}\r\n"); + head.push_str("Content-Type: application/json\r\n"); + head.push_str(&format!("Content-Length: {}\r\n", body.len())); + // This is live node state; a cached answer is a wrong answer. + head.push_str("Cache-Control: no-store\r\n"); + head.push_str("Connection: close\r\n"); + if let Some(origin) = origin { + head.push_str(&format!("Access-Control-Allow-Origin: {origin}\r\n")); + // Tell caches the body varies by origin, so an allowed origin's response can never be + // replayed to a disallowed one. + head.push_str("Vary: Origin\r\n"); + if preflight { + head.push_str("Access-Control-Allow-Methods: GET, OPTIONS\r\n"); + head.push_str("Access-Control-Allow-Headers: Authorization\r\n"); + head.push_str("Access-Control-Max-Age: 600\r\n"); + } + } + head.push_str("\r\n"); + stream.write_all(head.as_bytes())?; + stream.write_all(body.as_bytes())?; + stream.flush()?; + Ok(()) +} + +/// Drain and discard — kept for symmetry with future routes that accept a body. +#[allow(dead_code)] +fn discard_body(reader: &mut impl Read) { + let mut sink = Vec::new(); + let _ = reader.read_to_end(&mut sink); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_comparison_rejects_wrong_and_missing_and_prefixless() { + let t = "s3cret-token"; + assert!(token_ok(t, Some(&format!("Bearer {t}")))); + assert!(!token_ok(t, None)); + assert!(!token_ok(t, Some(&t.to_string()))); // no "Bearer " prefix + assert!(!token_ok(t, Some(&"Bearer wrong".to_string()))); + // A correct prefix must not pass — this is the bug constant-time comparison exists for. + assert!(!token_ok(t, Some(&"Bearer s3cret".to_string()))); + assert!(!token_ok(t, Some(&"Bearer s3cret-token-plus".to_string()))); + } + + #[test] + fn unknown_routes_404_and_scene_paths_take_exactly_one_segment() { + let root = Path::new("/nonexistent-root-for-routing-test"); + let query = BTreeMap::new(); + assert_eq!(route("/v1/nope", &query, root).0, 404); + assert_eq!(route("/v1/scenes/a/b", &query, root).0, 404); + assert_eq!(route("/v1/scenes/", &query, root).0, 404); + // Health needs no filesystem, so it answers even on a bogus root. + assert_eq!(route("/v1/health", &query, root).0, 200); + } + + #[test] + fn health_body_is_valid_json() { + let (status, body) = route("/v1/health", &BTreeMap::new(), Path::new("/tmp")); + assert_eq!(status, 200); + let parsed: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(parsed["ok"], true); + } +} diff --git a/src/lifecycle.rs b/src/lifecycle.rs new file mode 100644 index 0000000..95181eb --- /dev/null +++ b/src/lifecycle.rs @@ -0,0 +1,915 @@ +//! Transactional Scene lifecycle and persisted desired-state recovery. + +use anyhow::{bail, Context, Result}; +use std::collections::{BTreeMap, HashSet}; +use std::fs::{self, File, OpenOptions}; +use std::os::fd::AsRawFd; +use std::path::Path; +use std::time::Duration; + +use crate::config::{find_scene, Model, Registry, Scene}; +use crate::governor; +use crate::mem; +use crate::proc::{self, Proc, State}; + +/// What activating a Scene would do, decided before anything is mutated. +/// +/// `activate` renders this for the CLI and the MCP server returns it verbatim, +/// so a plan an operator reads and a plan an agent reads can never drift apart. +#[derive(Debug, Clone)] +pub struct Plan { + pub scene: String, + pub budget_gb: f64, + /// Registered models serving outside the target Scene; stopped first. + pub stop: Vec, + /// Scene models not yet serving, in the order they would be admitted. + pub start: Vec, +} + +pub struct TransitionLock { + file: File, +} + +impl TransitionLock { + pub fn acquire(root: &Path) -> Result { + let dir = root.join(".compute"); + fs::create_dir_all(&dir)?; + let path = dir.join("transition.lock"); + let file = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("opening transition lock {}", path.display()))?; + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if result != 0 { + return Err(std::io::Error::last_os_error()) + .context("another Scene transition is already running; wait for it to finish"); + } + Ok(Self { file }) + } +} + +impl Drop for TransitionLock { + fn drop(&mut self) { + unsafe { + libc::flock(self.file.as_raw_fd(), libc::LOCK_UN); + } + } +} + +pub fn adopt(root: &Path, name: &str) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + + let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); + let running = governor::running_ids(®istry); + let extras: Vec = running + .iter() + .filter(|id| !target.contains(id.as_str())) + .cloned() + .collect(); + if !extras.is_empty() { + bail!( + "cannot adopt '{name}': registered model(s) outside the Scene are serving: {}", + extras.join(", ") + ); + } + + let previous = State::load_checked(root)?; + let mut adopted = BTreeMap::new(); + for id in &scene.models { + let model = ®istry.models[id]; + if !governor::is_running(model) { + bail!("cannot adopt '{name}': exact model '{id}' is not healthy"); + } + let legacy = previous.procs.get(id).with_context(|| { + format!( + "cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute" + ) + })?; + let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port) + .with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?; + adopted.insert(id.clone(), captured); + } + + let mut state = previous; + state.procs = adopted; + state.desired_scene = Some(name.to_string()); + state.active_scene = Some(name.to_string()); + state.last_known_good_scene = Some(name.to_string()); + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("adopted exact running Scene '{name}' and captured process identities"); + Ok(()) +} + +pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let prior = State::load_checked(root)?; + let prior_active = prior.active_scene.clone(); + + if dry_run { + return activate_once(root, name, true); + } + + match activate_once(root, name, false) { + Ok(()) => { + let mut state = State::load_checked(root)?; + if prior_active.as_deref() != Some(name) { + if let Some(previous) = prior_active { + state.last_known_good_scene = Some(previous); + } + } + state.desired_scene = Some(name.to_string()); + state.active_scene = Some(name.to_string()); + if state.last_known_good_scene.is_none() { + state.last_known_good_scene = Some(name.to_string()); + } + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("Scene '{name}' is active and persisted as desired"); + Ok(()) + } + Err(target_error) => { + let target_message = format!("activating '{name}' failed: {target_error:#}"); + eprintln!("{target_message}"); + let transition_started = + State::load_checked(root)?.transition_scene.as_deref() == Some(name); + // Where a failed transition would roll back to. Absent when the scene being + // activated is already the active one, which is the ordinary case for + // "start the models in this scene that are currently down". + let rollback_target = prior_active.as_deref().filter(|previous| *previous != name); + // Only tear the box down when there is somewhere to put it back. + // + // This used to run whenever a transition had begun, including when the target + // scene *was* the prior scene — so a refused admission stopped every running + // model and then skipped the rollback, because the rollback target had been + // filtered out for being the same scene. Twice on 2026-08-14 that turned one + // model failing admission into an empty box, taking brain-nemotron and music + // down with it. + // + // With no rollback target, leaving the partially-started scene running is + // strictly better: those models are healthy and they belong to the scene that + // was asked for. The refusal is reported either way. + let cleanup_error = if transition_started && rollback_target.is_some() { + stop_all_owned(root).err() + } else { + None + }; + let rollback = if transition_started && cleanup_error.is_none() { + rollback_target + .map(|previous| (previous.to_string(), activate_once(root, previous, false))) + } else { + None + }; + + let mut state = State::load_checked(root)?; + state.transition_scene = None; + state.last_error = Some(target_message.clone()); + match rollback { + Some((previous, Ok(()))) => { + state.desired_scene = Some(previous.clone()); + state.active_scene = Some(previous.clone()); + if state.last_known_good_scene.is_none() { + state.last_known_good_scene = Some(previous.clone()); + } + state.save(root)?; + eprintln!("rolled back to Scene '{previous}'"); + bail!("{target_message}; rolled back to '{previous}'") + } + Some((previous, Err(rollback_error))) => { + state.active_scene = None; + state.save(root)?; + bail!( + "{target_message}; rollback to '{previous}' also failed: {rollback_error:#}" + ) + } + None if !transition_started => { + state.save(root)?; + bail!("{target_message}") + } + None if cleanup_error.is_none() => { + state.active_scene = None; + state.save(root)?; + bail!("{target_message}") + } + None => { + state.active_scene = None; + state.save(root)?; + let cleanup_error = cleanup_error.expect("guarded by match condition"); + bail!("{target_message}; cleanup also failed: {cleanup_error:#}") + } + } + } + } +} + +fn stop_all_owned(root: &Path) -> Result<()> { + let mut state = State::load_checked(root)?; + let owned: Vec<(String, Proc)> = state + .procs + .iter() + .filter(|(_, process)| process.owned_alive()) + .map(|(id, process)| (id.clone(), process.clone())) + .collect(); + for (id, process) in owned { + proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?; + state.procs.remove(&id); + state.save(root)?; + } + Ok(()) +} + +pub fn resume(root: &Path) -> Result<()> { + let state = State::load_checked(root)?; + let desired = state + .desired_scene + .clone() + .or_else(|| state.last_known_good_scene.clone()) + .context("no desired Scene is persisted; activate or adopt one first")?; + let fallback = state.last_known_good_scene.clone(); + + match activate(root, &desired, false) { + Ok(()) => Ok(()), + Err(desired_error) => { + let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else { + return Err(desired_error).context("resuming desired Scene"); + }; + eprintln!( + "desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'" + ); + activate(root, &fallback, false).with_context(|| { + format!( + "desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed" + ) + }) + } + } +} + +/// Restart one explicitly supervised model without disturbing the other models +/// in the desired Scene. +/// +/// This is deliberately narrower than `activate`: a resident recovery attempt +/// must not turn one failed voice runtime into a stop-all rollback of an +/// otherwise healthy Scene. The same transition lock, identity ownership, +/// declared/observed admission checks and exact health gate still apply. +/// Returns `true` when a new process was started and `false` when the model +/// recovered before the lock was acquired. +pub fn restart_supervised_model(root: &Path, id: &str, timeout: Duration) -> Result { + let _lock = TransitionLock::acquire(root)?; + let result = restart_supervised_model_locked(root, id, timeout); + if let Err(error) = &result { + if let Ok(mut state) = State::load_checked(root) { + state.active_scene = None; + state.last_error = Some(format!("supervision could not restart '{id}': {error:#}")); + let _ = state.save(root); + } + } + result +} + +fn restart_supervised_model_locked(root: &Path, id: &str, timeout: Duration) -> Result { + let registry = Registry::load(root)?; + let model = registry + .models + .get(id) + .with_context(|| format!("no registered model '{id}'"))?; + model + .supervision + .as_ref() + .with_context(|| format!("model '{id}' did not opt into supervision"))?; + let port = model + .serve + .port + .with_context(|| format!("supervised model '{id}' has no serving port"))?; + + let mut state = State::load_checked(root)?; + let desired = state + .desired_scene + .clone() + .context("no desired Scene is persisted")?; + let scene = find_scene(root, &desired)?; + validate_scene(®istry, &scene)?; + if !scene.models.iter().any(|candidate| candidate == id) { + bail!("model '{id}' is not part of desired Scene '{desired}'"); + } + + // The probe that triggered supervision ran before the transition lock. A + // late request or a runtime finishing its own recovery may have repaired the + // model in that gap, so re-check before signalling anything. + if governor::is_running(model) { + return Ok(false); + } + + match state.procs.get(id).cloned() { + Some(process) if process.owned_alive() => { + // Health is bad but this is still the exact process Compute started. + // Stop only its process group; no sibling in the Scene is touched. + proc::stop_owned(&process)?; + } + Some(_) if governor::port_open(port) => { + bail!( + "port {port} is occupied but the ownership record for '{id}' is stale; refusing to signal it" + ); + } + None if governor::port_open(port) => { + bail!( + "port {port} is occupied by an unowned process; refusing to replace supervised model '{id}'" + ); + } + Some(_) | None => {} + } + + state.procs.remove(id); + state.active_scene = None; + state.last_error = Some(format!("supervision is restarting '{id}'")); + state.save(root)?; + + // Re-read both ceilings after the old process has released its memory. The + // registry number prevents paper overcommit; MemAvailable prevents an + // unregistered workload from turning a nominally valid restart into a wedged + // unified-memory node. + let committed = governor::committed_gb(®istry); + let available = mem::read().ok().map(|memory| memory.available_gb); + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + if !governor::can_admit(model.footprint_gb, committed, budget, available) { + match available { + Some(available) => bail!( + "'{id}' ({:.0} GB) restart refused: {:.0} GB committed against a {:.0} GB budget, \ + {:.1} GB actually available, {:.0} GB margin required", + model.footprint_gb, + committed, + budget, + available, + governor::SAFETY_MARGIN_GB + ), + None => bail!( + "'{id}' restart would exceed the {:.0} GB Scene budget with the safety margin", + budget + ), + } + } + + let pid = proc::spawn(root, id, model)?; + state.seq += 1; + let process = Proc::capture(pid, state.seq, model.serve.port) + .with_context(|| format!("capturing ownership for supervised '{id}'"))?; + state.procs.insert(id.to_string(), process.clone()); + state.save(root)?; + + if !governor::wait_healthy(model, timeout) { + let _ = proc::stop_owned(&process); + state.procs.remove(id); + state.last_error = Some(format!( + "supervised model '{id}' did not report exact health within {} seconds", + timeout.as_secs() + )); + state.save(root)?; + bail!( + "supervised model '{id}' did not report exact health within {} seconds", + timeout.as_secs() + ); + } + + let unhealthy: Vec = scene + .models + .iter() + .filter(|candidate| !governor::is_running(®istry.models[*candidate])) + .cloned() + .collect(); + if unhealthy.is_empty() { + state.active_scene = Some(desired); + state.last_error = None; + } else { + state.active_scene = None; + state.last_error = Some(format!( + "supervised model '{id}' recovered, but desired Scene is still unhealthy: {}", + unhealthy.join(", ") + )); + } + state.save(root)?; + Ok(true) +} + +pub fn deactivate(root: &Path) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let mut state = State::load_checked(root)?; + let ids: Vec = state.procs.keys().cloned().collect(); + for id in ids { + let process = state.procs[&id].clone(); + if process.owned_alive() { + println!(" stopping {id} (pid {})", process.pid); + proc::stop_owned(&process)?; + } + state.procs.remove(&id); + state.save(root)?; + } + state.desired_scene = None; + state.active_scene = None; + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("all Compute-managed models stopped; no Scene is desired"); + Ok(()) +} + +/// Diff the target Scene against what is actually serving. Pure: it decides +/// *what* would change, never *whether* the change is allowed. +fn plan_transition(registry: &Registry, scene: &Scene) -> Plan { + let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); + let running = governor::running_ids(registry); + let stop: Vec = running + .iter() + .filter(|id| !target.contains(id.as_str())) + .cloned() + .collect(); + let mut start: Vec = scene + .models + .iter() + .filter(|id| !governor::is_running(®istry.models[*id])) + .cloned() + .collect(); + let listed = scene + .activation + .as_ref() + .and_then(|activation| activation.order.as_deref()) + .map(|order| order == "listed") + .unwrap_or(false); + if !listed { + start.sort_by(|a, b| { + registry.models[a] + .footprint_gb + .partial_cmp(®istry.models[b].footprint_gb) + .unwrap() + }); + } + Plan { + scene: scene.metadata.name.clone(), + budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB), + stop, + start, + } +} + +/// Everything that must hold before the first process is signalled. Runs to +/// completion with nothing mutated, so a rejection here leaves the currently +/// active Scene exactly as it was. +fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> { + for id in &scene.models { + if governor::is_running(®istry.models[id]) { + let process = state.procs.get(id).with_context(|| { + format!( + "'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first" + ) + })?; + if !process.owned_alive() { + bail!( + "ownership record for serving model '{id}' is stale; adopt the active Scene first" + ); + } + } + } + + for id in &plan.stop { + let process = state.procs.get(id).with_context(|| { + format!( + "'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching" + ) + })?; + if !process.owned_alive() { + bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid"); + } + } + for id in &plan.start { + let model = ®istry.models[id]; + if let Some(port) = model.serve.port { + // A model in `plan.stop` may currently hold this port. Every stop runs before + // any start, so that is a handoff, not a conflict. Without this exemption every + // same-port swap is rejected — including brain -> brain-laguna/brain-gemma, + // which share :8001 by design because the alias downstream agents call must + // survive a weight swap. + let freed_by_stop = plan.stop.iter().any(|stopping| { + registry + .models + .get(stopping) + .and_then(|stopped| stopped.serve.port) + == Some(port) + }); + if !freed_by_stop && governor::port_open(port) && !governor::is_running(model) { + bail!("port {port} is occupied by a different model; refusing to start '{id}'"); + } + } + } + Ok(()) +} + +/// `scene activate --dry-run` as data rather than as printed lines: the same +/// validation, the same plan, nothing written. Callers that need the plan +/// programmatically use this instead of scraping stdout. +pub fn plan(root: &Path, name: &str) -> Result { + let _lock = TransitionLock::acquire(root)?; + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + let plan = plan_transition(®istry, &scene); + let mut state = State::load_checked(root)?; + state.procs.retain(|_, process| process.owned_alive()); + preflight(®istry, &scene, &plan, &state)?; + Ok(plan) +} + +fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> { + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + let plan = plan_transition(®istry, &scene); + + println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb); + println!(" stop : {}", format_ids(&plan.stop)); + println!(" start: {}", format_ids(&plan.start)); + + let mut state = State::load_checked(root)?; + state.procs.retain(|_, process| process.owned_alive()); + preflight(®istry, &scene, &plan, &state)?; + + if dry_run { + println!(" dry run: validation passed; nothing changed"); + return Ok(()); + } + + state.transition_scene = Some(name.to_string()); + state.save(root)?; + + for id in &plan.stop { + let process = state.procs[id].clone(); + println!(" stopping {id} (pid {})", process.pid); + proc::stop_owned(&process)?; + state.procs.remove(id); + state.save(root)?; + } + + let mut committed: f64 = scene + .models + .iter() + .filter(|id| governor::is_running(®istry.models[*id])) + .map(|id| registry.models[id].footprint_gb) + .sum(); + let wait_healthy = scene + .activation + .as_ref() + .and_then(|activation| activation.wait_healthy) + .unwrap_or(true); + + for id in &plan.start { + let model: &Model = ®istry.models[id]; + // Re-read the pool before every start rather than once per activation: each model + // that comes up consumes real memory, and its true appetite is only knowable after + // it has allocated. A footprint that was optimistic shows up here, on the next + // model, instead of taking the box down. + let available_gb = mem::read().ok().map(|m| m.available_gb); + if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) { + match available_gb { + Some(available) => bail!( + "'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \ + {:.1} GB actually available, {:.0} GB margin required", + model.footprint_gb, + committed, + plan.budget_gb, + available, + governor::SAFETY_MARGIN_GB + ), + None => bail!( + "'{id}' would exceed the {:.0} GB Scene budget with the safety margin", + plan.budget_gb + ), + } + } + let pid = proc::spawn(root, id, model)?; + state.seq += 1; + let process = Proc::capture(pid, state.seq, model.serve.port) + .with_context(|| format!("capturing ownership for newly started '{id}'"))?; + state.procs.insert(id.clone(), process.clone()); + state.save(root)?; + committed += model.footprint_gb; + println!(" started {id} (pid {pid}); waiting for exact health"); + if wait_healthy && !governor::wait_healthy(model, health_timeout()) { + let _ = proc::stop_owned(&process); + state.procs.remove(id); + state.save(root)?; + bail!("'{id}' did not report its exact health marker before timeout"); + } + } + + let unhealthy: Vec = scene + .models + .iter() + .filter(|id| !governor::is_running(®istry.models[*id])) + .cloned() + .collect(); + if !unhealthy.is_empty() { + bail!( + "Scene '{name}' is incomplete; exact health failed for: {}", + unhealthy.join(", ") + ); + } + Ok(()) +} + +fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> { + let missing: Vec<&String> = scene + .models + .iter() + .filter(|id| !registry.models.contains_key(*id)) + .collect(); + if !missing.is_empty() { + bail!( + "Scene '{}' references unknown model id(s): {}", + scene.metadata.name, + missing + .iter() + .map(|id| id.as_str()) + .collect::>() + .join(", ") + ); + } + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + let footprint: f64 = scene + .models + .iter() + .map(|id| registry.models[id].footprint_gb) + .sum(); + if footprint + governor::SAFETY_MARGIN_GB > budget { + bail!( + "Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget", + scene.metadata.name, + footprint + governor::SAFETY_MARGIN_GB, + budget + ); + } + Ok(()) +} + +fn health_timeout() -> Duration { + Duration::from_secs(if cfg!(test) { 5 } else { 900 }) +} + +fn format_ids(ids: &[String]) -> String { + if ids.is_empty() { + "(none)".to_string() + } else { + ids.join(", ") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn fixture_root() -> (PathBuf, u16) { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}")); + fs::create_dir_all(root.join("registry")).unwrap(); + fs::create_dir_all(root.join("scenes")).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let bad_port = bad_listener.local_addr().unwrap().port(); + drop(bad_listener); + let registry = format!( + "apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n supervision:\n restart: always\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n" + ); + fs::write(root.join("registry/models.yaml"), registry).unwrap(); + fs::write( + root.join("scenes/test.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n", + ) + .unwrap(); + (root, port) + } + + #[test] + fn activation_persists_and_deactivation_stops_owned_process() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let state = State::load_checked(&root).unwrap(); + assert_eq!(state.desired_scene.as_deref(), Some("test")); + assert_eq!(state.active_scene.as_deref(), Some("test")); + assert!(state.procs["fake"].owned_alive()); + deactivate(&root).unwrap(); + assert!(State::load_checked(&root).unwrap().procs.is_empty()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn supervision_restarts_only_the_failed_owned_model() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let before = State::load_checked(&root).unwrap(); + let old = before.procs["fake"].clone(); + proc::stop_owned(&old).unwrap(); + + assert!(restart_supervised_model(&root, "fake", Duration::from_secs(5)).unwrap()); + let after = State::load_checked(&root).unwrap(); + assert_ne!(after.procs["fake"].pid, old.pid); + assert!(after.procs["fake"].owned_alive()); + assert_eq!(after.desired_scene.as_deref(), Some("test")); + assert_eq!(after.active_scene.as_deref(), Some("test")); + + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn supervision_refuses_to_replace_an_unowned_process() { + let (root, port) = fixture_root(); + let registry = Registry::load(&root).unwrap(); + let model = ®istry.models["fake"]; + let pid = proc::spawn(&root, "fake", model).unwrap(); + assert!(governor::wait_healthy(model, Duration::from_secs(5))); + + // Make exact health fail while the unowned server continues to hold the + // port. Supervision must not infer ownership from a port number. + let registry_path = root.join("registry/models.yaml"); + let contents = fs::read_to_string(®istry_path).unwrap(); + fs::write( + ®istry_path, + contents.replace( + &format!("health: http://localhost:{port}/"), + &format!( + "health: http://localhost:{port}/\n health_contains: marker-that-is-not-served" + ), + ), + ) + .unwrap(); + fs::create_dir_all(root.join(".compute")).unwrap(); + State { + desired_scene: Some("test".to_string()), + active_scene: Some("test".to_string()), + last_known_good_scene: Some("test".to_string()), + ..State::default() + } + .save(&root) + .unwrap(); + + let error = restart_supervised_model(&root, "fake", Duration::from_millis(50)) + .unwrap_err() + .to_string(); + assert!(error.contains("unowned process")); + assert!(governor::port_open(port)); + + let owned = Proc::capture(pid, 1, Some(port)).unwrap(); + proc::stop_owned(&owned).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + /// Two models sharing one port, distinguishable by `health_contains` — the + /// brain/brain-laguna shape. Each serves its own directory so the health probe + /// can tell which one is actually up. + fn same_port_root() -> PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("lumbridge-compute-swap-{unique}")); + fs::create_dir_all(root.join("registry")).unwrap(); + fs::create_dir_all(root.join("scenes")).unwrap(); + let dir_a = root.join("srv-alpha"); + let dir_b = root.join("srv-beta"); + fs::create_dir_all(&dir_a).unwrap(); + fs::create_dir_all(&dir_b).unwrap(); + fs::write(dir_a.join("alpha-marker.txt"), "a").unwrap(); + fs::write(dir_b.join("beta-marker.txt"), "b").unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let (a, b) = (dir_a.display(), dir_b.display()); + let registry = format!( + "apiVersion: lumbridge/v1\nmodels:\n alpha:\n name: Alpha\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: alpha-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{a}']\n beta:\n name: Beta\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: beta-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{b}']\n" + ); + fs::write(root.join("registry/models.yaml"), registry).unwrap(); + fs::write( + root.join("scenes/a.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: a\n version: 1\nmodels: [alpha]\nbudget_gb: 100\n", + ) + .unwrap(); + fs::write( + root.join("scenes/b.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: b\n version: 1\nmodels: [beta]\nbudget_gb: 100\n", + ) + .unwrap(); + root + } + + #[test] + fn same_port_swap_is_allowed_when_the_occupant_is_being_stopped() { + let root = same_port_root(); + activate(&root, "a", false).unwrap(); + assert!(State::load_checked(&root) + .unwrap() + .procs + .contains_key("alpha")); + + // Regression: this used to fail with "port N is occupied by a different model; + // refusing to start 'beta'". The pre-flight port check ran over `plan.start` + // without exempting ports released by `plan.stop`, so every same-port swap was + // rejected even though stops precede starts. + activate(&root, "b", false).unwrap(); + + let state = State::load_checked(&root).unwrap(); + assert_eq!(state.active_scene.as_deref(), Some("b")); + assert!(state.procs.contains_key("beta")); + assert!(!state.procs.contains_key("alpha")); + + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn already_serving_scene_requires_explicit_adoption() { + let (root, port) = fixture_root(); + let registry = Registry::load(&root).unwrap(); + let model = ®istry.models["fake"]; + let pid = proc::spawn(&root, "fake", model).unwrap(); + assert!(governor::wait_healthy(model, Duration::from_secs(5))); + let error = activate(&root, "test", true).unwrap_err().to_string(); + assert!(error.contains("adopt the active Scene")); + let owned = Proc::capture(pid, 1, Some(port)).unwrap(); + proc::stop_owned(&owned).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn preflight_rejection_preserves_healthy_active_scene() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let before = State::load_checked(&root).unwrap(); + let pid = before.procs["fake"].pid; + fs::write( + root.join("scenes/invalid.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n", + ) + .unwrap(); + assert!(activate(&root, "invalid", false).is_err()); + let after = State::load_checked(&root).unwrap(); + assert_eq!(after.active_scene.as_deref(), Some("test")); + assert_eq!(after.desired_scene.as_deref(), Some("test")); + assert_eq!(after.procs["fake"].pid, pid); + assert!(after.procs["fake"].owned_alive()); + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn failed_activation_keeps_healthy_partial_scene_without_rollback_target() { + let (root, port) = fixture_root(); + fs::write( + root.join("scenes/failing.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n", + ) + .unwrap(); + assert!(activate(&root, "failing", false).is_err()); + let state = State::load_checked(&root).unwrap(); + assert!(state.procs.contains_key("fake")); + assert!(!state.procs.contains_key("bad")); + assert!(state.active_scene.is_none()); + assert!(governor::port_open(port)); + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn checked_state_load_rejects_malformed_yaml() { + let (root, _port) = fixture_root(); + fs::create_dir_all(root.join(".compute")).unwrap(); + fs::write(root.join(".compute/state.yaml"), "desired_scene: [").unwrap(); + assert!(State::load_checked(&root).is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn state_write_replaces_complete_yaml_atomically() { + let (root, _port) = fixture_root(); + let state = State { + desired_scene: Some("test".to_string()), + ..State::default() + }; + state.save(&root).unwrap(); + let loaded = State::load_checked(&root).unwrap(); + assert_eq!(loaded.desired_scene.as_deref(), Some("test")); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..fbc6806 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,561 @@ +//! Lumbridge Compute — safe AI workload orchestration for accelerator nodes. + +mod agent; +mod config; +mod eval; +mod gateway; +mod governor; +mod http; +mod lifecycle; +mod mcp; +mod mem; +mod proc; +mod telemetry; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use config::{find_scene, load_scenes, Registry, Scene}; +use proc::State; + +#[derive(Parser)] +#[command( + name = "lumbridge-compute", + version, + about = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes." +)] +struct Cli { + /// Root dir containing registry/ and scenes/ (default: $LUMBRIDGE_COMPUTE_ROOT or current dir) + #[arg(long, global = true)] + root: Option, + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Governor status: memory, budget, running set, headroom + Status, + /// Manage the model registry + Model { + #[command(subcommand)] + cmd: ModelCmd, + }, + /// Manage scenes (bundles of models) + Scene { + #[command(subcommand)] + cmd: SceneCmd, + }, + /// Run reproducible capability and serving-performance evaluations + Eval { + #[command(subcommand)] + cmd: EvalCmd, + }, + /// Inspect durable, privacy-safe model usage and concurrency history + Usage { + #[command(subcommand)] + cmd: UsageCmd, + }, + /// Run the memory watchdog in the foreground (kills the newest model before OOM-wedge) + Watchdog { + /// Kill the newest model if MemAvailable dips below this many GB + #[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)] + floor: f64, + }, + /// Run the stable streaming gateway in the foreground + Gateway { + #[arg(long, default_value = "127.0.0.1:8011")] + listen: String, + #[arg(long, default_value = "127.0.0.1:8001")] + upstream: String, + }, + /// Serve the Governor as a read-only JSON API over HTTP + /// + /// A second transport over the same operations the MCP server exposes, for a web UI. It + /// never mutates: no activation, no stop, no eval run. + Api { + /// Loopback by default. This API has no authentication unless --token is set, and it + /// reveals which models this node runs, so widening the bind is an explicit act. + #[arg(long, default_value = "127.0.0.1:8012")] + listen: String, + /// Browser origin permitted to read this API. Repeatable. Empty means no browser may + /// read it; `*` is deliberately not supported, because any page you visit would then + /// be able to fingerprint this machine. + #[arg(long = "allow-origin")] + allow_origin: Vec, + /// Require `Authorization: Bearer ` on every request. + #[arg(long)] + token: Option, + }, + /// Serve the Governor to AI agents as MCP tools over stdio + Mcp { + /// Let an agent perform a real Scene transition, not just plan one. + /// Off by default: the agent writes its own tool arguments, so the only + /// meaningful gate on a production switch is one the operator sets here. + #[arg(long)] + allow_activate: bool, + }, + /// Run the resident supervisor: resume desired Scene, gateway, and memory floor + Agent { + #[arg(long, default_value = "127.0.0.1:8011")] + listen: String, + #[arg(long, default_value = "127.0.0.1:8001")] + upstream: String, + #[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)] + floor: f64, + /// Local runtime Prometheus endpoint. SGLang must use --enable-metrics. + #[arg(long, default_value = "http://127.0.0.1:8001/metrics")] + metrics_url: String, + /// Registry model id for metrics, or auto to resolve the active Scene by port. + #[arg(long, default_value = "auto")] + metrics_model: String, + /// Sampling interval for time-weighted C0-C4 and queue history. + #[arg(long, default_value_t = 5)] + telemetry_interval_sec: u64, + /// Disable resident metrics collection while keeping supervision and the gateway. + #[arg(long)] + no_telemetry: bool, + }, +} + +#[derive(Subcommand)] +enum ModelCmd { + /// List registered models and their live state + Ls, +} + +#[derive(Subcommand)] +enum SceneCmd { + /// List scenes with total footprint + Ls, + /// Show a scene: models, footprints, and the Governor's admission verdict + Show { name: String }, + /// Activate a scene: stop what's not in it, admit + start what is + Activate { + name: String, + /// Print the plan without changing anything + #[arg(long)] + dry_run: bool, + }, + /// Adopt an already-running exact Scene and bind legacy PIDs to process identities + Adopt { name: String }, + /// Resume the persisted desired Scene, falling back to the previous known-good Scene + Resume, + /// Stop all Lumbridge Compute-managed models + Deactivate, +} + +#[derive(Subcommand)] +enum EvalCmd { + /// List available evaluation suites + Ls, + /// Run a suite against an OpenAI-compatible endpoint + Run { + suite: String, + #[arg(long, default_value = "http://127.0.0.1:8001/v1")] + base_url: String, + #[arg(long, default_value = "brain")] + model: String, + /// Override the suite's repetitions per case + #[arg(long)] + repeat: Option, + }, +} + +#[derive(Subcommand)] +enum UsageCmd { + /// Scrape the runtime once and append a durable snapshot + Collect { + #[arg(long, default_value = "http://127.0.0.1:8001/metrics")] + metrics_url: String, + #[arg(long, default_value = "auto")] + model: String, + }, + /// Report calls, tokens, vision, concurrency, queueing, and agent labels + Summary { + /// Window ending now, such as 24h, 7d, or 4w. + #[arg(long, default_value = "24h")] + since: String, + #[arg(long)] + json: bool, + }, + /// Report bounded client/agent/workload usage labels + Agents { + #[arg(long, default_value = "7d")] + since: String, + #[arg(long)] + json: bool, + }, + /// Report the time-sampled C0-C4+ distribution and queue pressure + Concurrency { + #[arg(long, default_value = "7d")] + since: String, + #[arg(long)] + json: bool, + }, +} + +fn root_dir(cli: &Cli) -> PathBuf { + cli.root + .clone() + .or_else(|| { + std::env::var("LUMBRIDGE_COMPUTE_ROOT") + .ok() + .map(PathBuf::from) + }) + .unwrap_or_else(|| PathBuf::from(".")) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = root_dir(&cli); + match &cli.cmd { + Cmd::Status => cmd_status(&root), + Cmd::Model { cmd } => match cmd { + ModelCmd::Ls => cmd_model_ls(&root), + }, + Cmd::Scene { cmd } => match cmd { + SceneCmd::Ls => cmd_scene_ls(&root), + SceneCmd::Show { name } => cmd_scene_show(&root, name), + SceneCmd::Activate { name, dry_run } => lifecycle::activate(&root, name, *dry_run), + SceneCmd::Adopt { name } => lifecycle::adopt(&root, name), + SceneCmd::Resume => lifecycle::resume(&root), + SceneCmd::Deactivate => lifecycle::deactivate(&root), + }, + Cmd::Eval { cmd } => match cmd { + EvalCmd::Ls => eval::list(&root), + EvalCmd::Run { + suite, + base_url, + model, + repeat, + } => eval::run(&root, suite, base_url, model, *repeat), + }, + Cmd::Usage { cmd } => cmd_usage(&root, cmd), + Cmd::Watchdog { floor } => cmd_watchdog(&root, *floor), + Cmd::Gateway { listen, upstream } => gateway::run(listen, upstream), + Cmd::Api { + listen, + allow_origin, + token, + } => http::run( + listen, + http::Config { + root: root.clone(), + allowed_origins: allow_origin.clone(), + token: token.clone(), + }, + ), + Cmd::Mcp { allow_activate } => mcp::run(&root, *allow_activate), + Cmd::Agent { + listen, + upstream, + floor, + metrics_url, + metrics_model, + telemetry_interval_sec, + no_telemetry, + } => agent::run( + &root, + listen, + upstream, + *floor, + if *no_telemetry { + None + } else { + Some(agent::TelemetryConfig { + metrics_url: metrics_url.clone(), + model: metrics_model.clone(), + interval: Duration::from_secs((*telemetry_interval_sec).max(1)), + }) + }, + ), + } +} + +fn cmd_usage(root: &Path, command: &UsageCmd) -> Result<()> { + match command { + UsageCmd::Collect { metrics_url, model } => { + let result = telemetry::collect_once(root, metrics_url, model)?; + println!("{}", serde_json::to_string_pretty(&result)?); + } + UsageCmd::Summary { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + println!("{}", telemetry::render_text(&report)); + } + } + UsageCmd::Agents { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report.agents)?); + } else if report.agents.is_empty() { + println!("no agent-labelled usage in this window"); + } else { + for row in report.agents { + println!( + "{}/{}/{} {:.0} requests · {:.0} prompt · {:.0} generated tokens", + row.client, + row.agent, + row.workload, + row.requests, + row.prompt_tokens, + row.generation_tokens + ); + } + } + } + UsageCmd::Concurrency { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report.concurrency)?); + } else if report.concurrency.samples == 0 { + println!("no concurrency samples in this window"); + } else { + for bucket in report.concurrency.distribution { + println!( + "{} {} samples · {:.1}%", + bucket.concurrency, + bucket.samples, + bucket.share * 100.0 + ); + } + println!( + "peak running {} · peak queued {} · queue observed in {} sample(s)", + report.concurrency.peak_running, + report.concurrency.peak_queued, + report.concurrency.queued_samples + ); + } + } + } + Ok(()) +} + +fn cmd_status(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + let m = mem::read()?; + let committed = governor::committed_gb(®); + let running = governor::running_ids(®); + let budget = governor::DEFAULT_BUDGET_GB; + // The binding constraint, not the generous one: admission enforces the declared + // budget AND observed memory, so reporting only the declared headroom would promise + // room the next `scene activate` is going to refuse. + let headroom = governor::headroom_gb(committed, budget, Some(m.available_gb)).max(0.0); + let declared_headroom = (budget - committed - governor::SAFETY_MARGIN_GB).max(0.0); + let managed = State::load_checked(root)?; + + println!("Lumbridge Compute · governor"); + println!( + " memory {:.1} GB total · {:.1} GB available", + m.total_gb, m.available_gb + ); + println!( + " budget {:.0} GB (safety margin {:.0} · watchdog floor {:.0})", + budget, + governor::SAFETY_MARGIN_GB, + governor::WATCHDOG_FLOOR_GB + ); + println!( + " committed {committed:.1} GB across {} model(s)", + running.len() + ); + if headroom < declared_headroom { + println!( + " headroom {headroom:.1} GB admittable (budget allows {declared_headroom:.1}; \ + MemAvailable is the tighter limit)" + ); + } else { + println!(" headroom {headroom:.1} GB admittable"); + } + println!(); + println!( + " scene active={} desired={} fallback={}", + managed.active_scene.as_deref().unwrap_or("-"), + managed.desired_scene.as_deref().unwrap_or("-"), + managed.last_known_good_scene.as_deref().unwrap_or("-") + ); + if let Some(error) = &managed.last_error { + println!(" last error {error}"); + } + + if running.is_empty() { + println!(" (no registered models currently serving)"); + } else { + println!(" running:"); + for id in &running { + let mdl = ®.models[id]; + let tag = if managed + .procs + .get(id) + .is_some_and(|process| process.owned_alive()) + { + "compute" + } else { + "ext " + }; + println!( + " ● [{tag}] {:<12} {:>5.0} GB :{:<5} {}", + id, + mdl.footprint_gb, + port_str(mdl.serve.port), + mdl.name + ); + } + } + Ok(()) +} + +fn cmd_model_ls(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + println!( + "{:<12} {:>6} {:<6} {:<6} MODEL", + "ID", "GB", "STATE", "PORT" + ); + for (id, m) in ®.models { + println!( + "{:<12} {:>6.0} {:<6} {:<6} {}", + id, + m.footprint_gb, + if governor::is_running(m) { + "up" + } else { + "down" + }, + port_str(m.serve.port), + m.name + ); + } + Ok(()) +} + +fn cmd_scene_ls(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + let scenes = load_scenes(root)?; + if scenes.is_empty() { + println!("no scenes in {}/scenes", root.display()); + return Ok(()); + } + println!("{:<12} {:>6} {:<30} MODELS", "SCENE", "GB", "DESCRIPTION"); + for s in &scenes { + let total = scene_footprint(s, ®); + println!( + "{:<12} {:>6.0} {:<30} {}", + s.metadata.name, + total, + truncate(&s.metadata.description, 30), + s.models.join(", ") + ); + } + Ok(()) +} + +fn cmd_scene_show(root: &Path, name: &str) -> Result<()> { + let reg = Registry::load(root)?; + let scene = find_scene(root, name)?; + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + + println!( + "scene {} (v{})", + scene.metadata.name, scene.metadata.version + ); + if !scene.metadata.description.is_empty() { + println!(" {}", scene.metadata.description); + } + println!(" budget {budget:.0} GB\n models:"); + + let mut total = 0.0; + let mut missing = Vec::new(); + for id in &scene.models { + match reg.models.get(id) { + Some(m) => { + total += m.footprint_gb; + let state = if governor::is_running(m) { + "up" + } else { + "down" + }; + println!( + " {:<12} {:>5.0} GB [{:<4}] {}", + id, m.footprint_gb, state, m.name + ); + } + None => { + missing.push(id.clone()); + println!(" {id:<12} ? [MISSING from registry]"); + } + } + } + + let needed = total + governor::SAFETY_MARGIN_GB; + println!( + "\n total footprint {total:.1} GB (+{:.0} safety = {needed:.1} GB)", + governor::SAFETY_MARGIN_GB + ); + let verdict = if !missing.is_empty() { + format!("✗ {} model(s) missing from registry", missing.len()) + } else if needed <= budget { + format!("✓ fits — {:.1} GB to spare", budget - needed) + } else { + format!("✗ exceeds budget by {:.1} GB", needed - budget) + }; + println!(" admission {verdict}"); + Ok(()) +} + +fn cmd_watchdog(root: &Path, floor: f64) -> Result<()> { + println!( + "Lumbridge Compute watchdog — killing the newest model if MemAvailable < {floor:.1} GB. Ctrl-C to stop." + ); + loop { + let m = mem::read()?; + if m.available_gb < floor { + let mut state = State::load_checked(root)?; + match state.newest_alive() { + Some((id, p)) => { + eprintln!( + "watchdog: MemAvailable {:.1} GB < floor {:.1} — killing newest '{id}' (pid {})", + m.available_gb, floor, p.pid + ); + proc::stop_owned(&p)?; + state.procs.remove(&id); + state.save(root)?; + } + None => eprintln!( + "watchdog: MemAvailable {:.1} GB < floor {:.1} but no Lumbridge Compute-managed model to kill!", + m.available_gb, floor + ), + } + } + std::thread::sleep(Duration::from_secs(1)); + } +} + +// ---- helpers --------------------------------------------------------------- + +fn scene_footprint(s: &Scene, reg: &Registry) -> f64 { + s.models + .iter() + .filter_map(|id| reg.models.get(id)) + .map(|m| m.footprint_gb) + .sum() +} + +fn port_str(p: Option) -> String { + p.map(|p| p.to_string()).unwrap_or_else(|| "-".to_string()) +} + +fn truncate(s: &str, n: usize) -> String { + if s.chars().count() <= n { + s.to_string() + } else { + format!( + "{}…", + s.chars().take(n.saturating_sub(1)).collect::() + ) + } +} diff --git a/src/mcp.rs b/src/mcp.rs new file mode 100644 index 0000000..ba7bd74 --- /dev/null +++ b/src/mcp.rs @@ -0,0 +1,617 @@ +//! MCP server: the Governor's surface, exposed to agents as tools. +//! +//! Agents already drive Compute by shelling out to the CLI and scraping the +//! column-aligned output. That works until a column moves. Speaking MCP means +//! the agent gets the same numbers the Governor reasons about, as data, with a +//! schema attached — and it means the *shape* of what an agent may do becomes +//! something this file decides rather than something `bash` decides. +//! +//! Two deliberate constraints shape everything below. +//! +//! **Read-first.** Every tool here except `scene_activate` is a pure read. +//! Activating a Scene stops every registered model that is not in it, which on +//! this box means taking down whatever is currently serving — a live voice +//! pipeline included. There is no undo an agent can reach for: if the target +//! Scene then fails to come up, recovery depends on rollback that may itself +//! fail. That asymmetry is why `scene adopt`, `scene resume`, and +//! `scene deactivate` are absent entirely; they are operator verbs whose +//! correctness depends on knowing what the box was doing five minutes ago. +//! `scene_activate` is exposed because planning a switch is genuinely the +//! useful thing an agent wants, and it defaults to planning only. +//! +//! **The safety boundary is the operator's, not the agent's.** A `dry_run` +//! argument defaulting to `true` documents intent but guards nothing: the agent +//! writes the arguments, so it can write `false`. The only boundary an agent +//! cannot cross is one set before it connects, so a real transition also +//! requires `lumbridge-compute mcp --allow-activate`, chosen by the human who +//! launched the server. Without that flag `dry_run: false` is refused, and the +//! refusal says so rather than silently planning instead. + +use anyhow::{Context, Result}; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::{Implementation, ServerCapabilities, ServerInfo}; +use rmcp::{tool, tool_handler, tool_router, ErrorData, Json, ServerHandler, ServiceExt}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::os::fd::FromRawFd; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::config::{find_scene, load_scenes, Registry}; +use crate::proc::State; +use crate::{eval, governor, lifecycle, mem}; + +pub fn run(root: &Path, allow_activate: bool) -> Result<()> { + let transport_stdout = hand_over_stdout()?; + // The banner has to go to stderr for the same reason everything else does; + // it doubles as confirmation to the operator that the mutating tool is off. + eprintln!( + "Lumbridge Compute MCP server on stdio · root {} · scene activation {}", + root.display(), + if allow_activate { + "ENABLED (--allow-activate)" + } else { + "disabled; plan only" + } + ); + + let server = ComputeMcp::new(root.to_path_buf(), allow_activate); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("building the MCP runtime")?; + runtime.block_on(async move { + let transport = ( + tokio::io::stdin(), + tokio::fs::File::from_std(transport_stdout), + ); + let service = server + .serve(transport) + .await + .context("negotiating the MCP stdio session")?; + service.waiting().await.context("serving MCP over stdio")?; + Ok(()) + }) +} + +/// Hand the real stdout to the transport and point fd 1 at stderr. +/// +/// On stdio transport fd 1 *is* the JSON-RPC framing, and this crate reports +/// progress with `println!` throughout — `lifecycle::activate` narrates every +/// stop and start. One such line interleaved into the framing desynchronises +/// the client mid-transition, which is the worst possible moment to lose it. +/// Rather than audit every print (and every future one), move the file +/// descriptor: library output lands on stderr, where operators already read +/// this server's logs, and the protocol gets a channel nothing else can write. +fn hand_over_stdout() -> Result { + let saved = unsafe { libc::dup(libc::STDOUT_FILENO) }; + if saved < 0 { + return Err(std::io::Error::last_os_error()) + .context("duplicating stdout for the MCP transport"); + } + if unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) } < 0 { + let error = std::io::Error::last_os_error(); + unsafe { libc::close(saved) }; + return Err(error).context("redirecting stdout to stderr"); + } + // SAFETY: `saved` is a fresh descriptor from `dup` that nothing else owns. + Ok(unsafe { std::fs::File::from_raw_fd(saved) }) +} + +#[derive(Clone)] +struct ComputeMcp { + root: Arc, + allow_activate: bool, + tool_router: ToolRouter, +} + +#[tool_router] +impl ComputeMcp { + fn new(root: PathBuf, allow_activate: bool) -> Self { + Self { + root: Arc::new(root), + allow_activate, + tool_router: Self::tool_router(), + } + } + + /// Governor status for this node: unified-memory totals, the hard budget + /// and its margins, how much is committed by models that are actually + /// serving, how much is still admittable, the persisted Scene state, and + /// the last transition or watchdog error. Read this before reasoning about + /// whether anything else will fit. + #[tool( + name = "governor_status", + annotations( + title = "Governor status", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn governor_status(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_status(&root)).await.map(Json) + } + + /// List every model in the node's registry with its declared worst-case + /// footprint, serving port, and whether it is currently serving. Footprints + /// are what admission control is decided against, not observed usage. + #[tool( + name = "model_list", + annotations( + title = "List registered models", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn model_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_models(&root)).await.map(Json) + } + + /// List the Scenes this node can activate, with each one's total footprint + /// and member models. A Scene is a named set of models brought up as a + /// single unit. + #[tool( + name = "scene_list", + annotations(title = "List scenes", read_only_hint = true, open_world_hint = false) + )] + async fn scene_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_scenes(&root)).await.map(Json) + } + + /// Show one Scene: its models, their footprints and live state, and the + /// Governor's admission verdict — whether the Scene fits its budget once + /// the safety margin is added, and by how much it fits or misses. + #[tool( + name = "scene_show", + annotations( + title = "Show a scene and its admission verdict", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn scene_show( + &self, + Parameters(params): Parameters, + ) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_scene(&root, ¶ms.name)) + .await + .map(Json) + } + + /// List the evaluation suites available on this node, with their versions + /// and case counts. Running a suite is deliberately not exposed: it drives + /// real load against a serving model for an unbounded time. + #[tool( + name = "eval_list", + annotations( + title = "List eval suites", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn eval_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_evals(&root)).await.map(Json) + } + + /// Plan — or, if the operator allowed it, perform — a Scene transition. + /// + /// With `dry_run` true (the default) this runs every check that guards a + /// real transition and returns the models it would stop and start, changing + /// nothing. With `dry_run` false it actually switches the node: models + /// outside the Scene are stopped, which will interrupt anything they serve, + /// and the call blocks until every Scene model reports healthy or the + /// transition rolls back. A real transition additionally requires the + /// server to have been started with `--allow-activate`; without it, + /// `dry_run: false` is refused. Ask a human before setting it. + #[tool( + name = "scene_activate", + annotations( + title = "Activate a scene (plans by default)", + read_only_hint = false, + destructive_hint = true, + idempotent_hint = false, + open_world_hint = false + ) + )] + async fn scene_activate( + &self, + Parameters(params): Parameters, + ) -> Result, ErrorData> { + if !params.dry_run && !self.allow_activate { + return Err(ErrorData::invalid_params( + "this MCP server is running plan-only: a real Scene transition needs an \ + operator to restart it as `lumbridge-compute mcp --allow-activate`. \ + Re-run with dry_run: true to see the plan.", + None, + )); + } + let root = self.root.clone(); + offload(move || activate_scene(&root, ¶ms.name, params.dry_run)) + .await + .map(Json) + } +} + +#[tool_handler(router = self.tool_router)] +impl ServerHandler for ComputeMcp { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info(Implementation::new( + "lumbridge-compute", + env!("CARGO_PKG_VERSION"), + )) + .with_instructions( + "Lumbridge Compute runs many AI models on one unified-memory box. There is a \ + single memory pool shared by CPU and GPU, so over-committing does not fail \ + gracefully — the machine thrashes and wedges before the OOM killer acts. The \ + Governor prevents that by admitting a model only if its declared footprint \ + plus a safety margin still fits a hard budget.\n\n\ + Start from `governor_status` for what is committed and what is admittable, \ + and `scene_show` for whether a named Scene would be admitted. Everything is \ + read-only except `scene_activate`, which plans by default. Activating a Scene \ + stops every registered model outside it, so treat it as a production change \ + and get a human's agreement before asking for a non-dry run.", + ) + } +} + +// ---- tool parameters ------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +struct SceneNameParams { + /// Scene name, as reported by `scene_list`. + name: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct SceneActivateParams { + /// Scene name, as reported by `scene_list`. + name: String, + /// Validate and report the plan without changing anything. Defaults to + /// true; set it to false only with a human's explicit agreement. + #[serde(default = "yes")] + dry_run: bool, +} + +fn yes() -> bool { + true +} + +// ---- tool results ---------------------------------------------------------- +// +// Lists are wrapped in objects rather than returned bare: MCP structured +// content must be a JSON object, and a named field leaves room to report +// alongside the list later without breaking a client's schema. + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct StatusReport { + memory_total_gb: f64, + memory_available_gb: f64, + budget_gb: f64, + safety_margin_gb: f64, + watchdog_floor_gb: f64, + /// Sum of the footprints of every registered model currently serving. + committed_gb: f64, + /// What a new model could still claim without breaching the safety margin. + headroom_gb: f64, + /// Scene whose exact model health was last verified. + active_scene: Option, + /// Scene the node should return to after a restart. + desired_scene: Option, + /// Previous proven Scene, used if the desired one cannot resume. + last_known_good_scene: Option, + last_error: Option, + running: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct RunningModel { + id: String, + name: String, + footprint_gb: f64, + port: Option, + /// True when Compute owns this process identity and may signal it. False + /// means the model is serving but was started outside Compute, so a Scene + /// transition will refuse to touch it until it is adopted. + compute_owned: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct ModelList { + models: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct ModelEntry { + id: String, + name: String, + /// Declared worst-case unified memory once serving, not observed usage. + footprint_gb: f64, + port: Option, + running: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct SceneList { + scenes: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct SceneEntry { + name: String, + version: u32, + description: String, + /// Total footprint of the Scene's models that exist in the registry. + footprint_gb: f64, + models: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct SceneDetail { + name: String, + version: u32, + description: String, + budget_gb: f64, + models: Vec, + /// Model ids the Scene references that the registry does not define. + missing_models: Vec, + footprint_gb: f64, + /// Footprint plus the Governor's safety margin — the figure compared + /// against the budget. + required_gb: f64, + /// The Governor's verdict: would this Scene be admitted? + admits: bool, + verdict: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct SceneModel { + id: String, + /// Absent when the id is missing from the registry. + name: Option, + footprint_gb: Option, + running: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct EvalList { + suites: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct EvalSuite { + name: String, + version: u32, + cases: usize, + description: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct ActivationReport { + scene: String, + budget_gb: f64, + /// Serving models outside the Scene, which the transition stops first. + stop: Vec, + /// Scene models not yet serving, in the order they would be admitted. + start: Vec, + /// False when this was a plan and the node was left untouched. + applied: bool, +} + +// ---- collectors ------------------------------------------------------------ +// +// These compose the same primitives the CLI prints from — `governor`, `mem`, +// `config`, `proc::State` — rather than parsing the CLI's output. The numbers +// an agent sees are therefore the numbers admission control uses, by +// construction. + +pub(crate) fn collect_status(root: &Path) -> Result { + let registry = Registry::load(root)?; + let memory = mem::read()?; + let committed = governor::committed_gb(®istry); + let managed = State::load_checked(root)?; + let budget = governor::DEFAULT_BUDGET_GB; + + let running = governor::running_ids(®istry) + .into_iter() + .map(|id| { + let model = ®istry.models[&id]; + RunningModel { + name: model.name.clone(), + footprint_gb: model.footprint_gb, + port: model.serve.port, + compute_owned: managed + .procs + .get(&id) + .is_some_and(|process| process.owned_alive()), + id, + } + }) + .collect(); + + Ok(StatusReport { + memory_total_gb: memory.total_gb, + memory_available_gb: memory.available_gb, + budget_gb: budget, + safety_margin_gb: governor::SAFETY_MARGIN_GB, + watchdog_floor_gb: governor::WATCHDOG_FLOOR_GB, + // `+ 0.0` normalises the negative zero an empty sum can produce, which serialises as + // "-0.0" and reads as a bug to anyone consuming this JSON. + committed_gb: committed + 0.0, + // The BINDING constraint, not the declared one. This used to be + // `budget - committed - margin`, which ignores how much memory the box actually has — + // so on a 62 GB machine with a 100 GB budget it reported 92 GB of headroom that the + // next admission would refuse. cmd_status was fixed when admission started consulting + // MemAvailable; this path was missed, which meant every agent driving the node over + // MCP got the optimistic number. + headroom_gb: governor::headroom_gb(committed, budget, Some(memory.available_gb)).max(0.0), + active_scene: managed.active_scene, + desired_scene: managed.desired_scene, + last_known_good_scene: managed.last_known_good_scene, + last_error: managed.last_error, + running, + }) +} + +pub(crate) fn collect_models(root: &Path) -> Result { + let registry = Registry::load(root)?; + Ok(ModelList { + models: registry + .models + .iter() + .map(|(id, model)| ModelEntry { + id: id.clone(), + name: model.name.clone(), + footprint_gb: model.footprint_gb, + port: model.serve.port, + running: governor::is_running(model), + }) + .collect(), + }) +} + +pub(crate) fn collect_scenes(root: &Path) -> Result { + let registry = Registry::load(root)?; + Ok(SceneList { + scenes: load_scenes(root)? + .into_iter() + .map(|scene| SceneEntry { + name: scene.metadata.name, + version: scene.metadata.version, + description: scene.metadata.description, + footprint_gb: scene + .models + .iter() + .filter_map(|id| registry.models.get(id)) + .map(|model| model.footprint_gb) + .sum(), + models: scene.models, + }) + .collect(), + }) +} + +pub(crate) fn collect_scene(root: &Path, name: &str) -> Result { + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + + let mut footprint = 0.0; + let mut missing = Vec::new(); + let mut models = Vec::new(); + for id in &scene.models { + match registry.models.get(id) { + Some(model) => { + footprint += model.footprint_gb; + models.push(SceneModel { + id: id.clone(), + name: Some(model.name.clone()), + footprint_gb: Some(model.footprint_gb), + running: governor::is_running(model), + }); + } + None => { + missing.push(id.clone()); + models.push(SceneModel { + id: id.clone(), + name: None, + footprint_gb: None, + running: false, + }); + } + } + } + + // Same three-way verdict the CLI prints: a missing id is fatal regardless + // of arithmetic, because the Scene cannot be resolved at all. + let required = footprint + governor::SAFETY_MARGIN_GB; + let (admits, verdict) = if !missing.is_empty() { + ( + false, + format!("{} model(s) missing from the registry", missing.len()), + ) + } else if required <= budget { + ( + true, + format!("fits with {:.1} GB to spare", budget - required), + ) + } else { + ( + false, + format!("exceeds the budget by {:.1} GB", required - budget), + ) + }; + + Ok(SceneDetail { + name: scene.metadata.name, + version: scene.metadata.version, + description: scene.metadata.description, + budget_gb: budget, + models, + missing_models: missing, + footprint_gb: footprint, + required_gb: required, + admits, + verdict, + }) +} + +pub(crate) fn collect_evals(root: &Path) -> Result { + Ok(EvalList { + suites: eval::catalogue(root)? + .into_iter() + .map(|suite| EvalSuite { + name: suite.name, + version: suite.version, + cases: suite.cases, + description: suite.description, + }) + .collect(), + }) +} + +fn activate_scene(root: &Path, name: &str, dry_run: bool) -> Result { + // Plan first either way. On a real transition this is the plan we report; + // `activate` re-derives and re-validates its own under the transition lock, + // so the report describes intent and the lock still owns the truth. + let plan = lifecycle::plan(root, name)?; + if !dry_run { + lifecycle::activate(root, name, false)?; + } + Ok(ActivationReport { + scene: plan.scene, + budget_gb: plan.budget_gb, + stop: plan.stop, + start: plan.start, + applied: !dry_run, + }) +} + +/// Run a synchronous Governor call off the transport's thread. +/// +/// Every read here touches `/proc` and probes serving ports with blocking +/// socket timeouts, and a real transition can sit for minutes waiting on model +/// health. Doing that inline would stall the JSON-RPC reader for the duration, +/// so the synchronous core stays on the blocking pool where it belongs. +async fn offload(work: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(work).await { + Ok(Ok(value)) => Ok(value), + // `{:#}` keeps anyhow's context chain, which is where the actionable + // half of a Compute error lives ("adopt the active Scene first"). + Ok(Err(error)) => Err(ErrorData::internal_error(format!("{error:#}"), None)), + Err(join) => Err(ErrorData::internal_error( + format!("Compute worker task failed: {join}"), + None, + )), + } +} diff --git a/src/mem.rs b/src/mem.rs new file mode 100644 index 0000000..840c646 --- /dev/null +++ b/src/mem.rs @@ -0,0 +1,89 @@ +//! Unified-memory sensing. On a shared-memory box there's one pool, so `MemAvailable` +//! from /proc/meminfo is the single source of truth the Governor guards. + +use anyhow::{Context, Result}; +use std::fs; + +pub struct MemInfo { + pub total_gb: f64, + pub available_gb: f64, +} + +pub fn read() -> Result { + let s = fs::read_to_string("/proc/meminfo").context("reading /proc/meminfo")?; + Ok(parse_meminfo(&s)) +} + +/// Split out from `read` so the parser is testable without a real /proc. +pub fn parse_meminfo(s: &str) -> MemInfo { + let mut total = 0.0; + let mut available = 0.0; + for line in s.lines() { + if let Some(v) = line.strip_prefix("MemTotal:") { + total = kb_to_gb(v); + } else if let Some(v) = line.strip_prefix("MemAvailable:") { + available = kb_to_gb(v); + } + } + MemInfo { + total_gb: total, + available_gb: available, + } +} + +/// Parse a " 123456 kB" meminfo value into GB. +fn kb_to_gb(v: &str) -> f64 { + v.split_whitespace() + .next() + .and_then(|n| n.parse::().ok()) + .unwrap_or(0.0) + / 1024.0 + / 1024.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPARK: &str = "\ +MemTotal: 127512345 kB +MemFree: 2048000 kB +MemAvailable: 104857600 kB +Buffers: 123456 kB +"; + + #[test] + fn reads_total_and_available_and_ignores_other_fields() { + let m = parse_meminfo(SPARK); + assert!((m.total_gb - 121.6).abs() < 0.1, "total was {}", m.total_gb); + assert!( + (m.available_gb - 100.0).abs() < 0.1, + "available was {}", + m.available_gb + ); + } + + #[test] + fn missing_fields_read_as_zero_rather_than_panicking() { + // A zero here is what makes admission refuse, so an unparseable /proc must not + // look like an empty box with room to spare. + let m = parse_meminfo("SomethingElse: 1 kB\n"); + assert_eq!(m.total_gb, 0.0); + assert_eq!(m.available_gb, 0.0); + } + + #[test] + fn malformed_values_do_not_panic() { + let m = parse_meminfo("MemTotal: not-a-number kB\nMemAvailable:\n"); + assert_eq!(m.total_gb, 0.0); + assert_eq!(m.available_gb, 0.0); + } + + #[test] + fn memavailable_is_not_confused_with_memfree() { + // MemFree is much smaller than MemAvailable on a box with page cache; picking the + // wrong one would make the watchdog fire constantly. + let m = parse_meminfo(SPARK); + assert!(m.available_gb > 50.0); + } +} diff --git a/src/proc.rs b/src/proc.rs new file mode 100644 index 0000000..de9a24f --- /dev/null +++ b/src/proc.rs @@ -0,0 +1,470 @@ +//! Process lifecycle: spawn a model in its own process group, track it in a state +//! file, and stop it (SIGTERM → SIGKILL to the whole group). The Governor decides +//! *whether* to start; this module *how*. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{self, Command, Stdio}; +use std::time::Duration; + +use crate::config::Model; + +/// One Lumbridge Compute-managed process. `seq` is a monotonic launch counter so the watchdog +/// can always find the *newest* model to sacrifice first. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Proc { + pub pid: i32, + pub seq: u64, + pub port: Option, + /// Linux boot id plus process start ticks bind ownership to one process + /// incarnation, preventing a reused PID from ever being signalled. + #[serde(default)] + pub boot_id: Option, + #[serde(default)] + pub start_time_ticks: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct State { + #[serde(default)] + pub seq: u64, + #[serde(default)] + pub procs: BTreeMap, + #[serde(default)] + pub desired_scene: Option, + #[serde(default)] + pub active_scene: Option, + /// Previous proven scene used if the desired scene cannot resume. + #[serde(default)] + pub last_known_good_scene: Option, + #[serde(default)] + pub transition_scene: Option, + #[serde(default)] + pub last_error: Option, +} + +fn state_dir(root: &Path) -> PathBuf { + root.join(".compute") +} + +impl State { + pub fn load_checked(root: &Path) -> Result { + let p = state_dir(root).join("state.yaml"); + match fs::read_to_string(&p) { + Ok(s) => serde_yaml::from_str(&s) + .with_context(|| format!("parsing persisted state {}", p.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(State::default()), + Err(error) => Err(error).with_context(|| format!("reading {}", p.display())), + } + } + + pub fn save(&self, root: &Path) -> Result<()> { + let dir = state_dir(root); + fs::create_dir_all(&dir)?; + let p = dir.join("state.yaml"); + let tmp = dir.join(format!(".state.yaml.{}.tmp", process::id())); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp) + .with_context(|| format!("creating {}", tmp.display()))?; + file.write_all(serde_yaml::to_string(self)?.as_bytes())?; + file.sync_all()?; + fs::rename(&tmp, &p).with_context(|| format!("atomically replacing {}", p.display()))?; + File::open(&dir) + .with_context(|| format!("opening state directory {}", dir.display()))? + .sync_all() + .with_context(|| format!("syncing state directory {}", dir.display()))?; + Ok(()) + } + + /// Newest still-alive managed model id, if any. + pub fn newest_alive(&self) -> Option<(String, Proc)> { + self.procs + .iter() + .filter(|(_, p)| p.owned_alive()) + .max_by_key(|(_, p)| p.seq) + .map(|(id, p)| (id.clone(), p.clone())) + } +} + +#[derive(Debug)] +struct ProcessIdentity { + boot_id: String, + start_time_ticks: u64, + process_group_id: i32, + state: char, +} + +impl Proc { + /// Capture the identity of a process that Lumbridge Compute just spawned or + /// that an operator explicitly adopted from legacy state. + pub fn capture(pid: i32, seq: u64, port: Option) -> Result { + let identity = process_identity(pid)?; + if identity.process_group_id != pid { + bail!( + "refusing process {pid}: process group {} does not equal pid", + identity.process_group_id + ); + } + Ok(Proc { + pid, + seq, + port, + boot_id: Some(identity.boot_id), + start_time_ticks: Some(identity.start_time_ticks), + }) + } + + pub fn owned_alive(&self) -> bool { + self.validate_owned().is_ok() + } + + fn validate_owned(&self) -> Result<()> { + let expected_boot = self + .boot_id + .as_deref() + .context("legacy process record has no boot identity; adopt the active scene first")?; + let expected_start = self + .start_time_ticks + .context("legacy process record has no start identity; adopt the active scene first")?; + let current = process_identity(self.pid)?; + if current.boot_id != expected_boot || current.start_time_ticks != expected_start { + bail!( + "pid {} no longer identifies the process Compute started", + self.pid + ); + } + if current.state == 'Z' { + bail!("pid {} is a zombie awaiting reap", self.pid); + } + if current.process_group_id != self.pid { + bail!( + "pid {} now belongs to process group {}; refusing group signal", + self.pid, + current.process_group_id + ); + } + Ok(()) + } +} + +fn process_identity(pid: i32) -> Result { + let boot_id = fs::read_to_string("/proc/sys/kernel/random/boot_id") + .context("reading Linux boot id")? + .trim() + .to_string(); + let stat_path = format!("/proc/{pid}/stat"); + let stat = fs::read_to_string(&stat_path) + .with_context(|| format!("reading process identity {stat_path}"))?; + let (state, process_group_id, start_time_ticks) = parse_stat_identity(&stat)?; + Ok(ProcessIdentity { + boot_id, + start_time_ticks, + process_group_id, + state, + }) +} + +fn parse_stat_identity(stat: &str) -> Result<(char, i32, u64)> { + // `comm` is parenthesized and may contain spaces, so locate its final `)` + // before indexing the fields that follow it. The remainder starts at field 3. + let close = stat + .rfind(')') + .context("malformed /proc stat: missing process name terminator")?; + let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect(); + let state = fields + .first() + .and_then(|value| value.chars().next()) + .context("malformed /proc stat: missing process state")?; + let process_group_id = fields + .get(2) + .context("malformed /proc stat: missing process group")? + .parse::() + .context("parsing process group id")?; + let start_time_ticks = fields + .get(19) + .context("malformed /proc stat: missing start time")? + .parse::() + .context("parsing process start time")?; + Ok((state, process_group_id, start_time_ticks)) +} + +/// Expand a leading `~/` to `$HOME`. +fn expand(s: &str) -> String { + if let Some(rest) = s.strip_prefix("~/") { + if let Ok(home) = std::env::var("HOME") { + return format!("{home}/{rest}"); + } + } + s.to_string() +} + +/// The fraction of the pool a vLLM model may reserve, derived from its declared +/// footprint rather than typed separately. +/// +/// `footprint_gb` is what the Governor budgets against; `--gpu-memory-utilization` is +/// what actually reserves the memory. Keeping them as two hand-copied numbers means +/// admission control can be arithmetically correct and still wrong about the machine — +/// the registry says a model takes 66 GB while the flag lets it reserve 0.55 of a +/// 121.6 GB pool, which is 67. Deriving one from the other makes the declared footprint +/// the single source of truth. +/// +/// An explicit `gpu-memory-utilization` in `serve.args` always wins; this only fills in +/// the gap. Returns `None` when the pool size is unknown, in which case vLLM's own +/// default applies exactly as before. +fn derived_gpu_memory_utilization(m: &Model, mem_total_gb: Option) -> Option { + if m.serve.args.contains_key("gpu-memory-utilization") { + return None; + } + let total = mem_total_gb?; + if total <= 0.0 || m.footprint_gb <= 0.0 { + return None; + } + let fraction = m.footprint_gb / total; + // Never hand vLLM a fraction that would reserve the whole box. + (fraction > 0.0 && fraction < 0.95).then_some((fraction * 1000.0).round() / 1000.0) +} + +/// `kind`-based builder. argv[0] is the program. +pub fn build_argv(m: &Model, mem_total_gb: Option) -> Result> { + if !m.serve.command.is_empty() { + return Ok(m.serve.command.iter().map(|s| expand(s)).collect()); + } + match m.serve.kind.as_str() { + "vllm" => { + let executable = m + .serve + .executable + .as_deref() + .map(expand) + .unwrap_or_else(|| "vllm".to_string()); + let mut a = vec![executable, "serve".to_string()]; + if let Some(w) = &m.serve.weights { + a.push(expand(w)); + } + if !m.serve.served_name.is_empty() { + a.push("--served-model-name".into()); + a.extend(m.serve.served_name.iter().cloned()); + } + a.push("--host".into()); + a.push("0.0.0.0".into()); + if let Some(p) = m.serve.port { + a.push("--port".into()); + a.push(p.to_string()); + } + if let Some(fraction) = derived_gpu_memory_utilization(m, mem_total_gb) { + a.push("--gpu-memory-utilization".into()); + a.push(format!("{fraction}")); + } + for (k, v) in &m.serve.args { + render_arg(&mut a, k, v); + } + Ok(a) + } + other => bail!( + "model '{}' has no explicit `command` and kind '{}' has no builder yet — \ + add a `command: [...]` to the registry entry", + m.name, + other + ), + } +} + +fn render_arg(out: &mut Vec, k: &str, v: &serde_yaml::Value) { + use serde_yaml::Value; + let flag = format!("--{k}"); + match v { + Value::Bool(true) => out.push(flag), + Value::Bool(false) => {} + Value::String(s) => { + out.push(flag); + out.push(s.clone()); + } + Value::Number(n) => { + out.push(flag); + out.push(n.to_string()); + } + _ => { + out.push(flag); + out.push( + serde_yaml::to_string(v) + .unwrap_or_default() + .trim() + .to_string(), + ); + } + } +} + +/// Spawn `id`'s model detached in its own process group, logs → `.compute/logs/.log`. +/// Returns the child pid (also its process-group id). +pub fn spawn(root: &Path, id: &str, m: &Model) -> Result { + let argv = build_argv(m, crate::mem::read().ok().map(|info| info.total_gb))?; + let logdir = state_dir(root).join("logs"); + fs::create_dir_all(&logdir)?; + let log = File::create(logdir.join(format!("{id}.log")))?; + let errlog = log.try_clone()?; + + let mut cmd = Command::new(&argv[0]); + cmd.args(&argv[1..]) + .stdin(Stdio::null()) + .stdout(Stdio::from(log)) + .stderr(Stdio::from(errlog)) + .process_group(0); // own group → clean group-kill, immune to CLI's signals + for (k, v) in &m.serve.env { + cmd.env(k, expand(v)); + } + let mut child = cmd + .spawn() + .with_context(|| format!("spawning '{id}': {}", argv.join(" ")))?; + let pid = child.id() as i32; + // A resident agent may outlive many model processes. Reap each child when + // it exits so failed runtimes cannot accumulate as zombies under the agent. + std::thread::spawn(move || { + let _ = child.wait(); + }); + Ok(pid) +} + +/// Stop only the exact process group captured in this ownership record. +pub fn stop_owned(proc: &Proc) -> Result<()> { + if !proc.owned_alive() { + return Ok(()); + } + proc.validate_owned()?; + let term = unsafe { libc::kill(-proc.pid, libc::SIGTERM) }; + if term != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("sending SIGTERM to owned process group {}", proc.pid)); + } + for _ in 0..60 { + if !proc.owned_alive() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + proc.validate_owned()?; + let kill = unsafe { libc::kill(-proc.pid, libc::SIGKILL) }; + if kill != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("sending SIGKILL to owned process group {}", proc.pid)); + } + for _ in 0..20 { + if !proc.owned_alive() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + bail!("owned process group {} survived SIGKILL", proc.pid) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a Model the way the registry does — through serde — so these tests exercise + /// the real deserialization path rather than a hand-assembled struct. + fn vllm_model(footprint_gb: f64, extra_args: &str) -> Model { + let yaml = format!( + "name: brain\nfootprint_gb: {footprint_gb}\nserve:\n kind: vllm\n port: 8001\n args:\n max-model-len: 8192\n{extra_args}" + ); + serde_yaml::from_str(&yaml).expect("test model yaml") + } + + #[test] + fn gpu_memory_utilization_is_derived_from_the_declared_footprint() { + // 66 GB of a 121.6 GB pool is 0.543 — not the 0.55 that used to be typed in + // separately, which is the whole point: one number, not two that can drift. + let m = vllm_model(66.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), Some(0.543)); + let argv = build_argv(&m, Some(121.6)).unwrap(); + let i = argv + .iter() + .position(|a| a == "--gpu-memory-utilization") + .unwrap(); + assert_eq!(argv[i + 1], "0.543"); + } + + #[test] + fn an_explicit_flag_in_the_registry_always_wins() { + let m = vllm_model(66.0, " gpu-memory-utilization: 0.8\n"); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None); + // ...and it is rendered exactly once, from serve.args. + let argv = build_argv(&m, Some(121.6)).unwrap(); + assert_eq!( + argv.iter() + .filter(|a| *a == "--gpu-memory-utilization") + .count(), + 1 + ); + let i = argv + .iter() + .position(|a| a == "--gpu-memory-utilization") + .unwrap(); + assert_eq!(argv[i + 1], "0.8"); + } + + #[test] + fn unknown_pool_size_leaves_vllms_own_default_alone() { + let m = vllm_model(66.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, None), None); + assert!(!build_argv(&m, None) + .unwrap() + .iter() + .any(|a| a == "--gpu-memory-utilization")); + } + + #[test] + fn a_footprint_that_would_claim_the_whole_box_is_not_derived() { + // Better to let vLLM apply its own default than to hand it 0.99 and wedge the box. + let m = vllm_model(120.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None); + } + + #[test] + fn nonsense_inputs_do_not_produce_a_flag() { + assert_eq!( + derived_gpu_memory_utilization(&vllm_model(0.0, ""), Some(121.6)), + None + ); + assert_eq!( + derived_gpu_memory_utilization(&vllm_model(66.0, ""), Some(0.0)), + None + ); + } + + #[test] + fn an_explicit_command_bypasses_the_builder_entirely() { + let mut m = vllm_model(66.0, ""); + m.serve.command = vec!["python".into(), "-m".into(), "server".into()]; + assert_eq!(build_argv(&m, Some(121.6)).unwrap(), m.serve.command); + } + + #[test] + fn proc_stat_parser_handles_spaces_in_process_name() { + let stat = "123 (worker process) S 1 123 123 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 4567"; + let (state, pgrp, start) = parse_stat_identity(stat).unwrap(); + assert_eq!(state, 'S'); + assert_eq!(pgrp, 123); + assert_eq!(start, 4567); + } + + #[test] + fn legacy_records_are_never_treated_as_owned() { + let proc = Proc { + pid: std::process::id() as i32, + seq: 1, + port: None, + boot_id: None, + start_time_ticks: None, + }; + assert!(!proc.owned_alive()); + } +} diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 0000000..14afedb --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,1158 @@ +//! Privacy-safe, durable model-serving telemetry. +//! +//! The model runtime remains the authority for scheduling, token, latency, and +//! multimodal encoder metrics. Compute samples its Prometheus endpoint and stores +//! only numeric measurements plus bounded labels. Prompts, outputs, image data, +//! URLs, and arbitrary HTTP headers never enter this database. + +use anyhow::{bail, Context, Result}; +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::io::{Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::config::{find_scene, Registry}; +use crate::proc::State; + +const DB_RELATIVE_PATH: &str = ".compute/usage/telemetry.sqlite3"; +const MAX_METRICS_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +const COUNTER_SNAPSHOT_INTERVAL_MS: i64 = 60_000; +const ALLOWED_CUSTOM_LABELS: &[&str] = &["client", "agent", "workload"]; + +const COUNTERS: &[&str] = &[ + "sglang:prompt_tokens_total", + "sglang:generation_tokens_total", + "sglang:cached_tokens_total", + "sglang:num_requests_total", + "sglang:num_aborted_requests_total", + "sglang:encoder_requests_received_total", + "sglang:encoder_cache_total_tokens_total", + "sglang:encoder_cache_hit_tokens_total", + "sglang:encoder_mm_items_per_request_sum", + "sglang:encoder_mm_items_per_request_count", + "sglang:time_to_first_token_seconds_sum", + "sglang:time_to_first_token_seconds_count", + "sglang:e2e_request_latency_seconds_sum", + "sglang:e2e_request_latency_seconds_count", +]; + +#[derive(Debug, Clone, PartialEq)] +struct Metric { + name: String, + labels: BTreeMap, + value: f64, +} + +#[derive(Debug, Clone)] +pub struct Target { + pub model: String, + pub scene: String, + pub max_concurrency: u32, + pub config_fingerprint: String, +} + +#[derive(Debug, Serialize)] +pub struct CollectResult { + pub timestamp_ms: i64, + pub model: String, + pub scene: String, + pub max_concurrency: u32, + pub metrics_recorded: usize, + pub running: u32, + pub queued: u32, +} + +#[derive(Debug, Clone, Serialize, Default)] +pub struct UsageTotals { + pub requests: f64, + pub aborted_requests: f64, + pub prompt_tokens: f64, + pub generation_tokens: f64, + pub cached_tokens: f64, + pub vision_requests: f64, + pub vision_request_share: f64, + pub vision_request_source: String, + pub vision_prompt_tokens: f64, + pub vision_items: f64, + pub vision_encoder_tokens: f64, + pub vision_encoder_cached_tokens: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencyBucket { + pub concurrency: String, + pub samples: u64, + pub share: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencySummary { + pub samples: u64, + pub queued_samples: u64, + pub peak_running: u32, + pub peak_queued: u32, + /// A time-sampled distribution, not an admission-count distribution. + pub distribution: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AgentUsage { + pub client: String, + pub agent: String, + pub workload: String, + pub requests: f64, + pub prompt_tokens: f64, + pub generation_tokens: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct UsageSummary { + pub since_ms: i64, + pub until_ms: i64, + pub database: String, + pub totals: UsageTotals, + pub concurrency: ConcurrencySummary, + pub agents: Vec, +} + +pub struct Store { + path: PathBuf, + conn: Connection, +} + +impl Store { + pub fn open(root: &Path) -> Result { + let path = root.join(DB_RELATIVE_PATH); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("creating telemetry directory {}", parent.display()))?; + } + let conn = Connection::open(&path) + .with_context(|| format!("opening telemetry database {}", path.display()))?; + conn.busy_timeout(Duration::from_secs(5))?; + conn.execute_batch( + "PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS telemetry_schema ( + version INTEGER PRIMARY KEY, + applied_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS config_snapshots ( + fingerprint TEXT PRIMARY KEY, + first_seen_ms INTEGER NOT NULL, + model TEXT NOT NULL, + scene TEXT NOT NULL, + max_concurrency INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS engine_samples ( + timestamp_ms INTEGER NOT NULL, + model TEXT NOT NULL, + scene TEXT NOT NULL, + config_fingerprint TEXT NOT NULL, + max_concurrency INTEGER NOT NULL, + running REAL NOT NULL, + queued REAL NOT NULL, + generation_tps REAL, + cache_hit_rate REAL, + PRIMARY KEY (timestamp_ms, model) + ); + CREATE INDEX IF NOT EXISTS engine_samples_time + ON engine_samples(timestamp_ms); + CREATE TABLE IF NOT EXISTS counter_samples ( + timestamp_ms INTEGER NOT NULL, + model TEXT NOT NULL, + metric TEXT NOT NULL, + labels_json TEXT NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (timestamp_ms, model, metric, labels_json) + ); + CREATE INDEX IF NOT EXISTS counter_samples_metric_time + ON counter_samples(metric, timestamp_ms); + INSERT OR IGNORE INTO telemetry_schema(version, applied_at_ms) + VALUES (1, CAST(strftime('%s','now') AS INTEGER) * 1000);", + )?; + Ok(Self { path, conn }) + } + + fn record(&mut self, target: &Target, metrics: &[Metric]) -> Result { + self.record_with_counter_interval(target, metrics, COUNTER_SNAPSHOT_INTERVAL_MS) + } + + fn record_with_counter_interval( + &mut self, + target: &Target, + metrics: &[Metric], + counter_interval_ms: i64, + ) -> Result { + let timestamp_ms = unix_time_ms()?; + let running = default_gauge(metrics, "sglang:num_running_reqs").unwrap_or(0.0); + let queued = default_gauge(metrics, "sglang:num_queue_reqs").unwrap_or(0.0); + let generation_tps = default_gauge(metrics, "sglang:gen_throughput"); + let cache_hit_rate = default_gauge(metrics, "sglang:cache_hit_rate"); + + let last_counter_ms: Option = self.conn.query_row( + "SELECT MAX(timestamp_ms) FROM counter_samples WHERE model = ?1", + params![target.model], + |row| row.get(0), + )?; + let counter_due = + last_counter_ms.is_none_or(|previous| timestamp_ms - previous >= counter_interval_ms); + + let transaction = self.conn.transaction()?; + transaction.execute( + "INSERT OR IGNORE INTO config_snapshots + (fingerprint, first_seen_ms, model, scene, max_concurrency) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + target.config_fingerprint, + timestamp_ms, + target.model, + target.scene, + target.max_concurrency + ], + )?; + transaction.execute( + "INSERT INTO engine_samples + (timestamp_ms, model, scene, config_fingerprint, max_concurrency, + running, queued, generation_tps, cache_hit_rate) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + timestamp_ms, + target.model, + target.scene, + target.config_fingerprint, + target.max_concurrency, + running, + queued, + generation_tps, + cache_hit_rate + ], + )?; + + let mut recorded = 0usize; + if counter_due { + for metric in metrics + .iter() + .filter(|metric| COUNTERS.contains(&metric.name.as_str())) + { + let labels = retained_labels(&metric.labels); + transaction.execute( + "INSERT INTO counter_samples + (timestamp_ms, model, metric, labels_json, value) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + timestamp_ms, + target.model, + metric.name, + serde_json::to_string(&labels)?, + metric.value + ], + )?; + recorded += 1; + } + } + transaction.commit()?; + + Ok(CollectResult { + timestamp_ms, + model: target.model.clone(), + scene: target.scene.clone(), + max_concurrency: target.max_concurrency, + metrics_recorded: recorded, + running: running.max(0.0).round() as u32, + queued: queued.max(0.0).round() as u32, + }) + } + + pub fn summary(&self, since_ms: i64, until_ms: i64) -> Result { + let image = |labels: &BTreeMap| { + labels.get("modality").is_some_and(|value| value == "image") + }; + let vision_workload = |labels: &BTreeMap| { + labels + .get("workload") + .is_some_and(|value| value == "vision") + }; + let has_encoder_request_metrics = + self.has_metric_samples("sglang:encoder_requests_received_total", since_ms, until_ms)?; + let vision_requests = if has_encoder_request_metrics { + self.metric_delta( + "sglang:encoder_requests_received_total", + since_ms, + until_ms, + image, + )? + } else { + self.metric_delta( + "sglang:num_requests_total", + since_ms, + until_ms, + vision_workload, + )? + }; + let mut totals = UsageTotals { + requests: self + .metric_delta("sglang:num_requests_total", since_ms, until_ms, |_| true)?, + aborted_requests: self.metric_delta( + "sglang:num_aborted_requests_total", + since_ms, + until_ms, + |_| true, + )?, + prompt_tokens: self.metric_delta( + "sglang:prompt_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + generation_tokens: self.metric_delta( + "sglang:generation_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + cached_tokens: self.metric_delta( + "sglang:cached_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + vision_requests, + vision_request_share: 0.0, + vision_request_source: if has_encoder_request_metrics { + "encoder_metrics".to_string() + } else { + "workload_label".to_string() + }, + vision_prompt_tokens: self.metric_delta( + "sglang:prompt_tokens_total", + since_ms, + until_ms, + vision_workload, + )?, + vision_items: self.metric_delta( + "sglang:encoder_mm_items_per_request_sum", + since_ms, + until_ms, + image, + )?, + vision_encoder_tokens: self.metric_delta( + "sglang:encoder_cache_total_tokens_total", + since_ms, + until_ms, + image, + )?, + vision_encoder_cached_tokens: self.metric_delta( + "sglang:encoder_cache_hit_tokens_total", + since_ms, + until_ms, + image, + )?, + }; + totals.vision_request_share = if totals.requests > 0.0 { + (totals.vision_requests / totals.requests).clamp(0.0, 1.0) + } else { + 0.0 + }; + + Ok(UsageSummary { + since_ms, + until_ms, + database: self.path.display().to_string(), + totals, + concurrency: self.concurrency(since_ms, until_ms)?, + agents: self.agent_usage(since_ms, until_ms)?, + }) + } + + fn metric_delta(&self, metric: &str, since_ms: i64, until_ms: i64, keep: F) -> Result + where + F: Fn(&BTreeMap) -> bool, + { + let total: f64 = self + .series_deltas(metric, since_ms, until_ms)? + .into_iter() + .filter(|(labels, _)| keep(labels)) + .map(|(_, delta)| delta) + .sum(); + Ok(if total == 0.0 { 0.0 } else { total }) + } + + fn has_metric_samples(&self, metric: &str, since_ms: i64, until_ms: i64) -> Result { + self.conn + .query_row( + "SELECT EXISTS( + SELECT 1 FROM counter_samples + WHERE metric = ?1 AND timestamp_ms >= ?2 AND timestamp_ms <= ?3 + )", + params![metric, since_ms, until_ms], + |row| row.get(0), + ) + .context("checking telemetry metric availability") + } + + fn series_deltas( + &self, + metric: &str, + since_ms: i64, + until_ms: i64, + ) -> Result, f64)>> { + let mut baseline_statement = self.conn.prepare( + "SELECT model, MIN(timestamp_ms) + FROM counter_samples + WHERE metric = ?1 AND timestamp_ms <= ?2 + GROUP BY model", + )?; + let baseline_rows = baseline_statement.query_map(params![metric, until_ms], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut model_baselines = HashMap::new(); + for row in baseline_rows { + let (model, timestamp_ms) = row?; + model_baselines.insert(model, timestamp_ms); + } + + let mut statement = self.conn.prepare( + "SELECT model, timestamp_ms, labels_json, value + FROM counter_samples + WHERE metric = ?1 AND timestamp_ms <= ?2 + ORDER BY model, labels_json, timestamp_ms", + )?; + let rows = statement.query_map(params![metric, until_ms], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, f64>(3)?, + )) + })?; + + let mut state: HashMap<(String, String), (Option, f64)> = HashMap::new(); + for row in rows { + let (model, timestamp_ms, labels_json, value) = row?; + let model_baseline = model_baselines.get(&model).copied().unwrap_or(timestamp_ms); + let entry = state.entry((model, labels_json)).or_insert((None, 0.0)); + if timestamp_ms >= since_ms { + if let Some(previous) = entry.0 { + entry.1 += if value >= previous { + value - previous + } else { + value + }; + } else if timestamp_ms > model_baseline { + // Prometheus creates a new labeled series on first use. The + // model-wide baseline predates it, so its initial value is + // usage in this window rather than unknown prehistory. + entry.1 += value; + } + } + entry.0 = Some(value); + } + + state + .into_iter() + .map(|((_model, labels), (_previous, delta))| { + let labels = serde_json::from_str(&labels) + .context("parsing labels stored in telemetry database")?; + Ok((labels, delta)) + }) + .collect() + } + + fn concurrency(&self, since_ms: i64, until_ms: i64) -> Result { + let mut statement = self.conn.prepare( + "SELECT running, queued FROM engine_samples + WHERE timestamp_ms >= ?1 AND timestamp_ms <= ?2 + ORDER BY timestamp_ms", + )?; + let rows = statement.query_map(params![since_ms, until_ms], |row| { + Ok((row.get::<_, f64>(0)?, row.get::<_, f64>(1)?)) + })?; + let mut counts: BTreeMap = BTreeMap::new(); + let mut samples = 0u64; + let mut queued_samples = 0u64; + let mut peak_running = 0u32; + let mut peak_queued = 0u32; + for row in rows { + let (running, queued) = row?; + let running = running.max(0.0).round() as u32; + let queued = queued.max(0.0).round() as u32; + let bucket = if running >= 4 { + "C4+".to_string() + } else { + format!("C{running}") + }; + *counts.entry(bucket).or_default() += 1; + samples += 1; + queued_samples += u64::from(queued > 0); + peak_running = peak_running.max(running); + peak_queued = peak_queued.max(queued); + } + let distribution = counts + .into_iter() + .map(|(concurrency, count)| ConcurrencyBucket { + concurrency, + samples: count, + share: if samples == 0 { + 0.0 + } else { + count as f64 / samples as f64 + }, + }) + .collect(); + Ok(ConcurrencySummary { + samples, + queued_samples, + peak_running, + peak_queued, + distribution, + }) + } + + fn agent_usage(&self, since_ms: i64, until_ms: i64) -> Result> { + #[derive(Default)] + struct Row { + requests: f64, + prompt_tokens: f64, + generation_tokens: f64, + } + let mut grouped: BTreeMap<(String, String, String), Row> = BTreeMap::new(); + for (metric, field) in [ + ("sglang:num_requests_total", 0), + ("sglang:prompt_tokens_total", 1), + ("sglang:generation_tokens_total", 2), + ] { + for (labels, delta) in self.series_deltas(metric, since_ms, until_ms)? { + let key = ( + label_or_unknown(&labels, "client"), + label_or_unknown(&labels, "agent"), + label_or_unknown(&labels, "workload"), + ); + let row = grouped.entry(key).or_default(); + match field { + 0 => row.requests += delta, + 1 => row.prompt_tokens += delta, + _ => row.generation_tokens += delta, + } + } + } + let mut rows: Vec<_> = grouped + .into_iter() + .filter(|(_, row)| { + row.requests != 0.0 || row.prompt_tokens != 0.0 || row.generation_tokens != 0.0 + }) + .map(|((client, agent, workload), row)| AgentUsage { + client, + agent, + workload, + requests: row.requests, + prompt_tokens: row.prompt_tokens, + generation_tokens: row.generation_tokens, + }) + .collect(); + rows.sort_by(|a, b| b.requests.total_cmp(&a.requests)); + Ok(rows) + } +} + +pub fn collect_once(root: &Path, metrics_url: &str, model_hint: &str) -> Result { + let target = resolve_target(root, metrics_url, model_hint)?; + let body = fetch_http(metrics_url)?; + let metrics = parse_prometheus(&body); + if !metrics + .iter() + .any(|metric| metric.name == "sglang:num_running_reqs") + { + bail!("{metrics_url} returned no SGLang scheduler metrics; start SGLang with --enable-metrics"); + } + Store::open(root)?.record(&target, &metrics) +} + +pub fn run_collector(root: PathBuf, metrics_url: String, model_hint: String, interval: Duration) { + let mut failures = 0u64; + loop { + match collect_once(&root, &metrics_url, &model_hint) { + Ok(result) => { + if failures > 0 { + eprintln!( + "telemetry: recovered after {failures} failed scrape(s); model={} scene={}", + result.model, result.scene + ); + } + failures = 0; + } + Err(error) => { + failures += 1; + if failures == 1 || failures.is_multiple_of(60) { + eprintln!("telemetry: scrape failed ({failures} consecutive): {error:#}"); + } + } + } + thread::sleep(interval.max(Duration::from_secs(1))); + } +} + +pub fn report(root: &Path, since: &str) -> Result { + let duration = parse_duration(since)?; + let until_ms = unix_time_ms()?; + let since_ms = until_ms.saturating_sub(duration.as_millis().min(i64::MAX as u128) as i64); + Store::open(root)?.summary(since_ms, until_ms) +} + +pub fn render_text(summary: &UsageSummary) -> String { + let totals = &summary.totals; + let mut lines = vec![ + "Lumbridge Compute · usage".to_string(), + format!(" requests {:>12.0}", totals.requests), + format!(" prompt tok {:>12.0}", totals.prompt_tokens), + format!(" output tok {:>12.0}", totals.generation_tokens), + format!(" cached tok {:>12.0}", totals.cached_tokens), + format!(" aborted {:>12.0}", totals.aborted_requests), + format!(" vision calls {:>12.0}", totals.vision_requests), + format!( + " vision share {:>11.1}%", + totals.vision_request_share * 100.0 + ), + format!(" vision tok {:>12.0}", totals.vision_prompt_tokens), + format!(" image items {:>12.0}", totals.vision_items), + String::new(), + " time-sampled concurrency:".to_string(), + ]; + if summary.concurrency.samples == 0 { + lines.push(" (no samples in this window)".to_string()); + } else { + for bucket in &summary.concurrency.distribution { + lines.push(format!( + " {:>3} {:>8} samples {:>6.1}%", + bucket.concurrency, + bucket.samples, + bucket.share * 100.0 + )); + } + lines.push(format!( + " peak running {} · peak queued {} · queue observed in {} sample(s)", + summary.concurrency.peak_running, + summary.concurrency.peak_queued, + summary.concurrency.queued_samples + )); + } + if !summary.agents.is_empty() { + lines.push(String::new()); + lines.push(" agents:".to_string()); + for row in &summary.agents { + lines.push(format!( + " {}/{}/{} {:.0} req · {:.0} in · {:.0} out tok", + row.client, + row.agent, + row.workload, + row.requests, + row.prompt_tokens, + row.generation_tokens + )); + } + } + lines.join("\n") +} + +fn resolve_target(root: &Path, metrics_url: &str, model_hint: &str) -> Result { + let registry = Registry::load(root)?; + let state = State::load_checked(root)?; + let scene_name = state + .active_scene + .or(state.desired_scene) + .unwrap_or_else(|| "unknown".to_string()); + let port = parse_http_url(metrics_url)?.1; + let model = if model_hint != "auto" { + model_hint.to_string() + } else if scene_name != "unknown" { + find_scene(root, &scene_name)? + .models + .into_iter() + .find(|id| registry.models.get(id).and_then(|m| m.serve.port) == Some(port)) + .with_context(|| { + format!("active Scene '{scene_name}' has no model on metrics port {port}") + })? + } else { + registry + .models + .iter() + .find(|(_, model)| model.serve.port == Some(port)) + .map(|(id, _)| id.clone()) + .with_context(|| format!("registry has no model on metrics port {port}"))? + }; + let model_config = registry + .models + .get(&model) + .with_context(|| format!("no registry model named '{model}'"))?; + let max_concurrency = configured_concurrency(model_config); + let fingerprint_input = format!( + "{model}\n{scene_name}\n{max_concurrency}\n{:?}\n{:?}", + model_config.serve.command, model_config.serve.args + ); + Ok(Target { + model, + scene: scene_name, + max_concurrency, + config_fingerprint: format!("fnv1a64:{:016x}", fnv1a64(fingerprint_input.as_bytes())), + }) +} + +fn configured_concurrency(model: &crate::config::Model) -> u32 { + for flag in ["--max-running-requests", "--max-num-seqs"] { + if let Some(index) = model.serve.command.iter().position(|arg| arg == flag) { + if let Some(value) = model + .serve + .command + .get(index + 1) + .and_then(|value| value.parse().ok()) + { + return value; + } + } + } + for key in ["max-running-requests", "max-num-seqs"] { + if let Some(value) = model + .serve + .args + .get(key) + .and_then(serde_yaml::Value::as_u64) + { + return value.min(u32::MAX as u64) as u32; + } + } + 0 +} + +fn retained_labels(labels: &BTreeMap) -> BTreeMap { + labels + .iter() + .filter(|(key, _)| { + matches!( + key.as_str(), + "model_name" | "stream" | "reason" | "modality" | "status" | "priority" + ) || ALLOWED_CUSTOM_LABELS.contains(&key.as_str()) + }) + .map(|(key, value)| { + let value = if ALLOWED_CUSTOM_LABELS.contains(&key.as_str()) { + sanitize_custom_label(value) + } else { + value.chars().take(128).collect() + }; + (key.clone(), value) + }) + .collect() +} + +fn sanitize_custom_label(value: &str) -> String { + if !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + value.to_string() + } else { + "invalid".to_string() + } +} + +fn label_or_unknown(labels: &BTreeMap, key: &str) -> String { + labels + .get(key) + // SGLang emits empty custom labels on unlabeled traffic. They are + // deliberately collapsed to the same bounded sentinel as malformed + // values at ingestion, and both render as unattributed usage. + .filter(|value| !value.is_empty() && value.as_str() != "invalid") + .cloned() + .unwrap_or_else(|| "unknown".to_string()) +} + +fn default_gauge(metrics: &[Metric], name: &str) -> Option { + metrics + .iter() + .filter(|metric| metric.name == name) + .find(|metric| metric.labels.get("priority").is_none_or(String::is_empty)) + .map(|metric| metric.value) +} + +fn parse_prometheus(text: &str) -> Vec { + text.lines().filter_map(parse_metric_line).collect() +} + +fn parse_metric_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let split = line.rfind(char::is_whitespace)?; + let head = line[..split].trim(); + let value = line[split..].trim().parse::().ok()?; + if !value.is_finite() { + return None; + } + let (name, labels) = if let Some(open) = head.find('{') { + if !head.ends_with('}') { + return None; + } + ( + &head[..open], + parse_labels(&head[open + 1..head.len() - 1])?, + ) + } else { + (head, BTreeMap::new()) + }; + Some(Metric { + name: name.to_string(), + labels, + value, + }) +} + +fn parse_labels(input: &str) -> Option> { + let bytes = input.as_bytes(); + let mut labels = BTreeMap::new(); + let mut index = 0usize; + while index < bytes.len() { + while index < bytes.len() && (bytes[index] == b',' || bytes[index].is_ascii_whitespace()) { + index += 1; + } + if index == bytes.len() { + break; + } + let key_start = index; + while index < bytes.len() && bytes[index] != b'=' { + index += 1; + } + if index == bytes.len() { + return None; + } + let key = input[key_start..index].trim().to_string(); + index += 1; + if bytes.get(index) != Some(&b'"') { + return None; + } + index += 1; + let mut value = String::new(); + while index < bytes.len() { + match bytes[index] { + b'"' => { + index += 1; + break; + } + b'\\' => { + index += 1; + let escaped = *bytes.get(index)?; + value.push(match escaped { + b'n' => '\n', + b'\\' => '\\', + b'"' => '"', + other => other as char, + }); + index += 1; + } + byte => { + value.push(byte as char); + index += 1; + } + } + } + labels.insert(key, value); + } + Some(labels) +} + +fn fetch_http(url: &str) -> Result { + let (host, port, path) = parse_http_url(url)?; + let address = (host.as_str(), port) + .to_socket_addrs()? + .next() + .with_context(|| format!("resolving {host}:{port}"))?; + let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(3)) + .with_context(|| format!("connecting to {url}"))?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(3)))?; + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: {host}:{port}\r\nAccept: text/plain\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n" + )?; + let mut response = Vec::new(); + stream + .take((MAX_METRICS_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut response)?; + if response.len() > MAX_METRICS_RESPONSE_BYTES { + bail!("metrics response exceeded {MAX_METRICS_RESPONSE_BYTES} bytes"); + } + let head_end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("metrics endpoint returned no HTTP header terminator")?; + let head = String::from_utf8_lossy(&response[..head_end]); + let status = head.lines().next().unwrap_or_default(); + if !status + .split_whitespace() + .nth(1) + .is_some_and(|code| code == "200") + { + bail!("metrics endpoint returned {status}"); + } + let body = &response[head_end + 4..]; + let body = if head + .lines() + .any(|line| line.eq_ignore_ascii_case("transfer-encoding: chunked")) + { + decode_chunked(body)? + } else { + body.to_vec() + }; + String::from_utf8(body).context("metrics response was not UTF-8") +} + +fn decode_chunked(mut body: &[u8]) -> Result> { + let mut decoded = Vec::new(); + loop { + let line_end = body + .windows(2) + .position(|window| window == b"\r\n") + .context("invalid chunked metrics response")?; + let size_text = std::str::from_utf8(&body[..line_end])? + .split(';') + .next() + .unwrap_or_default(); + let size = usize::from_str_radix(size_text.trim(), 16)?; + body = &body[line_end + 2..]; + if size == 0 { + break; + } + if body.len() < size + 2 { + bail!("truncated chunked metrics response"); + } + decoded.extend_from_slice(&body[..size]); + body = &body[size + 2..]; + } + Ok(decoded) +} + +fn parse_http_url(url: &str) -> Result<(String, u16, String)> { + let rest = url + .strip_prefix("http://") + .with_context(|| format!("telemetry metrics URL must use local http://, got '{url}'"))?; + let (authority, path) = rest.split_once('/').unwrap_or((rest, "")); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("metrics URL must include a port: '{url}'"))?; + let port = port.parse::().context("parsing metrics URL port")?; + if host.is_empty() { + bail!("metrics URL has an empty host"); + } + Ok((host.to_string(), port, format!("/{path}"))) +} + +fn parse_duration(value: &str) -> Result { + let split = value + .find(|ch: char| !ch.is_ascii_digit()) + .unwrap_or(value.len()); + let amount = value[..split] + .parse::() + .with_context(|| format!("invalid duration '{value}'"))?; + let unit = &value[split..]; + let seconds = match unit { + "s" => amount, + "m" => amount.saturating_mul(60), + "h" => amount.saturating_mul(60 * 60), + "d" => amount.saturating_mul(24 * 60 * 60), + "w" => amount.saturating_mul(7 * 24 * 60 * 60), + _ => bail!("duration must end in s, m, h, d, or w (for example 24h or 7d)"), + }; + Ok(Duration::from_secs(seconds)) +} + +fn unix_time_ms() -> Result { + let milliseconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock is before Unix epoch")? + .as_millis(); + Ok(milliseconds.min(i64::MAX as u128) as i64) +} + +fn fnv1a64(input: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in input { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_root() -> PathBuf { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("lumbridge-telemetry-{suffix}")); + fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn prometheus_parser_keeps_escaped_bounded_labels() { + let metrics = parse_prometheus( + "# HELP ignored\nsglang:num_requests_total{model_name=\"brain\",agent=\"prime\\\"one\",stream=\"true\"} 12\nsglang:num_running_reqs 3\n", + ); + assert_eq!(metrics.len(), 2); + assert_eq!(metrics[0].name, "sglang:num_requests_total"); + assert_eq!(metrics[0].labels["agent"], "prime\"one"); + assert_eq!(metrics[1].value, 3.0); + } + + #[test] + fn counter_deltas_survive_runtime_resets() { + let root = temp_root(); + let mut store = Store::open(&root).unwrap(); + let target = Target { + model: "brain".to_string(), + scene: "solo".to_string(), + max_concurrency: 4, + config_fingerprint: "test".to_string(), + }; + let labels = BTreeMap::from([ + ("agent".to_string(), "a".to_string()), + ("client".to_string(), "c".to_string()), + ]); + let mut record = |value| { + store + .record_with_counter_interval( + &target, + &[ + Metric { + name: "sglang:num_running_reqs".to_string(), + labels: BTreeMap::new(), + value: 1.0, + }, + Metric { + name: "sglang:num_requests_total".to_string(), + labels: labels.clone(), + value, + }, + ], + 0, + ) + .unwrap(); + thread::sleep(Duration::from_millis(2)); + }; + record(10.0); + let since = unix_time_ms().unwrap() - 1; + record(14.0); + record(2.0); // runtime restarted + let summary = store.summary(since, unix_time_ms().unwrap()).unwrap(); + assert_eq!(summary.totals.requests, 6.0); + assert_eq!(summary.agents[0].requests, 6.0); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn new_labeled_series_count_from_first_use_and_identify_vision() { + let root = temp_root(); + let mut store = Store::open(&root).unwrap(); + let target = Target { + model: "brain".to_string(), + scene: "solo".to_string(), + max_concurrency: 4, + config_fingerprint: "test".to_string(), + }; + let gauge = Metric { + name: "sglang:num_running_reqs".to_string(), + labels: BTreeMap::new(), + value: 0.0, + }; + let baseline_labels = BTreeMap::from([ + ("agent".to_string(), "".to_string()), + ("client".to_string(), "".to_string()), + ("workload".to_string(), "".to_string()), + ]); + store + .record_with_counter_interval( + &target, + &[ + gauge.clone(), + Metric { + name: "sglang:num_requests_total".to_string(), + labels: baseline_labels.clone(), + value: 10.0, + }, + Metric { + name: "sglang:prompt_tokens_total".to_string(), + labels: baseline_labels, + value: 100.0, + }, + ], + 0, + ) + .unwrap(); + thread::sleep(Duration::from_millis(2)); + let since = unix_time_ms().unwrap() - 1; + let vision_labels = BTreeMap::from([ + ("agent".to_string(), "eye-1".to_string()), + ("client".to_string(), "cloud-agent".to_string()), + ("workload".to_string(), "vision".to_string()), + ]); + store + .record_with_counter_interval( + &target, + &[ + gauge, + Metric { + name: "sglang:num_requests_total".to_string(), + labels: vision_labels.clone(), + value: 1.0, + }, + Metric { + name: "sglang:prompt_tokens_total".to_string(), + labels: vision_labels, + value: 334.0, + }, + ], + 0, + ) + .unwrap(); + + let summary = store.summary(since, unix_time_ms().unwrap()).unwrap(); + assert_eq!(summary.totals.requests, 1.0); + assert_eq!(summary.totals.vision_requests, 1.0); + assert_eq!(summary.totals.vision_prompt_tokens, 334.0); + assert_eq!(summary.totals.vision_request_source, "workload_label"); + assert_eq!(summary.agents[0].agent, "eye-1"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn duration_parser_is_explicit() { + assert_eq!(parse_duration("24h").unwrap(), Duration::from_secs(86_400)); + assert_eq!(parse_duration("7d").unwrap(), Duration::from_secs(604_800)); + assert!(parse_duration("24").is_err()); + } + + #[test] + fn caller_labels_are_bounded_and_cannot_carry_structured_data() { + assert_eq!(sanitize_custom_label(""), "invalid"); + assert_eq!(sanitize_custom_label("worker-1"), "worker-1"); + assert_eq!(sanitize_custom_label("person@example.com"), "invalid"); + assert_eq!(sanitize_custom_label(&"x".repeat(65)), "invalid"); + assert_eq!( + label_or_unknown( + &BTreeMap::from([("agent".to_string(), "invalid".to_string())]), + "agent" + ), + "unknown" + ); + } +} diff --git a/systemd/lumbridge-compute-agent-user.service b/systemd/lumbridge-compute-agent-user.service new file mode 100644 index 0000000..571630a --- /dev/null +++ b/systemd/lumbridge-compute-agent-user.service @@ -0,0 +1,19 @@ +[Unit] +Description=Lumbridge Compute resident Scene supervisor, gateway, and telemetry collector +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Environment=HOME=%h +Environment=LUMBRIDGE_COMPUTE_ROOT=%h/lumbridge-prod/compute +ExecStart=%h/lumbridge-prod/target/release/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 --metrics-url http://127.0.0.1:8001/metrics --metrics-model auto --telemetry-interval-sec 5 +Restart=always +RestartSec=2 +# Model process groups are lifecycle-owned by Compute and must survive an agent +# binary restart; the new agent revalidates boot id, PID, and process start time. +KillMode=process +TimeoutStopSec=15 + +[Install] +WantedBy=default.target diff --git a/systemd/lumbridge-compute-agent.service b/systemd/lumbridge-compute-agent.service new file mode 100644 index 0000000..9bb2843 --- /dev/null +++ b/systemd/lumbridge-compute-agent.service @@ -0,0 +1,20 @@ +[Unit] +Description=Lumbridge Compute resident Scene supervisor and model gateway +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=lumbridge +Environment=HOME=/var/lib/lumbridge +Environment=LUMBRIDGE_COMPUTE_ROOT=$LUMBRIDGE_COMPUTE_ROOT +ExecStart=/usr/local/bin/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 +Restart=always +RestartSec=2 +# Model process groups are lifecycle-owned by Compute and must survive an agent +# binary restart; the agent will revalidate their boot/start identities on resume. +KillMode=process +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/tools/backfill_voice.py b/tools/backfill_voice.py new file mode 100644 index 0000000..aab770f --- /dev/null +++ b/tools/backfill_voice.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""One-time backfill: seed voice.jsonl from WAV files that already existed in +~/chatterbox/generations before per-request logging was added to voice_api.py. +No chars/voice/variant metadata exists for these -- marked backfill:true. +Safe to re-run: skips gen_ids already present in voice.jsonl. +""" +import contextlib +import json +import os +import wave + +GEN_DIR = os.path.expanduser("~/chatterbox/generations") +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") +VOICE_LOG = os.path.join(USAGE_DIR, "voice.jsonl") + + +def already_logged(): + seen = set() + if os.path.exists(VOICE_LOG): + with open(VOICE_LOG) as f: + for line in f: + try: + seen.add(json.loads(line)["gen_id"]) + except Exception: + continue + return seen + + +def main(): + os.makedirs(USAGE_DIR, exist_ok=True) + seen = already_logged() + added = 0 + skipped = 0 + errors = 0 + with open(VOICE_LOG, "a") as out: + for fname in os.listdir(GEN_DIR): + if not fname.endswith(".wav"): + continue + gen_id = fname[:-4] + if gen_id in seen: + skipped += 1 + continue + path = os.path.join(GEN_DIR, fname) + try: + with contextlib.closing(wave.open(path, "rb")) as wf: + frames = wf.getnframes() + rate = wf.getframerate() + duration = frames / float(rate) if rate else 0.0 + except Exception: + errors += 1 + continue + record = { + "ts": os.path.getmtime(path), + "gen_id": gen_id, + "duration_s": duration, + "sr": rate, + "chars": None, + "voice": None, + "variant_used": None, + "gen_ms": None, + "backfill": True, + } + out.write(json.dumps(record) + "\n") + added += 1 + print(f"added={added} skipped(already logged)={skipped} errors={errors}") + + +if __name__ == "__main__": + main() diff --git a/tools/scrape_vllm_metrics.py b/tools/scrape_vllm_metrics.py new file mode 100755 index 0000000..8c1347d --- /dev/null +++ b/tools/scrape_vllm_metrics.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Scrape vLLM's own Prometheus /metrics on each locally-served model and append +one JSON snapshot line per reachable model to vllm_snapshots.jsonl. + +vLLM counters (prompt_tokens_total, generation_tokens_total, request_success_total) +reset to zero on every process restart -- this is what turns them into a durable +history. A model that's down is skipped for this tick, not an error: run this +every few minutes from cron and it just accumulates whatever was actually up. +""" +import json +import os +import re +import time +import urllib.request + +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") +SNAPSHOT_FILE = os.path.join(USAGE_DIR, "vllm_snapshots.jsonl") + +# model_name -> port, per registry/models.yaml +MODELS = { + "brain": 8001, + "embed": 8012, + "ocr": 8013, +} + +# Prometheus exposition line: metric{labels} value +# vLLM metric names contain a colon (vllm:prompt_tokens_total), which \w does not match. +LINE_RE = re.compile(r'^([\w:]+)(\{[^}]*\})?\s+([0-9eE+\-.]+)\s*$') + +COUNTERS = ( + "vllm:prompt_tokens_total", + "vllm:generation_tokens_total", + "vllm:num_requests_running", +) +# request_success_total is split by finished_reason -- sum all reasons. +SUCCESS_METRIC = "vllm:request_success_total" + + +def _fetch(port): + url = f"http://127.0.0.1:{port}/metrics" + with urllib.request.urlopen(url, timeout=3) as resp: + return resp.read().decode() + + +def _parse(text): + values = {name: 0.0 for name in COUNTERS} + success_total = 0.0 + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + m = LINE_RE.match(line) + if not m: + continue + name, value = m.group(1), m.group(3) + try: + value = float(value) + except ValueError: + continue + if name in values: + values[name] = value + elif name == SUCCESS_METRIC: + success_total += value + values[SUCCESS_METRIC] = success_total + return values + + +def main(): + os.makedirs(USAGE_DIR, exist_ok=True) + ts = time.time() + lines = [] + for model, port in MODELS.items(): + try: + text = _fetch(port) + except Exception: + continue # model is down -- skip silently, not an error + parsed = _parse(text) + record = { + "ts": ts, + "model": model, + "prompt_tokens_total": parsed["vllm:prompt_tokens_total"], + "generation_tokens_total": parsed["vllm:generation_tokens_total"], + "request_success_total": parsed[SUCCESS_METRIC], + "num_requests_running": parsed["vllm:num_requests_running"], + } + lines.append(json.dumps(record)) + if lines: + with open(SNAPSHOT_FILE, "a") as f: + for line in lines: + f.write(line + "\n") + print(f"scraped {len(lines)}/{len(MODELS)} models reachable") + + +if __name__ == "__main__": + main() diff --git a/tools/usage_report.py b/tools/usage_report.py new file mode 100755 index 0000000..f3ff942 --- /dev/null +++ b/tools/usage_report.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""the reference node usage report: rolls up the JSONL usage logs (voice/stt/image/music +generations) and the scraped vLLM token-counter snapshots into real numbers. + +Usage: + usage_report.py # human-readable report, last 14 days + usage_report.py --days 7 # narrower daily window + usage_report.py --json # machine-readable, same data +""" +import argparse +import datetime +import json +import os + +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") + +EVENT_LOGS = { + "voice": ("voice.jsonl", "duration_s"), + "stt": ("stt.jsonl", "audio_duration_s"), + "image": ("image.jsonl", None), + "music": ("music.jsonl", "duration_s"), +} + +VLLM_MODELS = ("brain", "embed", "ocr") +VLLM_COUNTERS = ("prompt_tokens_total", "generation_tokens_total", "request_success_total") + + +def _read_jsonl(path): + rows = [] + if not os.path.exists(path): + return rows + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def _day(ts): + return datetime.datetime.fromtimestamp(ts).date().isoformat() + + +def summarize_events(name, filename, duration_field, days): + rows = _read_jsonl(os.path.join(USAGE_DIR, filename)) + cutoff = datetime.datetime.now().timestamp() - days * 86400 + by_day = {} + total_count = len(rows) + total_duration = 0.0 + for r in rows: + ts = r.get("ts") + if ts is None: + continue + dur = r.get(duration_field) or 0.0 if duration_field else 0.0 + total_duration += dur + day = _day(ts) + entry = by_day.setdefault(day, {"count": 0, "duration_s": 0.0}) + entry["count"] += 1 + entry["duration_s"] += dur + recent_days = {d: v for d, v in by_day.items() + if datetime.datetime.fromisoformat(d).timestamp() >= cutoff - 86400} + return { + "service": name, + "total_count": total_count, + "total_duration_s": round(total_duration, 1), + "total_duration_hours": round(total_duration / 3600, 2), + "has_duration": duration_field is not None, + "daily": dict(sorted(recent_days.items())), + } + + +def summarize_vllm(days): + rows = _read_jsonl(os.path.join(USAGE_DIR, "vllm_snapshots.jsonl")) + cutoff = datetime.datetime.now().timestamp() - days * 86400 + by_model = {m: [] for m in VLLM_MODELS} + for r in rows: + model = r.get("model") + if model in by_model: + by_model[model].append(r) + + result = {} + for model, snaps in by_model.items(): + snaps.sort(key=lambda r: r["ts"]) + current = snaps[-1] if snaps else None + tracked = {c: 0.0 for c in VLLM_COUNTERS} + prev = None + for s in snaps: + if s["ts"] < cutoff: + prev = s + continue + if prev is not None: + for c in VLLM_COUNTERS: + delta = s.get(c, 0) - prev.get(c, 0) + if delta > 0: + tracked[c] += delta + prev = s + result[model] = { + "reachable_now": current is not None and current["ts"] >= cutoff, + "snapshots_recorded": len(snaps), + "current": { + "prompt_tokens_total": current["prompt_tokens_total"], + "generation_tokens_total": current["generation_tokens_total"], + "request_success_total": current["request_success_total"], + } if current else None, + "tracked_since_monitoring_started": tracked, + } + return result + + +def render_text(events, vllm, days): + lines = [] + lines.append(f"=== the reference node usage report (last {days} days) ===\n") + + for e in events: + lines.append(f"-- {e['service']} --") + if e["has_duration"]: + lines.append(f" total: {e['total_count']} generations, " + f"{e['total_duration_hours']}h ({e['total_duration_s']}s)") + else: + lines.append(f" total: {e['total_count']} generations") + if not e["daily"]: + lines.append(" (no activity in this window)") + else: + for day, v in e["daily"].items(): + if e["has_duration"]: + lines.append(f" {day}: {v['count']:>5} gens, {round(v['duration_s']/60, 1):>7} min") + else: + lines.append(f" {day}: {v['count']:>5} gens") + lines.append("") + + lines.append("-- local LLM / vLLM token throughput --") + for model, v in vllm.items(): + if not v["current"]: + lines.append(f" {model}: not reachable (no snapshot ever recorded)") + continue + status = "up" if v["reachable_now"] else f"down (last seen in a prior snapshot)" + c = v["current"] + t = v["tracked_since_monitoring_started"] + lines.append(f" {model}: {status}") + lines.append(f" current counters (since last process restart): " + f"{int(c['prompt_tokens_total']):,} prompt tok, " + f"{int(c['generation_tokens_total']):,} gen tok, " + f"{int(c['request_success_total']):,} requests") + lines.append(f" tracked in window ({v['snapshots_recorded']} scrapes): " + f"{int(t['prompt_tokens_total']):,} prompt tok, " + f"{int(t['generation_tokens_total']):,} gen tok, " + f"{int(t['request_success_total']):,} requests") + lines.append("") + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--days", type=int, default=14) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + events = [summarize_events(name, filename, duration_field, args.days) + for name, (filename, duration_field) in EVENT_LOGS.items()] + vllm = summarize_vllm(args.days) + + if args.json: + print(json.dumps({"events": events, "vllm": vllm, "days": args.days}, indent=2)) + else: + print(render_text(events, vllm, args.days)) + + +if __name__ == "__main__": + main()