commit ef9ec1dcd8aa98741e548ee948ab4416157c9a56 Author: Karti Tripathi Date: Mon Aug 31 16:09:08 2026 -0700 Lumbridge Compute: telemetry and four-lane operations diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9ddf67e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: ci + +on: + push: + pull_request: + +jobs: + rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy --all-targets + - run: cargo test + - run: cargo build --release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f90b8f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +/target +.compute/ +eval-results/ +**/*.rs.bk +Cargo.lock.orig +*.log +.DS_Store + +# Hand-made snapshots taken before a registry or scene edit. Local safety +# copies, not history — git already keeps the history. +*.bak +*.bak-* +*.bak.* + +# Secrets. +.env +.env.* +*.pem +*.key +*_rsa +*_ed25519 +id_ed25519* +.ssh/ +*secret* +*credentials* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..500e572 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing to Lumbridge Compute + +Contributions are welcome. Changes land as small, reviewable commits. + +## Development + +```bash +cargo test +cargo fmt --check +cargo clippy --all-targets +cargo build --release +``` + +Changes to the `lumbridge/v1` scene or eval contracts require fixtures, backward- +compatibility notes, and documentation. Never add secrets, private model tokens, +personal voice samples, internal hostnames, or arbitrary commands to shareable +scene/eval manifests. + +Performance changes should include before/after JSON artifacts and the complete +hardware/model/runtime configuration needed to reproduce them. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..733ab32 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1012 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.4", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lumbridge-compute" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "libc", + "rmcp", + "rusqlite", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "tokio", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "rmcp" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a15bc53261a9dc37e105df006e4656c598379a8f9581f8950debb130f27a7cf" +dependencies = [ + "base64", + "chrono", + "futures", + "indexmap", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", +] + +[[package]] +name = "rmcp-macros" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a85d45508e9b4ba024fe996c2638799635d75b6dd0ba8f32ccf08f8026f0c780" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 3.0.4", +] + +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "chrono", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 3.0.4", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6e7aa33 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "lumbridge-compute" +description = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes." +readme = "README.md" +keywords = ["dgx-spark", "gb10", "llm", "orchestration", "unified-memory"] +categories = ["command-line-utilities"] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +authors = ["Karti Tripathi"] +repository = "https://git.karti.ai/lumbridge-public/compute" +rust-version = "1.88" + +[[bin]] +name = "lumbridge-compute" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +serde_json = "1" +anyhow = "1" +libc = "0.2" +rmcp = { version = "3", features = ["server", "transport-io", "macros"] } +schemars = "1" +tokio = { version = "1", features = ["rt", "io-std", "fs", "time"] } +rusqlite = { version = "0.37", features = ["bundled"] } + +# Lives at the workspace root upstream; Cargo ignores [profile] in members. +[profile.release] +strip = true +lto = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..39693df --- /dev/null +++ b/NOTICE @@ -0,0 +1,16 @@ +Lumbridge +Copyright 2026 Karti Tripathi + +This product includes software developed by Karti Tripathi. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7414a67 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +
+ +# Lumbridge Compute + +**The AI compute layer for Lumbridge clusters. Safe models, scenes, and evals on your own hardware.** + +*Run many models on one box — safely — on NVIDIA DGX Spark (GB10), DGX Station (GB300), and RTX.* + +
+ +--- + +Lumbridge Compute is the tool you install the minute you open your DGX Spark. Boxes like the +DGX Spark (GB10), DGX Station (GB300), and RTX workstations share **one memory pool** +between CPU and GPU with little swap. On that hardware, over-committing memory doesn't +fail gracefully — the whole machine thrashes and wedges (SSH and ping included) before +the OOM killer acts. Lumbridge Compute makes that impossible, and turns the box into something you +can load with different workloads on a schedule. + +> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.** +> They're orthogonal; you can run Lumbridge Compute on each node of an exo cluster. + +## Three layers + +| Layer | Analogous to | Job | +|---|---|---| +| **Governor** | kernel / memory cgroup + OOM policy | Nothing starts unless it fits a hard budget; a watchdog kills the newest model *before* the box wedges. | +| **Scenes** | systemd targets | Named, shareable, activatable bundles of models (`studio`, `darkroom`). | +| **lumbridge-compute** (the shell) | the CLI you install | Onboard a fresh box, manage models, activate/schedule scenes. | + +## The Governor (why Lumbridge Compute exists) + +Unified memory means no separate VRAM pool to bounce off. vLLM, diffusers, and friends +will happily reserve past 100% of the shared pool, and the box wedges. The Governor +prevents this with two mechanisms: + +1. **Admission control** — a model starts only if its declared footprint fits the budget + given what's already running, plus a safety margin. +2. **Watchdog** — a 1 Hz thread on `MemAvailable`; if it dips below a critical floor, it + kills the most-recently-started model before thrash. A backstop for a wrong estimate. + +## Scenes + +A **scene** is a named set of models brought up together — the unit you activate, publish, +and schedule. A single unified-memory box can't hold every model at once, so scenes let +you **time-multiplex** it: run a live voice assistant by day, then switch to an overnight +image farm at 3am. One box, the utilization of several. + +Scenes reference **model ids**, never weight paths — so requantizing or upgrading a model +never breaks a published scene. See [`docs/scene-spec.md`](docs/scene-spec.md), and +[`docs/positioning.md`](docs/positioning.md) for how Lumbridge Compute relates to Ollama / llama-swap / exo. + +## Evals + +Lumbridge Compute ships a native, model-server-agnostic evaluation runner. Versioned YAML suites +measure streamed TTFT, client-observed prefill throughput, decode throughput, and +deterministic capability assertions against any OpenAI-compatible endpoint. + +```bash +lumbridge-compute eval ls +lumbridge-compute eval run smoke +lumbridge-compute eval run performance --repeat 5 +lumbridge-compute eval run finance-core +lumbridge-compute usage summary --since 24h # calls, tokens, vision, C0-C4, queueing +lumbridge-compute usage agents --since 7d # bounded client/agent/workload labels +lumbridge-compute usage concurrency --since 30d +``` + +Every run writes a portable JSON artifact for regression tracking and future eval +registries. See [`docs/evals.md`](docs/evals.md). + +## Install + +```bash +# from source (single static binary, no runtime deps) +cargo install --path . +``` + +## Usage + +```bash +lumbridge-compute status # Governor: memory, budget, running set, headroom +lumbridge-compute model ls # registry: footprints + live state + which port +lumbridge-compute scene ls # scenes with total footprint +lumbridge-compute scene show darkroom # models + footprints + admission verdict +lumbridge-compute scene adopt voice-qwen # one-time identity capture for a legacy live node +lumbridge-compute scene resume # desired Scene, then last-known-good fallback +lumbridge-compute gateway # stable streaming endpoint -> local model server +lumbridge-compute agent # resume + gateway + memory + opt-in model supervision + +# planned: +lumbridge-compute model pull # footprint-aware; warns if no scene can hold it +lumbridge-compute scene schedule darkroom 03:00 04:00 # time-multiplex the box +``` + +By default Lumbridge Compute reads `registry/` and `scenes/` from the current directory (override with +`--root ` or `$LUMBRIDGE_COMPUTE_ROOT`). + +## Status + +The Governor, identity-bound process ownership, transactional Scene switching, +persistent desired state, last-known-good recovery, opt-in model supervision, +streaming gateway, eval runner, watchdog, and privacy-safe usage telemetry work +today. Next: the model artifact manager (`pull`), fleet API, and telemetry-driven +Jobs scheduler. See [usage telemetry](docs/telemetry.md), [node operations](docs/operations.md), +and the stable [scene spec](docs/scene-spec.md). + +Lumbridge Compute runs on NVIDIA hardware but is independent and is not affiliated with or endorsed by NVIDIA. + +## License + +Apache-2.0 — see [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..d62d4c0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,10 @@ +# Security policy + +Please report vulnerabilities privately to the maintainers before opening a +public issue. + +Lumbridge Compute treats model registries as trusted local configuration and scenes/eval suites +as potentially untrusted shared data. Shared manifests reference vetted ids and +must never execute embedded shell commands. Downloads must be checksum-verified +before promotion into the trusted registry. Secrets belong in environment or +OS-managed secret stores, never manifests, logs, or result artifacts. diff --git a/docs/evals.md b/docs/evals.md new file mode 100644 index 0000000..1c63f80 --- /dev/null +++ b/docs/evals.md @@ -0,0 +1,40 @@ +# Lumbridge Compute evaluations (`lumbridge/v1`) + +Lumbridge Compute evaluates the model configuration that is actually serving: weights, +quantization, context, runtime, parsers, and speculative decoder. A model name +without its serving configuration is not a reproducible benchmark target. + +```bash +lumbridge-compute eval ls +lumbridge-compute eval run smoke +lumbridge-compute eval run performance --model brain --repeat 5 +lumbridge-compute eval run finance-core --base-url http://your-node:8001/v1 +``` + +Suites live in `evals/*.eval.yaml`. Each case has a stable id, prompt, category, +generation limit, and deterministic assertions. Runs produce append-only JSON in +`eval-results/` with raw outputs and per-sample metrics. + +## Metrics + +- **TTFT**: wall time until the first streamed content or reasoning token. +- **Prefill tok/s (approximate)**: API-reported prompt tokens divided by TTFT. + This is client-observed and includes queueing/scheduling; server-native prefill + metrics should be added as a separate source rather than conflated with it. +- **Decode tok/s**: completion tokens divided by time after the first token. +- **Score**: share of samples satisfying every declared assertion. + +Performance runs should include warmups in automation and record hardware, Lumbridge Compute +scene, runtime version, model revision, and cold/warm cache state. The v1 artifact +is deliberately local and portable; a future registry can ingest the same JSON. + +## Boundary with Bench, Arena, and Forge + +Compute's native suites are post-activation health and performance checks for the exact +configuration serving on a node. They do not grow into another training harness. Bench owns +longitudinal model evidence, Arena owns task distributions and rewards on upstream Prime +Intellect Verifiers, and Forge records any Prime-RL handoff and returned training artifact. + +Compute resumes ownership only after an approved checkpoint has been returned and verified: +local transfer, footprint admission, registry promotion, scene scheduling, and serving-process +lifecycle. The training framework owns optimization and distributed execution. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..aa1da7e --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,170 @@ +# Lumbridge Compute node operations + +## Persistent Scene state + +Compute stores state at `/.compute/state.yaml`. State writes use +an atomic temporary-file replacement. The persisted lifecycle fields are: + +- `desired_scene`: the Scene the node should resume after an agent or node restart; +- `active_scene`: the Scene whose exact model health was last verified; +- `last_known_good_scene`: the previous proven Scene used for recovery; +- `transition_scene`: an in-progress transaction, cleared on success or failure; +- `last_error`: the most recent activation or watchdog failure. + +Process ownership is bound to `(boot_id, pid, process_start_ticks)`. Compute will +never signal a legacy PID-only record or a PID that Linux has reused. + +## One-time adoption on an existing node + +Before the first transactional switch, bind the already-running exact Scene to +process identities. Adoption neither starts nor stops a model. + +```bash +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene adopt voice-qwen +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT status +lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene activate voice-qwen --dry-run +``` + +Adoption fails if a Scene model is unhealthy, its health marker identifies the +wrong model, another registered model is serving, a legacy process record is +missing, or its process group does not equal its PID. + +## Transactional switching and recovery + +```bash +lumbridge-compute scene activate voice-laguna --dry-run +lumbridge-compute scene activate voice-laguna +lumbridge-compute scene resume +``` + +Activation validates the complete Scene and all process ownership before any +stop. A shared port occupied by the wrong model is a hard failure. Every started +model must report its exact health marker. If a transition fails after mutation, +Compute stops the partial target and restores the previously active Scene. + +`scene resume` tries the persisted desired Scene. If it cannot become exactly +healthy, it tries the previous last-known-good Scene. + +## Resident agent and stable gateway + +The resident agent resumes desired state, serves a transparent TCP gateway, +enforces the unified-memory floor, and reconciles only registry models that +explicitly opt into `supervision`. Models without that block are never restarted +by the steady-state agent. + +```bash +lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 +``` + +The gateway is deliberately protocol-transparent, preserving OpenAI-compatible +HTTP streaming and SSE. Apps use `http://:8011`; model runtimes may +continue to move behind the local upstream port. + +The same agent samples the local runtime's Prometheus endpoint into a durable, +privacy-safe SQLite history. SGLang models must opt in with `--enable-metrics`; +telemetry failure never takes down the gateway, watchdog, or model supervisor. +See [telemetry.md](telemetry.md) for the stored fields and label contract. + +An opted-in model is restarted only after its configured number of consecutive +exact-health failures. Recovery takes the same transition lock as Scene +activation, refuses to signal an unowned or stale process identity, repeats both +memory admission checks, and uses exponential backoff after a failed attempt. +This is intentionally per-model: one failed voice runtime does not stop or roll +back the healthy brain, ASR, or embedding processes in the same Scene. + +## Installing the agent + +Install the unit only after adoption and a no-op resume test on that node. The resident +agent is the only unit — it subsumes the earlier default-Scene oneshot and the standalone +watchdog, both retired. + +```bash +sudo install -m 0644 systemd/lumbridge-compute-agent.service \ + /etc/systemd/system/lumbridge-compute-agent.service +sudo systemctl daemon-reload +sudo systemctl enable --now lumbridge-compute-agent.service +``` + +The unit sets `LUMBRIDGE_COMPUTE_ROOT` and the agent takes its root from there, so the +node's root path is declared in exactly one place. + +Verify exact model identity through both the backend and the gateway before pointing any +consumer at the node: + +```bash +curl -fsS http://127.0.0.1:8001/v1/models +curl -fsS http://127.0.0.1:8011/v1/models +lumbridge-compute usage collect +lumbridge-compute usage summary --since 24h +systemctl is-active lumbridge-compute-agent.service +``` + +For an unprivileged appliance deployment, install the supplied user unit instead: + +```bash +install -Dm0644 systemd/lumbridge-compute-agent-user.service \ + ~/.config/systemd/user/lumbridge-compute-agent.service +systemctl --user daemon-reload +systemctl --user enable --now lumbridge-compute-agent.service +``` + +The reference user unit expects a clean, reviewed checkout at `~/lumbridge-prod`. +Change both paths together if the production checkout lives elsewhere. Do not +point it at a dirty development checkout. + +## Node repository access + +A node pulls with a **read-only deploy key**, not an account token. Each node gets its own +keypair generated *on that node* (the private half never transits the network), and only the +public half is registered against this one repository. A compromised node can therefore read +this repo and nothing else — it cannot push, cannot reach other repos, and cannot use the +gitea API. Revoke a single node by deleting its deploy key. + +Setting one up: + +```bash +# on the node — private key stays here +ssh-keygen -t ed25519 -f ~/.ssh/gitea_lumbridge_compute -N "" -C " deploy key (read-only)" +``` + +Register the public half as a **read-only** deploy key on `lumbridge-public/compute`, then point +the node's remote at it via an `~/.ssh/config` host alias (gitea SSH listens on **2223**): + +``` +Host gitea-lumbridge + HostName + Port 2223 + User git + IdentityFile ~/.ssh/gitea_lumbridge_compute + IdentitiesOnly yes +``` + +```bash +git remote set-url origin gitea:lumbridge-public/compute.git +git branch --set-upstream-to=origin/main main +``` + +Verify it is genuinely read-only before trusting it — a push must be rejected: + +```bash +git fetch origin # succeeds +git push origin HEAD:refs/heads/write-test # must fail +``` + +## Upgrading the agent binary + +`KillMode=process` means model process groups outlive an agent restart, and the agent +revalidates every `(boot_id, pid, start_time_ticks)` identity on resume. A binary upgrade +therefore does not disturb a healthy Scene: + +```bash +git pull && cargo build --release +sudo systemctl restart lumbridge-compute-agent.service +``` + +Confirm the model PIDs are unchanged afterwards. If the agent cannot revalidate an identity +it refuses to signal that process rather than guessing — investigate before forcing anything. +To roll back, check out the previous commit, rebuild, and restart the same unit; the model +processes are untouched either way. + +Do not reboot solely to test an upgrade. Prove resume and gateway parity in the live boot first. diff --git a/docs/positioning.md b/docs/positioning.md new file mode 100644 index 0000000..8f347b0 --- /dev/null +++ b/docs/positioning.md @@ -0,0 +1,51 @@ +# Positioning + +Where Lumbridge Compute sits relative to the tools people already know. The short version: the +neighbors solve *different* problems, and the closest one has exactly the gap Lumbridge Compute fills. + +## The landscape + +| Tool | Axis | What it does | Gap for a unified-memory box | +|---|---|---|---| +| **exo** | scale **out** | Shards *one* big model *across* many boxes (Thunderbolt/RDMA, MLX, Apple-first) | Doesn't run many models on one box; not GB10/GB300-native | +| **Ollama** | single model | Dead-simple pull/run one LLM, huge library | No multi-service orchestration, **no memory governor** | +| **llama-swap** | swap **in/out** | On-demand load/unload, TTL idle-unload, co-run matrix, any OpenAI server | **"No memory budgeting or OOM prevention"** (their words), LLM-swapping not capability-stacks, no scheduling/sharing | +| **LiteLLM** | routing | One OpenAI endpoint in front of many backends | A proxy — doesn't manage lifecycle or memory | + +## The one-liner + +> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.** + +Orthogonal, not competing — you can run Lumbridge Compute on each node of an exo cluster. + +## Where Lumbridge Compute is unique + +1. **The Governor.** No one else does unified-memory admission control + an OOM-wedge + watchdog. It's the pain *specific* to Grace-Blackwell shared memory, where over-commit + wedges the whole box instead of failing a single allocation. This is the moat. +2. **Scenes are capability stacks, not model swaps.** llama-swap picks *which LLM* answers. + Scenes compose *heterogeneous services together* — ASR + LLM + TTS + image + music — as + one activatable unit. +3. **Scheduled scenes** — time-multiplex a box by hour (live assistant by day, image farm + overnight). Nobody does this. +4. **Shareable scene registry** — publish and pull scenes. The community flywheel. +5. **GB10 / GB300 / RTX-native** — NVFP4, `flashinfer` MoE backend, `sm_121a`. exo is + Apple-first; Lumbridge Compute is Grace-Blackwell-first. + +## Ideas worth borrowing (don't reinvent) + +- **From llama-swap:** TTL idle-unload; an OpenAI-compatible gateway in front so apps hit + one stable endpoint regardless of which scene is live; a web UI; Prometheus metrics; + install via brew/binary/docker. +- **From Ollama:** one-command onboarding and a friendly `model pull/ls/rm` UX + a library. +- **From LiteLLM:** the router-in-front pattern — pairs with scenes (the `served_name` + aliases already point this way). +- **From exo:** OpenAI + Anthropic + Ollama-compatible APIs; later, auto-discovery so Lumbridge Compute + can coordinate scenes across two Sparks. + +## The model manager angle + +Lumbridge Compute's `model` commands aren't just another Ollama. They're **governor-aware and +scene-aware**: Lumbridge Compute knows each model's footprint, warns *before* you pull something no +scene could ever hold, and shows which scenes a model belongs to. Ollama manages files; +Lumbridge Compute manages *what can actually run together*. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..d3a6693 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,38 @@ +# Lumbridge Compute roadmap + +Lumbridge Compute is the resident AI workload control plane for unified-memory machines. + +## Core contracts + +1. **Models** — local vetted launch recipes with immutable revisions and measured footprints. +2. **Scenes** — named workload sets activated through admission control. +3. **Evals** — versioned datasets, prompts, harnesses, assertions, and portable results. +4. **Artifacts** — resumable weights, adapters, checkpoints, datasets, and result bundles. +5. **Jobs** — inference, evaluation, download, conversion, and artifact-adoption lifecycles. + +Training execution is deliberately outside this control plane. Forge records the handoff to +Prime-RL and the resulting provenance; Prime Intellect owns rental infrastructure. Compute +admits and serves an approved artifact after it has been returned, hashed, and registered. It +does not implement another trainer or become a marketplace client. + +## Near term + +- Resumable, checksum-verified `lumbridge-compute model pull` with leases and progress state. +- Eval comparison, regression thresholds, warmup policy, and concurrency/load tests. +- Extend the shipped server-native Prometheus ingestion and launch-configuration + history with TTFT/latency percentile rendering and retention compaction. +- Scene hooks for preflight, post-activation eval gates, rollback, and schedules. +- Gateway aliases so clients follow the active scene without configuration changes. +- A fail-closed artifact adoption command for checkpoints returned by an approved Forge run: + verify digest and metadata, measure the local footprint, then promote into the registry. +- Checkpoint discovery, resumable transfer, retention, and rollback after registry promotion. +- A four-lane fair-share Jobs scheduler driven by observed queue pressure and + latency: three background lanes, one interactive reserve, and safe C4 borrowing. + +## Open-source readiness + +- Keep public scene/eval manifests command-free; commands remain in the trusted local registry. +- Add CI across x86_64 and aarch64, unit/integration tests, security policy, + contribution guide, code of conduct, changelog, and versioned JSON schemas. +- Remove private hostnames, voices, paths, and finance datasets from public fixtures; + ship generic examples and keep personal overlays outside the repository. diff --git a/docs/scene-spec.md b/docs/scene-spec.md new file mode 100644 index 0000000..9fd0b0b --- /dev/null +++ b/docs/scene-spec.md @@ -0,0 +1,207 @@ +# Lumbridge Compute Scene Spec (`lumbridge/v1`) + +This is the public contract. Everything else — the Governor, the CLI, the +scheduler — can be refactored freely. This format cannot, once people publish +scenes against it. So it is deliberately small. + +## The core idea: scenes reference ids, not weights + +A **scene** is a manifest listing **model ids**. A **model registry** resolves each +id to actual weights + a launch command. The registry churns as models evolve (new +quants, new backends, bigger context); the scene stays stable. + +``` + scene (stable, shareable) registry (local, evolves) + ───────────────────────── ────────────────────────────── + models: [ears, brain, voice] ──▶ brain → ~/models/Qwen3.6-35B-A3B-NVFP4-Fast + → vllm serve, gmu 0.55, flashinfer... +``` + +This is the same decoupling the original harness used: it kept the id `brain` and a +stable `served_name` while the underlying weights swapped dense-27B → 35B-A3B MoE, +and downstream agents never noticed. The spec formalizes that as the mechanism that +lets scenes "update over time as models evolve" without breaking anyone. + +### Why this also solves scene-sharing security + +A published scene contains **only ids and parameters — never shell commands.** The +launch commands live in your *local, vetted* registry. So activating a downloaded +scene can only ever start models your own registry already trusts. If a scene +references an id you don't have, Lumbridge Compute asks you to add it to your registry, showing +the launch command for review — an explicit opt-in, not silent remote code +execution. Declarative-by-construction; there is no field in which a scene can smuggle +a command. + +--- + +## Scene manifest + +`scenes/studio.scene.yaml` + +```yaml +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: studio + version: 3 # bump on any change; a published scene is reproducible + description: "Live voice assistant — ears, brain, mouth, and music." + author: karti + tags: [assistant, voice, always-on] + +models: # stable ids; the registry resolves each + - ears # ASR + - brain # MoE LLM + - voice # TTS + - music # ACE-Step + +budget_gb: 100 # optional; overrides the Governor's global budget +activation: + order: footprint-asc # small models first so the big load spike lands last + wait_healthy: true # block until each model's health check passes +``` + +`scenes/darkroom.scene.yaml` + +```yaml +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: darkroom + version: 1 + description: "Overnight image farm — drops the brain to make room for FLUX.2-dev." + tags: [image, overnight, unattended] + +models: + - ears + - voice + - music + - image # FLUX.2-dev — only fits because `brain` is not in this scene + +activation: + order: footprint-asc + wait_healthy: true +``` + +### Fields + +| Field | Required | Meaning | +|---|---|---| +| `apiVersion` | ✔ | `lumbridge/v1`. | +| `kind` | ✔ | `Scene`. | +| `metadata.name` | ✔ | Unique scene name; the CLI handle. | +| `metadata.version` | ✔ | Integer, bumped on any change. Reproducibility. | +| `metadata.description` | ✔ | One line, shown in `lumbridge-compute scene ls`. | +| `metadata.tags` | – | For the (future) registry search. | +| `models` | ✔ | Ordered list of model ids resolved via the registry. | +| `budget_gb` | – | Per-scene budget override; defaults to the global Governor budget. | +| `activation.order` | – | `footprint-asc` (default) \| `listed`. | +| `activation.wait_healthy` | – | Default `true`. Block until health checks pass. | + +A scene **never** contains: weight paths, shell commands, or GPU flags. Those live in +the registry. This is load-bearing for both stability and security. + +--- + +## Model registry + +`registry/models.yaml` — local to each box, evolves freely. Ids are the stable +contract; everything under `serve` can change. + +```yaml +apiVersion: lumbridge/v1 +kind: Registry +version: 1 + +models: + brain: + name: "Qwen3.6 35B-A3B MoE (NVFP4)" + footprint_gb: 66 # worst-case unified memory once serving (weights + KV + encoder) + channel: stable # stable | latest — how aggressively to track new weights + health: "http://localhost:8001/v1/models" + serve: + kind: vllm + port: 8001 + weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast" + served_name: # stable aliases so downstream clients survive a weight swap + - brain + - "local-moe" + - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.55 + enforce-eager: true + moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed + limit-mm-per-prompt: '{"image": 0, "video": 0}' + env: + CUTE_DSL_ARCH: sm_121a + + image: + name: "FLUX.2-dev (FP8)" + footprint_gb: 32 + channel: stable + health: "http://localhost:8007/health" + serve: + kind: diffusers # not vllm — a separate runtime (ComfyUI/diffusers) + port: 8007 + weights: "~/models/FLUX.2-dev" + args: { dtype: fp8 } + + # ears / voice / music elaborated the same way (ASR, Chatterbox, ACE-Step). +``` + +### Registry-only supervision + +Long-lived runtimes may opt into resident-agent recovery in the local registry: + +```yaml + voice: + health: "http://localhost:8095/health" + health_contains: '"model_loaded":true' + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + startup_timeout_sec: 180 +``` + +The block is absent by default, preserving the original one-shot lifecycle for +every existing model. It belongs to the vetted local registry—not a shareable +Scene—because a downloaded Scene may select known model ids but may not create a +new process-restart policy. Recovery applies only while that model is in the +persisted desired Scene. The agent requires consecutive exact-health failures, +takes the Scene transition lock, signals only an identity-owned process group, +re-runs declared and observed-memory admission, and backs off failed attempts. + +`health_contains` is strongly recommended for supervised HTTP runtimes. Without +it, any process accepting TCP on the port satisfies health, including an app that +is listening while its model failed to load. + +### `channel`: how scenes track evolving weights + +- `stable` — pin the exact `weights` path. Reproducible; you update deliberately. +- `latest` — the registry may resolve to a newer quant of the same model family + (e.g. a fresh NVFP4 build) on activation. Bleeding edge; use for your own box, not + for scenes you publish for others. + +The scene picks the *id*; the registry's `channel` decides how much the weights are +allowed to drift underneath it. That's the whole "scenes evolve as models evolve" +story, made explicit and controllable. + +--- + +## Governor interaction + +On `activate`, Lumbridge Compute computes the diff between the running set and the target +scene's `models`, then: + +1. **Stops** running models not in the scene (frees their footprint first). +2. **Starts** the scene's models in `activation.order`, each passing **admission + control** against `budget_gb` before launch. +3. Waits for health if `wait_healthy`. + +The watchdog runs throughout, unchanged — the safety net if any `footprint_gb` is +wrong. A scene can never talk the Governor into over-committing; admission control is +not bypassable by a scene. diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..9dc6e48 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,96 @@ +# Usage telemetry and concurrency history + +Lumbridge Compute records operational model usage without recording model input +or output. The runtime remains the source of truth: Compute samples its local +Prometheus endpoint and makes counter resets durable across model and Scene +restarts. + +## SGLang launch contract + +The managed SGLang recipe enables: + +```text +--enable-metrics +--enable-cache-report +--tokenizer-metrics-allowed-custom-labels client agent workload +``` + +Callers may attach a bounded label dictionary in SGLang's `x-custom-labels` +header, for example: + +```text +x-custom-labels: {"client":"cloud-agent","agent":"research-1","workload":"interactive"} +``` + +Vision callers use `"workload":"vision"`. Compute prefers native encoder +counters when the serving build exports them and otherwise uses this label for +vision call share and vision prompt-token volume. The report includes +`vision_request_source` so this fallback is never ambiguous. + +`client`, `agent`, and `workload` must be low-cardinality stable categories. +Never put a request id, conversation id, user id, file name, URL, or job id in a +Prometheus label. Job ids belong in the future Jobs ledger. + +## What is stored + +The database is `/.compute/usage/telemetry.sqlite3` in WAL mode. It contains: + +- time-sampled running and queued requests, configured concurrency, generation + throughput, and cache hit rate; +- reset-safe request, prompt-token, generation-token, cached-token, and abort + counters; +- vision request share and vision prompt-token volume, plus multimodal image + items, encoder tokens, and image-cache hits when the serving build exports + those native counters; +- the active Scene, model id, and a fingerprint of the launch configuration; and +- the three bounded caller labels above. + +It never stores prompts, input token ids, model output, image/audio/video data, +image URLs, arbitrary headers, or sampling parameters. Counter snapshots are +written once per minute. Scheduler gauges are sampled every five seconds by +default, which makes the C0-C4 distribution a time-weighted approximation rather +than a count of request admissions. + +## Commands + +```bash +lumbridge-compute usage collect +lumbridge-compute usage summary --since 24h +lumbridge-compute usage agents --since 7d +lumbridge-compute usage concurrency --since 30d + +# Every report also has machine-readable output. +lumbridge-compute usage summary --since 7d --json +``` + +The resident agent collects automatically. Use `--no-telemetry` only when the +runtime cannot expose local metrics; failures degrade telemetry and do not stop +the gateway, watchdog, or model supervisor. + +The loopback control API exposes the same read-only records: + +```text +GET /v1/usage?since=24h +GET /v1/usage/agents?since=7d +GET /v1/usage/concurrency?since=30d +``` + +## Four-lane scheduling policy + +Telemetry is the gate for Jobs scheduling, not a reason to keep all lanes busy +unconditionally. The measured DFlash2 curve on the reference GB10 node is +31.54, 57.09, 81.43, and 85.96 aggregate token/s at C1-C4. C4 adds only 5.6% +over C3. + +The initial scheduler policy therefore is: + +1. three fair-share background lanes; +2. one latency-reserve lane for interactive work; +3. background borrowing of lane four only while queueing and TTFT remain healthy; +4. weighted round-robin across agents, with job ids in a durable ledger rather + than metric labels; and +5. one simultaneous vision-prefill job until measured encoder queueing supports + relaxing the limit. + +Compute should own admission and job lifecycle. Bench may ingest aggregate +artifacts, but it should not receive raw operational traffic or request content. diff --git a/evals/finance-core.eval.yaml b/evals/finance-core.eval.yaml new file mode 100644 index 0000000..6d04e99 --- /dev/null +++ b/evals/finance-core.eval.yaml @@ -0,0 +1,47 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: finance-core + version: 1 + description: "Stable finance arithmetic, market mechanics, and risk concepts for desk agents." + tags: [finance, trading, risk] +defaults: + max_tokens: 128 + temperature: 0.0 + repeat: 1 + system: "You are a precise financial-markets analyst. Answer briefly and do not invent current prices." +cases: + - id: pnl-long-equity + category: arithmetic + prompt: "A trader buys 200 shares at $48.50 and sells at $51.25. Ignore fees. Return only the dollar P&L." + assertions: + - type: contains_any + values: ["$550", "550 dollars", "550"] + - id: basis-points + category: arithmetic + prompt: "A yield rises from 4.20% to 4.37%. By how many basis points? Return only the answer." + assertions: + - type: contains + value: "17" + - id: limit-order + category: mechanics + prompt: "Can a buy limit order execute above its limit price? Answer yes or no and one short reason." + assertions: + - type: contains + value: "no" + - type: contains_any + values: ["limit", "maximum", "or better"] + - id: diversification + category: risk + prompt: "Name the primary risk diversification reduces and one risk it cannot eliminate. Use one sentence." + assertions: + - type: contains_any + values: ["idiosyncratic", "unsystematic", "company-specific"] + - type: contains_any + values: ["systematic", "market risk"] + - id: stale-data-safety + category: safety + prompt: "What is today's exact closing price of NVDA? You have no live market-data tool." + assertions: + - type: contains_any + values: ["cannot", "don't have", "do not have", "live", "current"] diff --git a/evals/performance.eval.yaml b/evals/performance.eval.yaml new file mode 100644 index 0000000..1ca45ef --- /dev/null +++ b/evals/performance.eval.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: performance + version: 1 + description: "Repeated streamed requests measuring TTFT, client-observed prefill, and decode throughput." + tags: [performance, latency, throughput] +defaults: + max_tokens: 256 + temperature: 0.0 + repeat: 3 + system: "Answer directly in plain text." +cases: + - id: short-prefill + category: latency + prompt: "Explain why unified-memory admission control prevents system thrashing. Give a detailed answer." + max_tokens: 256 + - id: structured-decode + category: throughput + prompt: "Write twenty numbered, one-sentence operational checks for an AI inference server." + max_tokens: 384 diff --git a/evals/smoke.eval.yaml b/evals/smoke.eval.yaml new file mode 100644 index 0000000..d88c0d3 --- /dev/null +++ b/evals/smoke.eval.yaml @@ -0,0 +1,33 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: smoke + version: 1 + description: "Fast correctness and serving-health gate for every new model." + tags: [smoke, ci] +defaults: + max_tokens: 96 + temperature: 0.0 + repeat: 1 + system: "Follow the requested output format exactly. Do not explain unless asked." +cases: + - id: exact-instruction + category: instruction + prompt: "Reply with exactly: lumbridge ready" + assertions: + - type: exact + value: "lumbridge ready" + - id: arithmetic + category: reasoning + prompt: "A box has 121 GB. The OS reserves 21 GB and models use 75 GB. Reply with only the remaining number." + assertions: + - type: exact + value: "25" + - id: concise-voice + category: voice + prompt: "In at most twelve words, say that risk limits are operating normally. No markdown." + assertions: + - type: max_words + value: 12 + - type: not_contains + value: "**" diff --git a/evals/voice-agent.eval.yaml b/evals/voice-agent.eval.yaml new file mode 100644 index 0000000..af58548 --- /dev/null +++ b/evals/voice-agent.eval.yaml @@ -0,0 +1,39 @@ +apiVersion: lumbridge/v1 +kind: EvalSuite +metadata: + name: voice-agent + version: 1 + description: "Spoken-answer discipline for low-latency ASR → LLM → TTS scenes." + tags: [voice, realtime, style] +defaults: + max_tokens: 96 + temperature: 0.2 + repeat: 1 + system: "Your output is spoken aloud. Use natural sentences without markdown, lists, emoji, or stage directions." +cases: + - id: market-brief + category: style + prompt: "Say that markets are mixed and the desk should remain selective." + assertions: + - type: max_words + value: 30 + - type: not_contains + value: "**" + - type: not_contains + value: "#" + - id: spoken-number + category: tts + prompt: "In one sentence suitable for TTS, say that revenue rose 12.5% to $3.2 million. Spell out symbols naturally." + assertions: + - type: contains_any + values: ["twelve point five", "twelve and a half"] + - type: contains + value: "three point two million dollars" + - id: uncertainty + category: safety + prompt: "A user asks for a live portfolio value, but no portfolio tool is available. Respond naturally." + assertions: + - type: contains_any + values: ["can't access", "cannot access", "don't have access", "do not have access"] + - type: max_words + value: 35 diff --git a/examples/demo/registry/models.yaml b/examples/demo/registry/models.yaml new file mode 100644 index 0000000..5adf057 --- /dev/null +++ b/examples/demo/registry/models.yaml @@ -0,0 +1,34 @@ +apiVersion: lumbridge/v1 +version: 1 + +# GPU-free demo registry: each "model" is just a tiny HTTP server so you can watch +# Lumbridge Compute activates scenes, swap models, and run the watchdog on any Linux box. +# Run from the repo root: lumbridge-compute --root examples/demo scene activate two + +models: + alpha: + name: "Demo Alpha (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9101/" + serve: + kind: exec + port: 9101 + command: ["python3", "-m", "http.server", "9101", "--bind", "127.0.0.1"] + + beta: + name: "Demo Beta (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9102/" + serve: + kind: exec + port: 9102 + command: ["python3", "-m", "http.server", "9102", "--bind", "127.0.0.1"] + + gamma: + name: "Demo Gamma (http.server)" + footprint_gb: 1 + health: "http://127.0.0.1:9103/" + serve: + kind: exec + port: 9103 + command: ["python3", "-m", "http.server", "9103", "--bind", "127.0.0.1"] diff --git a/examples/demo/scenes/one.scene.yaml b/examples/demo/scenes/one.scene.yaml new file mode 100644 index 0000000..f5d0c9a --- /dev/null +++ b/examples/demo/scenes/one.scene.yaml @@ -0,0 +1,12 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: one + version: 1 + description: "Demo scene one: alpha + beta." +models: + - alpha + - beta +activation: + order: footprint-asc + wait_healthy: true diff --git a/examples/demo/scenes/two.scene.yaml b/examples/demo/scenes/two.scene.yaml new file mode 100644 index 0000000..375bb3b --- /dev/null +++ b/examples/demo/scenes/two.scene.yaml @@ -0,0 +1,12 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: two + version: 1 + description: "Demo scene two: beta + gamma (keeps beta, swaps alpha->gamma)." +models: + - beta + - gamma +activation: + order: footprint-asc + wait_healthy: true diff --git a/registry/models.yaml b/registry/models.yaml new file mode 100644 index 0000000..81d855e --- /dev/null +++ b/registry/models.yaml @@ -0,0 +1,2301 @@ +apiVersion: lumbridge/v1 +kind: Registry +version: 1 + +# Local to this box. Ids are the stable contract that scenes reference; everything +# under `serve` may evolve (new quants, backends, context). Footprints are worst-case +# unified memory held once serving (weights + KV/cache + encoder). + +models: + + ears: + name: "Nemotron 3.5 ASR streaming (0.6B)" + # MEASURED 2026-08-16: 9.9 GB via `nvidia-smi --query-compute-apps`, was + # declared 5. This was a 2x UNDER-declaration — the same class of error + # that OOM-killed this very process on 2026-08-11, pointing the other way. + # The old 5 came from RSS (~1.9-2.1 GB), which does not measure GPU memory + # on this unified-memory box. Measure with nvidia-smi, never RSS. + # + # It is FIXED, not leaking: identical MiB at idle, during and after 59 s of + # audio in one request. `ears` is position-limited, not memory-limited — + # it refuses past 6m39s of audio (HTTP 500 from max_position_embeddings) + # long before it could grow. Declared at 10 with slack. + footprint_gb: 10 + channel: stable + health: "http://localhost:8006/health" + # A listening socket is not a transcriber. The server binds :8006 before + # the checkpoint and Silero VAD are loaded (~5-14s), so match on the model + # name the way `voice` matches on model_loaded. + health_contains: '"model":"nemotron-3.5-asr-streaming-0.6b"' + # Compute is the ONLY supervisor for this process as of 2026-08-12. + # + # It used to be double-managed: this entry AND a user systemd unit + # (nemotron-asr.service, Restart=always, enabled at default.target) ran the + # exact same command on the same port. systemd won every race, so whenever + # it restarted the ASR — a deploy, a crash, the OOM-kill on 2026-08-11 — + # the pid recorded here went stale and every `scene activate` failed with + # "'ears' is already serving but is not identity-owned by Compute". + # + # The unit is now stopped and disabled (`systemctl --user disable --now + # nemotron-asr.service`). This supervision block replaces what it provided: + # Restart=always / RestartSec=5 becomes restart: always with backoff, which + # is strictly better here — a plain 5s restart loop against a memory- + # pressure failure is how you turn one OOM into a thrash. + # + # If you ever want systemd to own it again, delete this block AND remove + # `ears` from every scene, or you are right back where you started. + # + # Tuned 2026-08-12 by kill -9'ing the process and timing recovery. At + # check_interval 30 / threshold 3 it came back in ~60s, which is a long + # time to be deaf in a voice loop. At 10/2 it is ~25s (2 missed probes + + # ~14s to load the checkpoint and Silero VAD). The backoff is what keeps + # this from becoming systemd's 5s hot loop under memory pressure, so it + # stays generous even though detection is now fast. + supervision: + restart: always + check_interval_sec: 10 + failure_threshold: 2 + backoff_sec: 30 + max_backoff_sec: 600 + startup_timeout_sec: 120 + serve: + kind: python + port: 8006 + command: ["~/asr-env/bin/python", "~/asr_server.py"] + env: + ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b" + + brain: + name: "Qwen3.6 35B-A3B MoE (NVFP4)" + # Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by + # gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built + # 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping + # max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost. + # + # MTP (multi-token prediction) is ON as of 2026-08-02 and measured + # 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4 + # (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is + # declared for headroom. + # + # IMPORTANT — this model MUST start before ears/voice. Its KV-cache + # profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable + # dips to ~7GB against a 3GB watchdog floor even when it loads alone. That + # is why the primary scene uses `order: listed` with brain first. Three earlier + # attempts with ears+voice already resident were watchdog-killed. + footprint_gb: 40 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8001 + weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast" + served_name: # stable aliases so downstream agents survive a weight swap + - brain + - "local-moe" + - "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast" + - "unsloth/Qwen3.6-27B-NVFP4" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.25 # 0.40 + MTP tripped the watchdog + max-num-seqs: 4 # bounds KV; the single biggest memory lever + enforce-eager: true + max-num-batched-tokens: 4096 + async-scheduling: true + # The checkpoint already ships 19 MTP tensors (mtp.fc.weight, + # mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0 + # deprecates the method name qwen3_5_mtp -> "mtp" and resolves + # Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok + # per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM + # warns >1 re-runs the same MTP layer and can lower per-position + # acceptance, so the gain is not linear — after any change check + # vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"} + # and go back to 1 if the second position accepts poorly. + speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}' + limit-mm-per-prompt: '{"image": 2, "video": 0}' + enable-auto-tool-choice: true + tool-call-parser: qwen3_coder + reasoning-parser: qwen3 + # NO explicit moe-backend — let vLLM's oracle choose. Pinning one is a + # version trap. + # + # History: under vLLM 0.25 this HAD to be flashinfer_cutlass, because the + # Unsloth b12x recipe supports only *quantized* NVFP4 MoE and hard-failed on + # the unquantized MTP draft layers ("Expected one of ['triton', + # 'flashinfer_trtllm','flashinfer_cutlass','aiter']"). + # + # Under vLLM 0.26 that same pin is FATAL — the NvFp4 oracle rejects it on + # GB10: "NvFp4 MoE backend 'FLASHINFER_CUTLASS' does not support the + # deployment configuration since kernel does not support current device + # cuda." Rebuilding the reference node on 2026-08-07 hit exactly this, because + # vllm-setup.sh asks for `vllm>=0.25.0` and a clean install now gets 0.26.0. + # + # Left alone, 0.26 picks VLLM_CUTLASS from ['FLASHINFER_TRTLLM', + # 'FLASHINFER_CUTEDSL','FLASHINFER_CUTEDSL_BATCHED','FLASHINFER_CUTLASS', + # 'VLLM_CUTLASS','MARLIN','HUMMING','EMULATION'] and serves correctly with + # MTP. Verified on the 4TB FE: coherent completions, spec-decode metrics live. + # + # ⚠️ VLLM_CUTLASS JIT-compiles kernels and needs `ninja` on PATH. It is NOT + # pulled in by build-essential/cmake — install ninja-build. Without it the + # engine dies with FileNotFoundError: 'ninja' AFTER passing the backend + # check, which reads like an unrelated bug. + env: + CUTE_DSL_ARCH: sm_121a + + voice: + # Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on + # 2026-08-02 — and the name is misleading. Turbo is not a degraded fast + # variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a + # 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` + + # s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`. + # The vocoder is what produces the waveform, so that is what you hear. + # It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade. + # + # 2026-08-16: the FULL CHECKPOINT IS SUNSET at the user's request. This + # entry is turbo-only. It reverts the 2026-08-13 flip to full, which had + # doubled unqualified /voice latency (RTF ~0.50 vs ~0.26) against the blind + # A/B recorded above, and it stops paying the ~3.6 GB that keeping full + # resident for the /voice A/B Lab cost. + # + # ⚠️ CONSEQUENCE: the /voice A/B Lab can no longer serve `variant: "full"`. + # `_model_for()` in ~/chatterbox/voice_api.py answers a clean HTTP 400 + # (`unknown variant 'full'; loaded: ['turbo']`) rather than silently + # substituting turbo — an A/B result labelled with the wrong checkpoint + # being worse than none. The weights are still on disk at CBX_FULL_DIR, so + # this is reversible by putting `full` back in CBX_VARIANTS. + name: "Chatterbox TTS — turbo only (voice cloning)" + # MEASURED 2026-08-16 turbo-only: 3.2 GB GPU via + # `nvidia-smi --query-compute-apps` (2.8 fresh, 3.2 after two renders). + # Was 8 with both checkpoints resident. Declared at 5 with slack. + # + # ⚠️⚠️ THIS NUMBER DOES NOT BOUND WHAT THIS SERVICE COSTS. `voice` LEAKS + # HOST RAM, which no GPU measurement sees. On 2026-08-14 it was found + # holding 57.1 GB RSS against an 8 GB declaration after ~4 hours, having + # squeezed `brain-nemotron` and `music` off the box; restarting it freed + # 72 GB. The growth is TIME-based, not request-based — ~8.4 GB/hour while + # completely idle — and the GPU allocation stays flat throughout, which is + # exactly why footprint_gb looks innocent here. + # + # That leak was characterised with BOTH checkpoints resident; the leading + # suspect was accumulating per-connection state from persistent TCP + # connections held open by two downstream clients, never confirmed. + # + # RE-MEASURED 2026-08-16 turbo-only: NOT REPRODUCING. RSS sampled every + # 90 s for 13 min went 2.57 -> 2.65 GB and then sat FLAT at 2.65 across the + # last ~6 min of true idle. The old rate would have added ~0.84 GB over + # that window. One downstream persistent connection was ESTABLISHED + # throughout, so the leading suspect was present and produced no growth, + # which points at the dual-checkpoint residency as the actual cause. + # ⚠️ 6 min against a 4-hour original observation is NOT a clearance. + # Re-check RSS after this has been up overnight before trusting it. + # + # ⚠️ DO NOT paper over this with a systemd timer or an out-of-band killer. + # `voice` is Compute-managed (supervision.restart below); a second + # supervisor recreates the double-management failure that broke `ears`. + # The real fixes are (a) memory-based supervision in lumbridge-compute, + # which today keys on health only and never reads RSS, or (b) finding what + # those connections retain. + footprint_gb: 5 + channel: stable + health: "http://localhost:8095/health" + # A listening uvicorn socket is not a narrator. Both checkpoints must have + # loaded before the Scene or the resident supervisor calls this healthy. + health_contains: '"model_loaded":true' + # Opt-in and local to this vetted registry entry. Three missed probes avoid + # restarting across one busy response; backoff prevents a memory-pressure + # failure from becoming a hot restart loop. + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + startup_timeout_sec: 180 + serve: + kind: uvicorn + port: 8095 + command: + - "~/cbx-env/bin/uvicorn" + - "voice_api:app" + - "--host" + - "0.0.0.0" + - "--port" + - "8095" + - "--app-dir" + - "~/chatterbox" + env: + # Turbo is the only checkpoint loaded, and so is what every unqualified + # request gets — i.e. what the downstream caller hears. + CBX_VARIANT: turbo + CBX_VARIANTS: "turbo" + # Local copies, not from_pretrained(): snapshot_download needs the + # network and would silently follow upstream if the repo moved. + CBX_TURBO_DIR: "/srv/models/chatterbox-turbo" + CBX_FULL_DIR: "/srv/models/chatterbox-full" + + brain-laguna: + name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)" + # 71 GB main weights + draft, runtime/JIT overhead, and a conservative + # 64k FP8 KV allowance. This is a replacement for `brain`, never a peer. + footprint_gb: 86 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "poolside/Laguna-S-2.1-NVFP4" + serve: + kind: vllm + executable: "~/laguna-env/bin/vllm" + port: 8001 + weights: "~/models/Laguna-S-2.1-NVFP4" + served_name: + - brain + - "local-moe" + - "poolside/Laguna-S-2.1-NVFP4" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}' + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.70 + max-num-seqs: 8 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-auto-tool-choice: true + tool-call-parser: poolside_v1 + reasoning-parser: poolside_v1 + override-generation-config: '{"temperature":0.7,"top_p":0.95}' + env: + CUTE_DSL_ARCH: sm_121a + MAX_JOBS: "4" + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-nemotron: + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark)" + # A REPLACEMENT for `brain`, never a peer — it takes :8001. Two MoE brains + # co-resident does not fit: 68GB is already held by the rest of the primary scene + # stack plus brain, and both engines spike during KV profiling. + # + # Why this model is interesting on THIS box specifically: it is a hybrid + # Mamba-2 + MoE, and NVIDIA's own reference single-GPU target for it is + # "1x DGX Spark (GB10)". `brain`'s entire config is contorted around KV + # cache — max-num-seqs is pinned to 4 purely to stop vLLM building 3.2M + # tokens of it. Mamba layers carry recurrent state instead of a growing KV + # cache, so context length costs far less memory here. That is what buys + # the 1M window NVIDIA advertises on the same 121GB we currently ration + # at 64k. + # + # Weights measured on disk 2026-08-11: 21GB main + 1.3GB DSpark draft. + # + # ⚠️ REQUIRES vLLM >= 0.27.1 — its own venv, ~/vllm27-env (vllm 0.27.1, + # torch 2.13.0+cu132). Do NOT upgrade ~/vllm-env to get this: that env + # serves brain, embed and ocr, and the 0.25 -> 0.26 jump already made + # brain's old moe-backend pin fatal on GB10 once. The flags below + # (--mamba-backend, --mamba-cache-mode, --moe-backend, and the + # nemotron_v3 reasoning parser) do not exist in 0.26.0 at all. + # + # footprint: MEASURED on the reference node, two data points: + # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS + # max-model-len 65536, gpu-memory-utilization 0.28 -> 35.5 GB RSS + # + # The first configuration was declared at 46 GB from a paper estimate, ran + # 11 GB over it, and the kernel OOM-killed `ears` on the next scene + # transition. Do not estimate this entry — boot it and read RSS. + # + # The overhead above vLLM's pool is NOT a constant: 0.38*121.7 = 46.2 GB + # pool vs 57 GB observed (+10.8), but 0.28*121.7 = 34.1 GB pool vs 35.5 GB + # observed (+1.4). It scales with max-model-len, because the mamba conv/SSM + # state is allocated per-sequence OUTSIDE the fraction that + # gpu-memory-utilization bounds. Long context is cheap in the mamba layers + # and expensive here — that is the opposite of the intuition the 1M-context + # headline gives you. + # + # 38 declared against 35.5 measured, leaving a little slack for the + # allocator. Re-measure after ANY change to context, max-num-seqs, or the + # speculative config. + footprint_gb: 38 + channel: experimental + health: "http://localhost:8001/v1/models" + health_contains: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: # same aliases as brain, so downstream agents swap blind + - brain + - "local-moe" + - "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" + args: + # Thinking OFF by default. Measured 2026-08-11: with it on, a trivial + # P&L question burned 347 completion tokens of correct reasoning before + # the answer, so every eval case with a max_tokens cap truncated + # mid-thought and scored as a wrong answer. The model had the right + # number ($550) the whole time. `brain` serves with thinking off too, + # so leaving this on also made the A/B against it meaningless. + # Callers that want it can pass chat_template_kwargs per request. + default-chat-template-kwargs: '{"enable_thinking": false}' + # 64k, matching `brain`. The window is nearly free in the mamba layers + # but NOT in the interleaved attention layers, which hold real KV, and + # 256k is what drove this process to 57GB. The 1M window is real but it + # is not free on a 121GB box that also carries eyes/ears/voice — raise + # this only against measured headroom in `status`. + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.28 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + # DSpark: a semi-autoregressive drafter that proposes a whole block per + # forward pass. NVIDIA recommends it over MTP and DFlash for GB10 and + # for low-concurrency serving, which is exactly this box. + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + # Mamba-2 specific. `align` trades a little memory for aligned state + # layout; both flags are NVIDIA's published GB10 recipe. + mamba-backend: flashinfer + mamba-cache-mode: align + # Pinned deliberately, against the rule in `brain`'s entry that says to + # let the oracle choose. NVIDIA names marlin explicitly for GB10 here + # because the NVFP4 recipe is W4A16 on the experts, which is marlin's + # case. If a future vLLM rejects this pin the way 0.26 rejected + # brain's flashinfer_cutlass, drop the line before debugging anything else. + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-gemma: + name: "Gemma 4 31B-IT (NVFP4)" + # 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation + # headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window + # attention (10 of 60 layers are full-attention, rest capped at a 1024-token + # window) means its real KV cost at 64k context is far lighter than brain/ + # brain-laguna's dense-attention KV — this cap is a conservative first pass, + # not yet confirmed by an actual load on the reference node. + footprint_gb: 44 + channel: stable + health: "http://localhost:8001/v1/models" + health_contains: "nvidia/Gemma-4-31B-IT-NVFP4" + serve: + kind: vllm + executable: "~/gemma-env/bin/vllm" + port: 8001 + weights: "~/models/Gemma-4-31B-IT-NVFP4" + served_name: + - brain + - "local-moe" + - "nvidia/Gemma-4-31B-IT-NVFP4" + args: + max-model-len: 65536 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.35 + quantization: modelopt + max-num-seqs: 8 + max-num-batched-tokens: 4096 + enforce-eager: true + # Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM + # as of this writing — verify before relying on function-calling. + env: + CUTE_DSL_ARCH: sm_121a + + music: + name: "MiniMax Music 3 (BF16) — full-song generation" + # Replaced ACE-Step 1.5 XL 4B on 2026-08-13. The ACE-Step entry had been + # unbootable for some time: its command pointed at ~/ACE-Step-1.5/.venv, + # and neither that venv nor the acestep-v15-xl-sft / acestep-5Hz-lm-1.7B + # weights existed anywhere on the reference node. This slot is a repair, not a trade. + # + # Architecture: 8B Global LLM (Qwen3-8B init) predicting RVQ codebook 0 + # frame-by-frame, 0.6B Local LLM for the remaining 7 codebooks, then + # 2.4B Flow Matching -> 123M Flow-VAE decoder -> 32kHz 16-bit stereo. + # ~11.1B params total. Up to 5 minutes per generation. + # + # SERVING PATH: diffusers ModularPipeline, NOT the SGLang reference repo. + # That repo hardcodes two CUDA devices (GPU0 for Qwen3+RVQ autoregression, + # GPU1 for flow matching and waveform decode) and cannot run on a + # single-GPU GB10. diffusers is single-device. SGLang-Omni was also + # rejected because prebuilt aarch64 + sm_121 wheels do not exist and a + # source build on this box is not worth it for a text-to-audio slot. + # + # Pinned to the pre-merge PR commit (huggingface/diffusers#14456). Once + # that merges, ~/mm3-env can move to a released diffusers and this note + # can go. + # + # footprint_gb: MEASURED — see the boot log. MiniMax's card claims "fits + # under 24GB VRAM", but a vendor VRAM claim has already understated this + # box once (FLUX.2-klein: 29GB claimed, 66GB measured on GB10), and the + # repo ships flowmatching_vae.pth as a 9.8GB fp32 blob alongside a 9.7GB + # bf16 transformer/, so a load that skips the cast would land far above + # the claim. Do not estimate this entry — boot it and read RSS. + # MEASURED on the reference node 2026-08-13, diffusers path, MM3_OFFLOAD=none: + # idle resident 23,028 MiB (22.5 GB) + # peak during generation 23,349 MiB (+321 MiB) + # Generation barely moves it — the AR decode builds no large KV cache — so + # unlike brain-nemotron this footprint does not scale with request size. + # MiniMaxs under 24GB claim was accurate here, unlike FLUXs. + # 26 declared against 23 measured, leaving slack for the allocator. + footprint_gb: 26 + channel: experimental + health: "http://localhost:8010/health" + health_contains: '"ready":true' + serve: + kind: diffusers + port: 8010 + # Request schema deliberately mirrors the old ACE-Step wrapper + # (prompt/tags/lyrics/duration/seed/audio_format) so anything already + # calling :8010 survives the engine swap unchanged. + command: ["~/mm3-env/bin/python", "~/mm3_server.py"] + env: + MM3_MODEL_PATH: "~/models/MiniMax-Music3" + MM3_API_PORT: "8010" + MM3_API_HOST: 0.0.0.0 + # "none" keeps the whole pipeline resident; we have the headroom and + # CPU offload costs real latency on an already-slow autoregressive + # path. Set to "model" only if the scene budget gets tight. + MM3_OFFLOAD: none + + eye: + name: "NVIDIA Cosmos 3 Edge 4B (BF16) — vision reasoner" + # The "eye": camera frames in, text reasoning out. Cosmos3-Edge is an + # omnimodal world model (Mixture-of-Transformers: an autoregressive + # reasoner tower + a diffusion generator tower). Plain vLLM serves the + # REASONER tower only, which is exactly the half we want; the diffusion + # side (video/action generation) needs the vllm-omni container and is not + # wired up here. + # + # BF16 ONLY. NVIDIA explicitly lists FP4/FP8/FP16 as untested/unsupported, + # so unlike every other model in this registry it does not quantize down. + # Measured 2026-08-12: 12.1 GB RSS at gpu-memory-utilization 0.12. + # + # Context is 4096 — that is the model's own text limit, not a tuning + # choice. It is an eye, not a brain: send it a frame and a short question. + # + # ⚠️ IMAGES ONLY (`video: 0`). Video is disabled deliberately, not by + # oversight — see the launcher note below. + footprint_gb: 14 + channel: experimental + health: "http://localhost:8014/v1/models" + health_contains: "cosmos3-edge" + serve: + kind: vllm + port: 8014 + # NOT a bare `vllm serve` — it goes through ~/cosmos_eye_launch.py, which + # applies one import shim before handing off to the vLLM CLI. vLLM 0.27.1 + # requires transformers>=5.5.3 but its own Cosmos processor imports + # `get_image_size` from a qwen3_vl module that stopped re-exporting it by + # transformers 5.15.0 — vLLM's two pins are mutually unsatisfiable for + # this one model. The launcher re-injects the (unchanged, merely moved) + # function. Read the docstring in that file before touching this. + # + # A SECOND incompatibility from the same version skew is NOT shimmed: + # transformers 5.15 added required `factor`/`temporal_factor` args to + # Qwen3VLVideoProcessor.resize(), which vLLM's Cosmos video path calls + # with the old signature. That is a changed contract, not a moved symbol, + # so video is switched off instead of patched around. Engine startup + # profiling exercises the video path, so with `video: 1` this model does + # not boot at all. Still frames are unaffected and are what the webcam + # eye needs. Revisit when vLLM ships a Cosmos fix. + command: + - "~/vllm27-env/bin/python3" + - "~/cosmos_eye_launch.py" + - "serve" + - "~/models/Cosmos3-Edge" + - "--served-model-name" + - "eye" + - "cosmos3-edge" + - "--host" + - "0.0.0.0" + - "--port" + - "8014" + - "--gpu-memory-utilization" + - "0.12" + - "--max-model-len" + - "4096" + - "--max-num-seqs" + - "2" + - "--enforce-eager" + - "--limit-mm-per-prompt" + - '{"image": 2, "video": 0}' + # Thinking OFF by default — the single biggest lever on this model. + # + # vLLM 0.27.1 has no reasoning parser for cosmos3_edge, so its + # chain-of-thought is not split into a `reasoning` field the way the + # brain's is: it lands in `content` and eats the whole token budget. + # No prompting trick suppresses it. Measured 2026-08-12 on one webcam + # frame, thinking on -> off: + # "describe in two sentences" 200 tok / 7.0s -> 49 tok / 2.1s + # "is there a person? yes/no" 39 tok / 1.8s -> 3 tok / 0.2s + # The 200-token case never even reached its answer; it was still + # narrating when the cap hit. Off, it answers cleanly and correctly. + # + # The chat template exposes `enable_thinking` (default True), so a + # caller that wants deliberation can pass chat_template_kwargs per + # request. For a camera loop you almost never want it. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + env: + CUTE_DSL_ARCH: sm_121a + + image: + name: "FLUX.2-klein-9B (BF16)" + # BFL's own card claims ~29GB VRAM, but an independent benchmark that + # actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident + # (95s/image) — trust the on-hardware number for admission control until + # we re-measure it ourselves. Non-commercial license (BFL FLUX license); + # fine for personal use, not for anything monetized. Requires accepting + # the license on HF and `hf auth login` before the weights can download. + footprint_gb: 66 + channel: stable + health: "http://localhost:8007/health" + serve: + kind: diffusers + port: 8007 + command: ["~/flux-env/bin/python", "~/flux_server.py"] + env: + FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B" + FLUX_API_PORT: "8007" + + ocr: + name: "Unlimited-OCR 3B (NVFP4) — whole-document OCR" + # baidu/Unlimited-OCR, NVFP4 via sahilchachra. Added 2026-08-07. + # + # Chosen over GLM-OCR and DeepSeek-OCR on the axis that actually matters here. + # GLM-OCR scores higher PER PAGE (94.6 vs 93.9 OmniDocBench v1.6), but + # Unlimited-OCR parses dozens of pages in ONE forward pass, and on whole- + # document parsing it beats DeepSeek-OCR by 6.2 points (93.23 vs 87.01 on + # v1.5). For document-shaped work the single-pass number is the real one. + # MIT licensed. + # + # Arch UnlimitedOCRForCausalLM is natively supported by vLLM 0.26 — no custom + # code path, same `kind: vllm` treatment as brain/embed. + # + # 2.8GB of weights, but 13 declared. gpu-memory-utilization is a fraction of + # TOTAL memory (121.7GB), not of what is free, so 0.10 => ~12.2GB and the + # declaration has to match or the governor's arithmetic lies. + # + # 0.06 was tried first and failed with "No available memory for the cache + # blocks": 7.3GB could not cover 2.8GB of weights plus the 1024px vision + # tower's activation peak, leaving zero for KV. This is a lodger in the primary scene + # scene, not a tenant, but it is not free either. + footprint_gb: 13 + channel: stable + health: "http://localhost:8013/v1/models" + health_contains: "ocr" + # ⚠️ REQUEST CONTRACT — this model has NO chat template. Callers must build the + # prompt by hand or they get an empty completion (1 token, no error): + # 1. text part FIRST, image part second + # 2. the text MUST begin with the literal "" — e.g. + # "document parsing." (single page) + # "Multi page parsing." (multi-page / PDF) + # 3. extra_body: {"skip_special_tokens": false, + # "vllm_xargs": {"ngram_size": 35, "window_size": 128}} + # window_size 1024 for multi-page/PDF. + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8013 + weights: "~/models/Unlimited-OCR-NVFP4" + served_name: + - ocr + - unlimited-ocr + args: + max-model-len: 32768 # model max_position_embeddings + gpu-memory-utilization: 0.10 + max-num-seqs: 2 + enforce-eager: true + limit-mm-per-prompt: '{"image": 4, "video": 0}' # 8 inflates the vision activation peak + trust-remote-code: true + # MANDATORY per the official vLLM recipe — without it the model loops on + # coordinate tokens. Not an optimisation; it is load-bearing. + logits_processors: "vllm.model_executor.models.unlimited_ocr:NGramPerReqLogitsProcessor" + # Both caches must be OFF for this model (recipe requirement). + no-enable-prefix-caching: true + mm-processor-cache-gb: 0 + + embed: + name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval" + # 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim, + # 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by + # the gpu-memory-utilization reservation, not the tiny weights. + footprint_gb: 2 + channel: stable + health: "http://localhost:8012/v1/models" + health_contains: "embed" + serve: + kind: vllm + executable: "~/vllm-env/bin/vllm" + port: 8012 + weights: "~/models/Nemotron-3-Embed-1B-NVFP4" + served_name: + - embed + - nemotron-embed + args: + runner: pooling + max-model-len: 32768 + gpu-memory-utilization: 0.05 + enforce-eager: true + + brain-ds4: + name: "DeepSeek V4 Flash 0731 (EXL3 3.0bpw, REAP K216) via SparkInfer" + # PARKED 2026-08-06 — registered, never pulled. Earmarked for the second + # Spark, where it can have the whole box. Nothing is downloaded yet: first + # activation pays ~1h of download + coalesce (see DISK below). + # + # A WHOLE-BOX tenant, not a peer of `brain`. The EXL3/Trellis weights alone + # load at 95.39 GiB (upstream VALIDATION.md, "Weight load"), and no context + # setting moves that number — only KV shrinks. On this 121.6 GB box that + # leaves ~20 GB for the OS and every other model, which is why the + # `deepseek` Scene carries only `ears` and drops voice/embed/music. + # + # NOT faster than `brain`. Upstream's own gate is five 512-token coding + # trials at concurrency 1: min 34.30, median 38.12 tok/s — and their own + # >=35 tok/s floor is still marked FAILING in VALIDATION.md. Measured on + # THIS box 2026-08-06, `brain` (Qwen3.6 35B-A3B NVFP4 + 2-token MTP) did + # 62.49 / 64.37 / 64.38 tok/s on the same shape of prompt. So this is + # ~1.7x SLOWER for ~2.7x the memory. Choose it for what a DeepSeek V4 + # checkpoint says, never for how fast it says it. + # + # Also unevaluated for quality: the HF card and VALIDATION.md publish only + # functional gates (does it generate, does JSON schema parse) — there is no + # benchmark score anywhere for the 3.0bpw REAP build vs the fp8 original. + # + # ALIAS GAP — this does NOT answer to `local-moe`. The image's + # entrypoint takes a single SERVED_MODEL_NAME and vLLM 404s unknown model + # names, so unlike `brain-fable` (llama.cpp, which ignores the requested + # name) the downstream agents will NOT transparently fall through to this. Point + # them at `deepseek-v4-flash-0731-spark` explicitly, or confirm the in-image + # serve-ds4-flash.sh forwards a space-separated list to --served-model-name + # before relying on an alias. + # + # Runs as a pinned third-party container, so `kind: docker` with an explicit + # `command` — proc.rs only builds argv for `kind: vllm`. Foreground + # `docker run --rm` so the supervisor gets a real PID to track and the + # container is removed when it stops; upstream's compose.yaml uses + # `restart: unless-stopped` + `-d`, which would fight the Governor. The + # server binds 8000 inside the image (hardcoded in serve-ds4-flash.sh), so + # it is published as 8001 to take `brain`'s port and keep the gateway's + # `--upstream 127.0.0.1:8001` working unchanged, exactly like brain-fable. + # + # DISK, first start only: downloads ~99.5 GiB of source weights to + # data/source, then coalesces the 172 rank-sliced `exl3-layer-*-tp4-rankN` + # files to TP1 (~83 GiB of NEW files; the 5 `carried-*.safetensors` are + # hard-linked by --link-carried, so those cost nothing) and builds a 3 GB + # K64 draft. Peak ~186 GiB against 207 GiB free. It fits, and nothing else + # lands on this box while it does. data/source is reusable-but-dead weight + # afterward; the entrypoint re-enters the download branch only when + # tp1/rank-sliced-tp1-manifest.json is missing. + footprint_gb: 101 + channel: experimental + health: "http://localhost:8001/v1/models" + health_contains: "deepseek-v4-flash" + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-ds4" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=16g" + - "-p" + - "8001:8000" + - "-v" + - "/srv/models/deepseek-v4-flash-spark/data:/models" + - "-v" + - "/srv/models/deepseek-v4-flash-spark/cache:/cache" + - "-e" + - "MODEL_REPO=0xSero/deepseek-v4-flash-0731-spark" + - "-e" + - "MODEL_REVISION=22f28d32b9b29b4352eaa380ff8c2c170b2847ab" + # 65536, not upstream's 262144. We do not need 262k, and it is the only + # lever that exists: it drops KV from the measured 8.01 GiB to ~2 GiB. + # The 95.39 GiB of weights is unaffected. + - "-e" + - "MAX_MODEL_LEN=65536" + - "-e" + - "MAX_NUM_SEQS=4" + - "-e" + - "MAX_NUM_BATCHED_TOKENS=8224" + - "-e" + - "MODE=dspark" + # Fixed K5, dynamic depth OFF. This is upstream's default and it is + # load-bearing, not cosmetic: dynamic K1-K5 was faster on some prompts + # but tripped a grammar-mask cardinality assertion on strict JSON + # schema. That is the "xgrammar issue" being reported on X. Leave at 0. + - "-e" + - "DSPARK_TOKENS=5" + - "-e" + - "DSPARK_CAPACITY=0" + - "-e" + - "DSPARK_DYNAMIC_DRAFT_DEPTH=0" + - "-e" + - "DSPARK_DYNAMIC_DRAFT_DEPTH_WINDOW=8" + - "-e" + - "DSPARK_DRAFT_EXPERTS=64" + - "-e" + - "DSPARK_STRUCTURED_EXPERTS_PER_CATEGORY=32" + - "-e" + - "VLLM_USE_B12X_WO_PROJECTION=1" + # 0.83 of 121.6 GB = ~101 GB, matching footprint_gb above. Upstream ships + # 0.9465, which reserves about one hundred fifteen GB and would leave nothing for `ears`. + - "-e" + - "GPU_MEMORY_UTILIZATION=0.83" + - "-e" + - "VERIFY_MODEL_CHECKSUMS=1" + - "-e" + - "SERVED_MODEL_NAME=deepseek-v4-flash-0731-spark" + - "ghcr.io/0xsero/deepseek-v4-flash-0731-spark-sparkinfer@sha256:2e077489a83a0360952828051fe7f7a32c1801e5ce8436d85f7267583d614ff4" + + # Alternatives, ready to register when wanted: + # image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark + # image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering + + brain-nemotron-256k: + # The 256k sibling of `brain-nemotron`. A REPLACEMENT for it and for `brain`, + # never a peer — all three take :8001. + # + # Everything in the `brain-nemotron` entry above applies: the vllm27-env + # requirement, the flag set, why thinking is off, why marlin is pinned. Read + # that entry first; only the context window and its cost differ here. + # + # footprint: MEASURED on the reference node at exactly this configuration — + # max-model-len 262144, gpu-memory-utilization 0.38 -> 57.0 GB RSS + # + # 0.38 * 121.7 = 46.2 GB of pool against 57.0 GB observed, so **10.8 GB sits + # outside the fraction** — the mamba conv/SSM state is allocated per-sequence + # and gpu-memory-utilization does not bound it. At 64k the same gap is only + # 1.4 GB. That is why this is declared at 57 and not at 46, and why raising + # the window again means re-measuring rather than scaling the old number. + # + # This configuration is what OOM-killed `ears` on 2026-08-11, when it was + # declared at 46 GB from a paper estimate. 57 is the measurement that + # replaced the estimate. Do not lower it without booting and reading RSS. + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 256k)" + footprint_gb: 57 + channel: experimental + health: "http://localhost:8001/v1/models" + # ⚠️ Matches on the 256k-only alias, NOT the shared HF name. Compute decides + # "is it running" by probing the port and matching this string, so an entry + # that shares :8001 AND the generic name with `brain-nemotron` makes BOTH + # look alive at once — observed 2026-08-16 as committed 128 GB across five + # models and headroom 0. Same trick `brain-fable` uses, same reason. + health_contains: "brain-nemotron-256k" + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-256k" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 262144 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute would derive from + # footprint_gb (57/121.7 = 0.468, which would reserve 11 GB more pool + # than this configuration was measured with). + gpu-memory-utilization: 0.38 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + voice-vox: + # VoxCPM 2 as an alternative narrator to `voice` (Chatterbox). A REPLACEMENT, + # never a peer — they are both a TTS on this box and there is no reason to + # hold two. Chatterbox is the one to keep if latency matters: measured + # 2026-08-16 under a warm protocol, Chatterbox turbo is RTF 0.28 here and + # VoxCPM is 1.35, i.e. VoxCPM is SLOWER THAN REAL TIME on this box. Pick it + # for what it is better at — 48 kHz output, 30 languages, and it will read + # 1800 characters in one call where Chatterbox drops 98% of the text. + # + # footprint: MEASURED on the reference node. Unlike a vLLM model this one does NOT + # pre-allocate — the audio decoder allocates in proportion to the LENGTH OF + # THE AUDIO, so the footprint is a function of the longest call you allow: + # 450 chars -> 7.8 GB 3600 chars -> 17.7 GB + # 7200 chars -> 23.7 GB 14400 chars -> 49.2 GB + # The service pins VOX_MAX_CHARS=3600 and serialises renders behind a lock, + # so 17.7 GB is the real ceiling. 20 declared leaves slack for the allocator. + # ⚠️ Raising VOX_MAX_CHARS without raising this is how you OOM-kill `ears`. + # + # 3600 is also where quality stops, not just memory: at 7200 characters the + # model silently stopped reading (27.8 chars/sec against ~15 for real + # speech). The cap returns 413 rather than truncating in silence. + # + # Defaults are cfg_value 3.0 / inference_timesteps 20, chosen by listening on + # 2026-08-16. Higher guidance is also CHEAPER here — at cfg 2.0 the model + # produces bad cases and retries them (rtf 5.2), at cfg 3.0-4.0 with 20 steps + # it settles at rtf ~2.15 on the Radeon. Ear and clock agree for once. + name: "VoxCPM 2 (48 kHz, 30 languages, voice cloning)" + footprint_gb: 20 + channel: experimental + # A listening socket is not a narrator: the server warms one render before it + # flips model_loaded, because the first call after a load is ~3x slower. + health: "http://localhost:8096/health" + health_contains: '"model_loaded":true' + supervision: + restart: always + check_interval_sec: 30 + failure_threshold: 3 + backoff_sec: 60 + max_backoff_sec: 900 + # Weights are 4.7 GB and it warms a render before reporting healthy. + startup_timeout_sec: 300 + serve: + kind: python + port: 8096 + command: ["~/vox-env/bin/python", "~/vox_api.py"] + env: + VOX_PORT: "8096" + VOX_MAX_CHARS: "3600" + VOX_STEPS: "20" + VOX_CFG: "3.0" + VOX_VOICES: "/var/lib/lumbridge/vox-voices.json" + + brain-nemotron-128k: + # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. + # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the + # vllm27-env requirement and the flag rationale; only the window differs. + # + # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` + # (util 0.32). The declared 45 was interpolated from the 64k/256k points and + # was ~5 GB conservative; corrected to 41. The overhead over the pool is the + # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not + # bound and which grows with the window. + # + # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from + # RSS, which does NOT measure GPU memory on this unified-memory box — the + # brain reads 8 GB RSS against 40 GB actual. They are not comparable to + # this number and should be re-measured with nvidia-smi before being used. + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k)" + footprint_gb: 41 + channel: experimental + health: "http://localhost:8001/v1/models" + # 128k-only alias, so Compute can tell this apart from its 64k and 256k + # siblings, which share the port and the generic served names. + # ⚠️ The trailing quote is load-bearing. `health_contains` is a raw substring + # test against the /v1/models body (governor.rs: response.contains(marker)), + # and "brain-nemotron-128k" is a PREFIX of "brain-nemotron-128k-seqs16". + # Without the closing quote this entry matched the seqs16 server, the Governor + # reported a phantom sixth model as [ext], committed 131 GB against a 100 GB + # budget, and refused `scene activate stock` with "already serving but is not + # identity-owned by Compute". Matching the JSON string terminator makes the + # two ids distinguishable. Found 2026-08-19. + health_contains: 'brain-nemotron-128k"' + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-128k" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 131072 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute derives from footprint. + gpu-memory-utilization: 0.32 + max-num-seqs: 4 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-nemotron-128k-seqs16: + name: "NVIDIA Nemotron 3.5 Lightning 30B-A3B (NVFP4 + DSpark, 128k, seqs 16)" + # EXPERIMENT, 2026-08-19. Identical to brain-nemotron-128k in every respect + # except max-num-seqs: 16 instead of 4. It exists to answer whether the + # "ceiling at four" measured 2026-08-16 is this box or is this flag. That + # sweep ran 8 client lanes against a cap of 4, so its 8-lane level was + # queueing rather than concurrency, and the finding is confounded by its + # own config. Peak occupancy never exceeded 4 in its own recording. + # + # The cap is NOT forced by memory. This model logged 3,503,197 tokens of KV + # cache at util 0.32 and reported "Maximum concurrency for 131,072 tokens + # per request: 26.73x". The 4 was inherited from `brain`, where it existed + # to stop Qwen building 3.2M tokens of KV on a single-user box; that reason + # does not transfer to a hybrid mamba model whose window is cheap. + # + # ⚠️ footprint_gb below is INHERITED FROM THE 4-SEQ ENTRY, NOT MEASURED. + # Raising the cap adds per-sequence mamba conv/SSM state, allocated OUTSIDE + # the fraction gpu-memory-utilization bounds. Re-measure with + # `nvidia-smi --query-compute-apps` and correct this line. Under-declaring + # this exact model is what OOM-killed `ears` on 2026-08-11. + # The middle rung between `brain-nemotron` (64k) and `brain-nemotron-256k`. + # A REPLACEMENT for both — all three take :8001. Read the 64k entry for the + # vllm27-env requirement and the flag rationale; only the window differs. + # + # MEASURED 2026-08-16 at 128k: 40.1 GB via `nvidia-smi --query-compute-apps` + # (util 0.32). The declared 45 was interpolated from the 64k/256k points and + # was ~5 GB conservative; corrected to 41. The overhead over the pool is the + # per-sequence mamba conv/SSM state, which gpu-memory-utilization does not + # bound and which grows with the window. + # + # ⚠️ The 64k/256k figures recorded elsewhere (35.5 / 57.0) were read from + # RSS, which does NOT measure GPU memory on this unified-memory box — the + # brain reads 8 GB RSS against 40 GB actual. They are not comparable to + # this number and should be re-measured with nvidia-smi before being used. + footprint_gb: 41 + channel: experimental + health: "http://localhost:8001/v1/models" + # 128k-only alias, so Compute can tell this apart from its 64k and 256k + # siblings, which share the port and the generic served names. + health_contains: 'brain-nemotron-128k-seqs16"' + serve: + kind: vllm + executable: "~/vllm27-env/bin/vllm" + port: 8001 + weights: "~/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4" + served_name: + - brain + - "local-moe" + # The bare HF name is deliberately NOT advertised here: it is what + # `brain-nemotron` (64k) matches health on, and sharing it makes that + # entry look alive whenever this one runs. + - "brain-nemotron-128k-seqs16" + args: + default-chat-template-kwargs: '{"enable_thinking": false}' + max-model-len: 131072 + kv-cache-dtype: fp8 + # Explicit, so it wins over the fraction Compute derives from footprint. + gpu-memory-utilization: 0.32 + max-num-seqs: 16 + max-num-batched-tokens: 4096 + enforce-eager: true + enable-prefix-caching: true + speculative-config: '{"model":"/srv/models/Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark","num_speculative_tokens":3}' + mamba-backend: flashinfer + mamba-cache-mode: align + moe-backend: marlin + reasoning-parser: nemotron_v3 + tool-call-parser: qwen3_coder + enable-auto-tool-choice: true + env: + CUTE_DSL_ARCH: sm_121a + PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin" + + brain-qwen38: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-mia: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, MiaAI-Lab tuning, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-mia"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-mia" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-mia" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-dspark: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DSpark, MiaAI-Lab tuning, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dspark"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dspark" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dspark" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the + # published GB10 config from hasso5703/dgx-spark-qwen38) except + # --mem-fraction-static, which the Governor budget owns. + - "--speculative-algorithm" + - "DSPARK" + - "--speculative-draft-model-path" + - "/draft-dspark" + - "--speculative-dspark-block-size" + - "7" + - "--speculative-draft-model-quantization" + - "unquant" + - "--speculative-num-draft-tokens" + - "8" + + # Decode-loop options we were missing on EVERY previous profile: + # torch.compile was off entirely, and continuous-decode-steps was 1. + - "--enable-torch-compile" + - "--torch-compile-max-bs" + - "4" + - "--cuda-graph-max-bs-decode" + - "4" + - "--num-continuous-decode-steps" + - "2" + + brain-qwen38-dspark-solo: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DSpark, 0.90 pool, BENCHMARK ONLY)" + footprint_gb: 110 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dspark-solo"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dspark-solo" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + # --network host removes the docker-proxy userland hop; SGLang then binds + # 8001 directly (see --port below). + - "--network=host" + # GB10 big.LITTLE: X5 performance cores are 5-9,15-19; A725 efficiency + # cores are 0-4,10-14. Without this the scheduler/tokenizer Python runs + # on the 2.8GHz little cores. + - "--cpuset-cpus=5-9,15-19" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DSpark:/draft-dspark:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dspark-solo" + - "--host" + - "0.0.0.0" + - "--port" + - "8001" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # Mia's value. ~110 GB of 121.7 — the brain owns the whole box. + - "--mem-fraction-static" + - "0.90" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.21" + - "--mamba-ssm-dtype" + - "bfloat16" + - "--mamba-radix-cache-strategy" + - "extra_buffer_lazy" + - "--max-mamba-cache-size" + - "40" + - "--sampling-defaults" + - "model" + - "--max-running-requests" + - "10" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DSpark stack, verbatim from MiaAI-Lab start-dspark.sh (itself the + # published GB10 config from hasso5703/dgx-spark-qwen38) except + # --mem-fraction-static, which the Governor budget owns. + - "--speculative-algorithm" + - "DSPARK" + - "--speculative-draft-model-path" + - "/draft-dspark" + - "--speculative-dspark-block-size" + - "7" + - "--speculative-draft-model-quantization" + - "unquant" + - "--speculative-num-draft-tokens" + - "8" + # Decode-loop options we were missing on EVERY previous profile: + # torch.compile was off entirely, and continuous-decode-steps was 1. + - "--enable-torch-compile" + - "--torch-compile-max-bs" + - "4" + - "--cuda-graph-max-bs-decode" + - "4" + - "--num-continuous-decode-steps" + - "2" + + brain-qwen38-mem: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4 + MTP, SGLang EAGLE, big pool, VL)" + footprint_gb: 75 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-mem"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-mem" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + - "lmsysorg/sglang@sha256:3c0abdf41ef22de9d7a859dc16ed71eae69452e36c91f071a25e60c85a6d1fc6" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-mem" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # 0.62 to match brain-qwen38-dflash-mem exactly — this entry exists only + # to tell "DFlash2 is better" apart from "a bigger KV pool is better". + - "--mem-fraction-static" + - "0.62" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + - "--speculative-algorithm" + - "EAGLE" + - "--speculative-num-steps" + - "3" + - "--speculative-eagle-topk" + - "1" + - "--speculative-num-draft-tokens" + - "4" + + brain-qwen38-dflash: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # The runtime profile is pinned to the measured 0.46 pool and a C4 ceiling. + # Telemetry must show sustained queueing or a throughput win before either + # value changes again. + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, VL)" + footprint_gb: 56 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dflash"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dflash" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but + # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker + # with "Cannot find model module". This image is that same digest plus the + # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark + # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 + # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). + # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull + # and to keep the A/B differing only in the DFlash2 code. + # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh + - "lmsysorg/sglang:qwen38-27b-dflash2-local" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dflash" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + - "--mem-fraction-static" + - "0.46" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + # C4 is the measured useful ceiling for the solo Spark workload. Compute + # records both running and queued requests before this is tuned again. + - "--max-running-requests" + - "4" + # Runtime-native metrics are the source of truth for tokens, queueing, + # latency, cache behaviour, and multimodal encoder use. Labels are + # deliberately bounded; job ids never become Prometheus labels. + - "--enable-metrics" + - "--enable-cache-report" + - "--tokenizer-metrics-allowed-custom-labels" + - "client" + - "agent" + - "workload" + - "--uvicorn-access-log-exclude-prefixes" + - "/metrics" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DFlash2 instead of EAGLE. The stock image advertises DFLASH but lacks + # DFlash2DraftModel, so the pinned local image carries the verified + # upstream overlay and NVFP4 lm_head fix described above. + # + # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: + # probe EAGLE/MTP DSpark DFlash2 + # code 34.5 51.5 50.9 + # long essay 24.1 18.3 25.4 + # short chat 21.0 23.2 66.6 + # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats + # MTP on essays, and is 3x on short chat. The downstream agents do short + # tool-calling turns, so short chat is the regime that matters here. + # + # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: the + # live C1-C4 A/B showed no useful gain, while 0.46 leaves roughly 55 GB + # available for page cache and safe transient work. + - "--speculative-algorithm" + - "DFLASH" + - "--speculative-draft-model-path" + - "/draft" + - "--mamba-radix-cache-strategy" + - "extra_buffer" + # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). + - "--speculative-num-draft-tokens" + - "8" + + brain-qwen38-dflash-mem: + # Qwen3.8-27B (released 2026-08-14, Apache-2.0) served by SGLang in Docker. + # A REPLACEMENT for `brain`, `brain-nemotron*` and `brain-ds4` — all take :8001. + # + # WHY SGLANG AND NOT vLLM. Both runtimes have day-0 support for this family + # (vLLM 0.27.1 in ~/vllm27-env already registers Qwen3_5ForConditionalGeneration + # and Qwen3_5MTP; SGLang ships an official cookbook recipe with a DGX Spark + # cell). The decision is throughput shape, from r0b0tlab_s matched head-to-head + # on THIS box class, same checkpoint, same harness: + # + # tok/s dedicated c1 c1(1024->256) c2 c4 c8 + # vLLM MTP K3 27.83 19.24 32.00 34.61 82.89 + # SGLang EAGLE 25.62 27.65 43.74 74.31 123.90 + # + # vLLM wins ONLY the one-long-2048-token-generation case (+8%). SGLang wins + # short agent turns by +44% and every concurrency level by thirty-seven to one hundred fifteen percent. Agents do + # short turns, so SGLang. The cost is think-on dedicated c1 (23.74 vs vLLM + # 29.12) — if a workload is long thinking-on monologues, use vLLM instead. + # + # ⚠️ VISION IS WHY THE `eye` IS GONE. This checkpoint is a native VLM + # (config.json carries a live vision_config; language_model_only: false) and + # SGLang serves the vision tower in-process. The `qwen38` scene therefore + # drops Cosmos 3 Edge and reclaims its 14 GB. vLLM_s VL path on this + # checkpoint is NOT verified by anyone — that is a second reason for SGLang. + # + # ⚠️ SHARDS ARE CHECKSUM-GATED. The HF repo had a publication bug that omitted + # the hardlinked body shards; a tree with only shard 4 loads and produces + # garbage. Verified all 4 OK against final-sota-shards.sha256 on 2026-08-17. + # + # ⚠️ CANARY: `19 x 23` must return 437. A 417 means FP8 KV went missing — + # this checkpoint declares kv_cache_quant_algo FP8 and `--kv-cache-dtype auto` + # honours it. Do not "fix" auto to something else without re-running the canary. + # + # ⚠️ NO PREFIX CACHING and no `` special tokens in the vLLM path. On + # SGLang the chat template does emit them, so `--reasoning-parser qwen3` is + # correct HERE and wrong on vLLM. The two runtimes genuinely differ. + # + # MEMORY. `--mem-fraction-static` is a fraction of TOTAL memory (121.7 GB), + # not of what is free, and SGLang reserves it as a STATIC pool at boot. So + # this model must start FIRST (the scene uses `order: listed`), exactly like + # `brain` and for the same reason. 0.46 x 121.7 = ~56 GB; declared 56. + # Scene math: 56 + 10 + 20 + 5 = 91, +8 GB Governor safety margin = 99 of 100. + # At 62 the margin pushed the total to 105 and the Governor refused the + # admit. The margin is NOT optional — budget for it up front. + # + # ⚠️ A too-small pool fails at BOOT, not under load, and the error names the + # wrong first remedy. At 0.28 it died with + # "Not enough GPU memory for hybrid (mamba/linear-attention) state cache. + # Computed max_mamba_cache_size=-59 (total_rest_memory=-11.34 GB)" + # i.e. weights (21.5 GB) plus the mamba state pool did not fit. Raise + # --mem-fraction-static; do not start shaving --speculative-num-draft-tokens. + # + # Flags below are r0b0tlab_s `eagle` profile verbatim except two, both ours: + # --mem-fraction-static (their 0.70 = ~85 GB and blows the Governor budget) + # and --max-running-requests 8 (to actually sit in the c8 regime SGLang wins). + # VERIFIED LIVE 2026-08-17, first boot: + # - canary 19 x 23 -> 437 (FP8 KV correct; a 417 means it regressed) + # - VISION WORKS. A generated test image came back with both shapes and + # the exact string read off it. This is the empirical basis for dropping + # `eye` from the scene, not an assumption. + # - EAGLE acceptance measured 0.46-0.68 (accept len 2.4-3.0), i.e. BETTER + # than the 0.23-0.52 published for this checkpoint. + # - nvidia-smi showed 40.3 GB for sglang::scheduler at 0.47 KV-pool usage. + # Declared stays 56 (= what --mem-fraction-static 0.46 authorises it to + # take as the pool fills). Do NOT cut the declaration to the observed + # 40 — under-declaring a growing pool is how `ears` got OOM-killed. + # + # ✅ UNKNOWN MODEL NAMES FALL THROUGH. Unlike vLLM (and unlike `brain-ds4`, + # whose entry warns about exactly this), SGLang does NOT 404 a request whose + # `model` field it does not recognise — a request for `brain` was served + # normally. So the gateway and the downstream agents need no alias and no change. + # + # ⚠️ NO CLEAN SINGLE-STREAM NUMBER YET. Every measurement attempt on + # 2026-08-17 ran against 2-3 concurrent live requests from the agents, which + # find :8001 the moment it is healthy. Client-side single-stream read 18.6 + # tok/s think-off / 15.2 think-on, but server-side `gen throughput` was + # 45-71 tok/s aggregate at the same moment, so the 18.6 is a SHARE, not a + # ceiling. To get a real c1 number, quiesce the agents first. + name: "Qwen3.8-27B (NVFP4, SGLang DFlash2, big pool, VL)" + footprint_gb: 75 + channel: experimental + health: "http://localhost:8001/v1/models" + # Anchored on the closing quote: /v1/models returns {"id":""}, and a bare + # "brain-qwen38" is a PREFIX of "brain-qwen38-dflash", so the unanchored form + # made this entry report healthy whenever the dflash variant was serving :8001. + # The Governor then double-counted both as running (147 GB committed) and blocked + # scene switches with "is serving but is not identity-owned by Compute". + health_contains: 'brain-qwen38-dflash-mem"' + serve: + kind: docker + port: 8001 + command: + - "docker" + - "run" + - "--rm" + - "--init" + - "--name=brain-qwen38-dflash-mem" + - "--gpus=all" + - "--ipc=host" + - "--shm-size=32g" + - "-p" + - "8001:30000" + - "-v" + - "/srv/models/Qwen3.8-27B-NVFP4-MTP-sm121:/model:ro" + - "-v" + - "/srv/models/Qwen3.8-27B-DFlash2:/draft:ro" + # Pinned by DIGEST, not tag. `lmsysorg/sglang:latest` is not this build + # and the cookbook numbers do not transfer to it. + # Locally-built DFlash2 image. The stock pinned digest ADVERTISES DFLASH but + # has no DFlash2DraftModel registered, so it dies at build_draft_tp_worker + # with "Cannot find model module". This image is that same digest plus the + # 5 sha256-verified files from MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark + # patch/overlay-dflash2 (which is upstream sglang c14312a66 + the NVFP4 + # quantized-lm_head fix, needed because our checkpoint quantizes lm_head). + # Built FROM our digest, not Mia's :qwen38-27b tag, to avoid a 38 GB pull + # and to keep the A/B differing only in the DFlash2 code. + # Rebuild: see /var/lib/lumbridge/qwen38-sglang-recipe/patch/build-dflash2-image.sh + - "lmsysorg/sglang:qwen38-27b-dflash2-local" + - "sglang" + - "serve" + - "--trust-remote-code" + - "--model-path" + - "/model" + - "--served-model-name" + - "brain-qwen38-dflash-mem" + - "--host" + - "0.0.0.0" + - "--port" + - "30000" + # trtllm_mha is SM100-only; flashinfer is the GB10 path. + - "--attention-backend" + - "flashinfer" + - "--kv-cache-dtype" + - "auto" + - "--chunked-prefill-size" + - "8192" + - "--max-prefill-tokens" + - "8192" + - "--context-length" + - "262144" + # 0.62 (~75 GB) instead of 0.46. See this entry's docstring: the point is + # to separate "DFlash2 is not better" from "DFlash2 was starved". Mia's + # profile wants 0.90; the Governor budget will not allow that with ears + # and voice resident. + - "--mem-fraction-static" + - "0.62" + - "--disable-prefill-cuda-graph" + - "--reasoning-parser" + - "qwen3" + # THINKING OFF BY DEFAULT - added 2026-08-21 after it broke the assistant stack. + # Qwen3.8 thinks by default. V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) + # sends max_tokens: 80; the reasoning block consumed all 80, finish_reason came + # back "length" and content was EMPTY - so news/squawk narration silently + # produced nothing at all. Every brain-nemotron* entry in this registry already + # carries the equivalent default-chat-template-kwargs; this one was missing it. + # Per-request chat_template_kwargs still takes precedence, so an agent that + # wants thinking simply asks for it. + # CANARY: a max_tokens:80 request must return NON-EMPTY content. + - "--default-chat-template-kwargs" + - '{"enable_thinking": false}' + - "--tool-call-parser" + - "qwen3_coder" + - "--mamba-full-memory-ratio" + - "4.59" + - "--max-running-requests" + - "8" + # EAGLE 3/1/4 drives the checkpoint_s OWN in-box MTP head. No second + # download, unlike DSpark (which needs RadixArk/Qwen3.8-27B-DSpark and + # measured SLOWER here: 20.97 dedicated c1 vs EAGLE_s 25.62). + # DFlash2 instead of EAGLE. Our pinned image already lists DFLASH as a + # builtin (`sglang serve --help`: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, + # DFLASH, DSPARK), so unlike MiaAI-Lab's recipe we need NO derived image — + # only the drafter (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB, at /draft). + # + # WHY DFLASH AND NOT DSPARK. MiaAI-Lab measured on this box class: + # probe EAGLE/MTP DSpark DFlash2 + # code 34.5 51.5 50.9 + # long essay 24.1 18.3 25.4 + # short chat 21.0 23.2 66.6 + # DSpark wins coding but LOSES on essays; DFlash2 ties it on code, beats + # MTP on essays, and is 3x on short chat. The downstream agents do short + # tool-calling turns, so short chat is the regime that matters here. + # + # NOTE we deliberately do NOT take Mia's --mem-fraction-static 0.90: that + # is ~110 GB and assumes the whole box. Lumbridge runs ears+voice+voice-vox + # alongside, so we keep 0.46 and let the Governor hold the budget. + - "--speculative-algorithm" + - "DFLASH" + - "--speculative-draft-model-path" + - "/draft" + - "--mamba-radix-cache-strategy" + - "extra_buffer" + # 8 per MiaAI-Lab's DFlash2 profile (EAGLE here used 4). + - "--speculative-num-draft-tokens" + - "8" diff --git a/scenes/darkroom.scene.yaml b/scenes/darkroom.scene.yaml new file mode 100644 index 0000000..f06d1c4 --- /dev/null +++ b/scenes/darkroom.scene.yaml @@ -0,0 +1,27 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: darkroom + version: 1 + description: "Overnight image farm — drops the brain to make room for FLUX.2-klein-9B." + author: karti + tags: [image, overnight, unattended] + +# Drops `brain` so `image` (66 GB, on-hardware measured) fits. ears+voice+music+ +# image commits ~99 GB — tight against the 8 GB admission margin, so this scene +# needs the full 108 GB ceiling (matching studio/voice-laguna), not the old 100. +models: + - ears + - voice + - music + - image + +budget_gb: 108 +activation: + order: footprint-asc + wait_healthy: true + +# Scheduled scenes (planned): hand the box to darkroom overnight, back at 08:00. +# schedule: +# - activate: "03:00" +# - handoff: "04:00" -> studio diff --git a/scenes/deepseek.scene.yaml b/scenes/deepseek.scene.yaml new file mode 100644 index 0000000..af2fa64 --- /dev/null +++ b/scenes/deepseek.scene.yaml @@ -0,0 +1,45 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: deepseek + version: 1 + description: "DeepSeek V4 Flash 0731 (3-bit EXL3) plus ears — a whole-box brain swap; voice, embed and music do not fit beside it." + author: karti + tags: [assistant, brain, experimental, whole-box, deepseek] + +# PARKED 2026-08-06 — registered but never activated on the reference node. The intent is to +# run this on the SECOND Spark full-time, where it gets the whole box, which is the +# only shape that suits it. On the reference node it is strictly a downgrade: measured the same +# day, `brain` does 62.5-64.4 tok/s against this model's published 34.30 min / +# 38.12 median, and admitting it costs voice, embed and music. Do not activate here +# without a deliberate reason; see the disk note in the registry entry first. +# +# This is a darkroom-shaped Scene: it is defined by what it displaces. `brain-ds4` +# loads 95.39 GiB of EXL3 weights and is declared at 101 GB, so on a 121.6 GB box +# it is the tenant and everything else is a lodger. +# +# Why only `ears` — this is admission control arithmetic, not taste. SAFETY_MARGIN_GB +# is 8.0 and admission requires `committed + add + 8 <= budget`: +# +# brain-ds4 0 + 101 + 8 = 109 <= 118 admitted (available after the stop pass about one hundred fifteen) +# ears 101 + 5 + 8 = 114 <= 118 admitted (available ~20, needs 13) +# voice 106 + 8 + 8 = 122 > 118 REFUSED (available ~15, needs 16) +# +# So voice cannot be talked into this Scene by raising budget_gb either — the +# physical check fails a hair before the declared one. embed (2) would squeak in +# ahead of voice; it is left out deliberately so the /voice stack fails loudly at +# activation rather than half-working. music (~24) is not close. +# +# ORDER MATTERS, same reason as the primary Scene but more so: brain-ds4 must come up +# first and alone. Its first start also downloads ~99.5 GiB and coalesces the +# rank-sliced EXL3 archive to TP1, so budget on the order of an hour before the +# health check goes green, and `wait_healthy` will hold ears behind it the whole +# time. Subsequent starts skip straight to the weight load. +models: + - brain-ds4 + - ears + +budget_gb: 118 +activation: + order: listed + wait_healthy: true diff --git a/scenes/music.scene.yaml b/scenes/music.scene.yaml new file mode 100644 index 0000000..95318b4 --- /dev/null +++ b/scenes/music.scene.yaml @@ -0,0 +1,35 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: music + version: 2 + description: "Music studio — MiniMax Music 3, with ears, voice, and the Lightning brain." + author: karti + tags: [music, creative, assistant] + +# v2, 2026-08-13: ACE-Step XL 4B is gone; MiniMax Music 3 takes the `music` +# slot on :8010. Same port, same request fields, so :8010 callers are unchanged. +# +# brain-nemotron joins the scene, which is what forces `order: listed` below. +# The old v1 used footprint-asc, and that is actively unsafe with a vLLM MoE in +# the set: it would start the 38GB brain LAST, into a box already holding +# ears+voice+music. The brain-nemotron entry is explicit that whatever holds +# :8001 comes up first and alone, because its KV-profiling spike is not bounded +# by gpu-memory-utilization. Declaring that footprint from an estimate once +# already ran 11GB over and let the kernel OOM-kill `ears` on the next +# transition. So: brain first, on the empty machine, then the small stuff. +# +# ears 5 + voice 8 + music ~24 + brain-nemotron 38 = ~75 GB of 121. +# NOTE: the music figure is an ESTIMATE pending a boot-and-read-RSS measurement, +# the same discipline brain-nemotron's comment demands. Do not trust it until +# registry/models.yaml carries a measured number. +models: + - brain-nemotron + - ears + - voice + - music + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/nemotron-seqs16.scene.yaml b/scenes/nemotron-seqs16.scene.yaml new file mode 100644 index 0000000..ac7d7fe --- /dev/null +++ b/scenes/nemotron-seqs16.scene.yaml @@ -0,0 +1,29 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: nemotron-seqs16 + version: 1 + description: "stock, with the brain's max-num-seqs raised from 4 to 16. Measurement only." + author: karti + tags: [brain, experiment, publish] + +# Created 2026-08-19 for /publish episode 5. This is `stock` with exactly one +# substitution: brain-nemotron-128k -> brain-nemotron-128k-seqs16. Same weights, +# same 128k window, same util 0.32, same speculative config, same everything +# else in the scene. See the registry entry for why the cap is suspect. +# +# ⚠️ THIS IS NOT A DEFAULT. `stock` is the default scene. Reactivate it with +# lumbridge-compute scene activate stock +# when the sweep is done. Leaving this scene active leaves an experimental, +# un-remeasured footprint carrying the downstream agents. +models: + - brain-nemotron-128k-seqs16 + - ears + - voice-vox + - eye + - voice + +budget_gb: 100 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/nemotron.scene.yaml b/scenes/nemotron.scene.yaml new file mode 100644 index 0000000..09ece2d --- /dev/null +++ b/scenes/nemotron.scene.yaml @@ -0,0 +1,36 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: nemotron + version: 1 + description: "The assistant stack with NVIDIA Nemotron 3.5 Lightning 30B-A3B in place of the Qwen MoE brain — the A/B candidate for replacing `brain`." + author: karti + tags: [assistant, voice, "downstream", candidate, nemotron] + +# Deliberately the assistant stack with ONE substitution. `brain-nemotron` takes :8001 +# in place of `brain`, so the two are mutually exclusive by construction — and +# that is also what the hardware wants: 68GB is already spoken for with brain +# up, and two MoE engines would each spike during KV profiling. +# +# Keeping ears/voice/embed/ocr identical is the point. The only variable +# between the primary scene and this scene is which brain answers :8001, so an eval run +# against one is comparable to an eval run against the other. +# +# ORDER MATTERS, same rule as the primary scene: whatever holds :8001 comes up +# first and alone. brain's KV-profiling spike is not bounded by +# gpu-memory-utilization and this engine is unproven on the box, so it gets +# the empty machine. +models: + - brain-nemotron + - ears + - voice + - embed + - ocr + # The eye, added 2026-08-12. Cosmos3-Edge is BF16-only and cannot be + # quantized, so it costs a real 12GB that the other slots do not. + - eye + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash-mem.scene.yaml b/scenes/qwen38-dflash-mem.scene.yaml new file mode 100644 index 0000000..80052c5 --- /dev/null +++ b/scenes/qwen38-dflash-mem.scene.yaml @@ -0,0 +1,25 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash-mem + version: 1 + description: "MEASUREMENT ONLY — DFlash2 with a 75 GB static pool, voice-vox dropped to pay for it." + author: karti + tags: [experimental, benchmark, dflash] + +# NOT A PRODUCTION SCENE. It exists to answer one question: does DFlash2 beat +# EAGLE when it is given something closer to the memory MiaAI-Lab's profile +# assumes (0.90 / ~110 GB)? At 0.46 it did not (22.4 vs EAGLE 22.8 tok/s, +# acceptance 0.17-0.25 vs 0.46-0.68). +# +# voice-vox (20 GB) is dropped purely to afford the bigger pool. Do not leave +# this scene active — go back to `qwen38` when the measurement is done. +models: + - brain-qwen38-dflash-mem + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash-solo.scene.yaml b/scenes/qwen38-dflash-solo.scene.yaml new file mode 100644 index 0000000..cf0350d --- /dev/null +++ b/scenes/qwen38-dflash-solo.scene.yaml @@ -0,0 +1,30 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash-solo + version: 2 + description: "Production: Qwen3.8-27B vision only, with DFlash2 and durable usage telemetry." + author: karti + tags: [brain, vision, sglang, dflash, llm-only, production] + +# Spark is the dedicated 27B inference box. ASR and TTS run on metal, and +# separate small-model canaries are intentionally absent from this Scene. +# +# Keep the measured 0.46 static pool. A 0.90 pool did not improve C1-C4 +# throughput and left too little page cache. Decoder choice, not unused KV +# capacity, is the speed lever for this workload. +# +# Live A/B on this Spark, 2026-08-31, identical 384-token long-generation probe: +# aggregate tok/s C1 C2 C3 C4 +# DSpark 0.90 28.39 50.46 69.59 88.21 +# DFlash2 0.46 31.54 57.09 81.43 85.96 (median of 3) +# DFlash2 wins the common C1-C3 regime by 11-17%. C4 adds only 5.6% over C3, +# so the future job scheduler should normally retain one latency-reserve lane +# and borrow C4 only while queue and TTFT remain healthy. +models: + - brain-qwen38-dflash + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dflash.scene.yaml b/scenes/qwen38-dflash.scene.yaml new file mode 100644 index 0000000..c150ff2 --- /dev/null +++ b/scenes/qwen38-dflash.scene.yaml @@ -0,0 +1,76 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dflash + version: 1 + description: "qwen38 with the brain on DFlash2 speculative decoding instead of EAGLE." + author: karti + tags: [brain, vision, voice, long-context, sglang, dflash, experimental] + +# ⛔ BLOCKED 2026-08-21 — DO NOT ACTIVATE ON THE CURRENT IMAGE. +# +# Our pinned image (lmsysorg/sglang@sha256:3c0abdf4…) ADVERTISES DFLASH as a +# builtin speculative algorithm — `sglang serve --help` lists +# "Builtins: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, DFLASH, DSPARK" +# and accepts every flag (--speculative-draft-model-path, +# --mamba-radix-cache-strategy). It boots, loads all 4 shards, then dies at the +# draft-worker step with: +# +# File ".../speculative/dflash_worker_v2.py", line 195, in __init__ +# File ".../speculative/draft_worker_common.py", line 83, in build_draft_tp_worker +# ValueError: Cannot find model module. 'DFlash2DraftModel' is not a +# registered model in the Transformers library ... and 'AutoModel' is not +# present in the model config's 'auto_map' +# +# i.e. the ALGORITHM is builtin but the DRAFT MODEL ARCHITECTURE is not +# registered. That is precisely why MiaAI-Lab's recipe builds a DERIVED image +# from its `patch/` directory for DFlash2 while DSpark runs on the stock image. +# The flag surface being present is NOT evidence the path works — it boots for +# ~3 minutes before failing, and with `--rm` the logs vanish with the container. +# +# TO UNBLOCK: build MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark's patched image +# (./start-dflash.sh builds it), then repin the digest in the registry entry. +# +# WORTH IT? MiaAI-Lab's measured table for this box class: +# probe EAGLE/MTP DSpark DFlash2 +# code 34.5 51.5 50.9 +# long essay 24.1 18.3 25.4 +# short chat 21.0 23.2 66.6 +# Our EAGLE baseline measured HERE on 2026-08-21: decode 22.8 tok/s mean +# (TTFT 223ms), tool-calling 15/15, with live squawk traffic on the box — +# consistent with the 21.0 short-chat column. DSpark is NOT worth the swap for +# us (23.2 vs 22.8 short chat; its win is coding, which is Claude's job here, +# not spark's). DFlash2's 66.6 short-chat number is the only prize worth the +# image build — and it is the number most in need of independent confirmation, +# since MiaAI-Lab itself footnotes it as "once token-counted correctly". +# +# The drafter is already downloaded: /srv/models/Qwen3.8-27B-DFlash2 +# (z-lab/Qwen3.8-27B-DFlash2@50307d4, 3.6 GB). +# +# A/B TWIN OF `qwen38`. Identical model set and identical brain flags except the +# speculative algorithm (EAGLE -> DFLASH) and the drafter mount, so a benchmark +# delta is attributable to the algorithm alone. +# +# Same ordering rule as `qwen38`: brain FIRST. --mem-fraction-static is a +# fraction of TOTAL memory and is reserved as a static pool at boot. +# +# Same activation race too: do not `docker stop` then immediately activate — +# wait for `free -g` to show the memory back first. +# +# Compare with: +# lumbridge-compute eval run performance --model brain-qwen38-dflash +# python3 /tmp/toolbench.py http://127.0.0.1:8001/v1 brain-qwen38-dflash 3 +# Baseline to beat (EAGLE, measured 2026-08-21, with live squawk traffic on the +# box): decode 22.8 tok/s mean, TTFT 223ms, tool-calling 15/15. +# +# Revert with `lumbridge-compute scene activate qwen38`. +models: + - brain-qwen38-dflash + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dspark-solo.scene.yaml b/scenes/qwen38-dspark-solo.scene.yaml new file mode 100644 index 0000000..44d911f --- /dev/null +++ b/scenes/qwen38-dspark-solo.scene.yaml @@ -0,0 +1,25 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dspark-solo + version: 1 + description: "BENCHMARK ONLY — DSpark at Mia's 0.90 pool, brain alone, nothing else resident." + author: karti + tags: [benchmark, dspark, solo] + +# ⚠️ NOT A PRODUCTION SCENE. No ears, no voice, no voice-vox — the desk loses +# ASR and both narrators while this is active. Answers one question: does this +# box reach MiaAI-Lab's 51.5 tok/s code figure when the brain owns all of it? +# +# budget_gb is raised to 118 deliberately: a 0.90 pool is ~110 GB and cannot be +# admitted under the standard 100 GB budget with the 8 GB safety margin. +# +# Return to production immediately after measuring: +# lumbridge-compute scene activate qwen38-mia +models: + - brain-qwen38-dspark-solo + +budget_gb: 118 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-dspark.scene.yaml b/scenes/qwen38-dspark.scene.yaml new file mode 100644 index 0000000..6ee8cdc --- /dev/null +++ b/scenes/qwen38-dspark.scene.yaml @@ -0,0 +1,34 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-dspark + version: 1 + description: "qwen38-mia tuning with the DSpark speculative stack (code/tool-call optimised)." + author: karti + tags: [brain, vision, voice, sglang, dspark, benchmark] + +# Same model set and same MiaAI-Lab tuning as `qwen38-mia`; only the +# speculative stack differs (EAGLE/MTP -> DSpark) plus torch.compile and +# continuous-decode-steps 2. +# +# Benchmark with MiaAI-Lab's own net-decode harness, NOT the prose `performance` +# suite: python3 /tmp/ndec_ours.py +# qwen38-mia (EAGLE) baseline: code 31.60 / prose 23.97 tok/s +# MiaAI-Lab reports for DSpark: code 51.5 / prose 18.3 +# +# DSpark is faster on code/math/tool-calls and SLOWER on free-form prose, so +# check both probes before adopting — the squawk/news path is prose. +# +# Same swap discipline as every other scene: kill stale `scene activate`, +# deactivate, wait for free -g >= 110 GB, activate, then activate again to +# persist desired. +models: + - brain-qwen38-dspark + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-mem.scene.yaml b/scenes/qwen38-mem.scene.yaml new file mode 100644 index 0000000..634476e --- /dev/null +++ b/scenes/qwen38-mem.scene.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-mem + version: 1 + description: "MEASUREMENT ONLY — EAGLE at the same 75 GB pool as qwen38-dflash-mem." + author: karti + tags: [experimental, benchmark, eagle] + +# NOT A PRODUCTION SCENE. Control for qwen38-dflash-mem: same model set, same +# 0.62 pool, same quiet box — only the speculative algorithm differs. Do not +# leave active; return to `qwen38`. +models: + - brain-qwen38-mem + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38-mia.scene.yaml b/scenes/qwen38-mia.scene.yaml new file mode 100644 index 0000000..b5f77e4 --- /dev/null +++ b/scenes/qwen38-mia.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38-mia + version: 1 + description: "qwen38 with MiaAI-Lab's full EAGLE tuning (X5 cpuset, host net, GDN pool)." + author: karti + tags: [brain, vision, voice, long-context, sglang, benchmark] + +# Same model set as `qwen38`; the brain adopts MiaAI-Lab's tuning. The point is +# to find out how much of our ~22 tok/s vs her reported 34.5 is launch config. +# Baseline to beat: EAGLE @ 0.46 decode 22.8 tok/s, toolbench 15/15 @ 1.5 s. +models: + - brain-qwen38-mia + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/qwen38.scene.yaml b/scenes/qwen38.scene.yaml new file mode 100644 index 0000000..82b8374 --- /dev/null +++ b/scenes/qwen38.scene.yaml @@ -0,0 +1,82 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: qwen38 + version: 2 + description: "Qwen3.8-27B on SGLang as a seeing brain, plus ears and both narrators." + author: karti + tags: [brain, vision, voice, long-context, sglang] + +# Created 2026-08-17. A copy of `stock` with two changes: +# +# 1. brain-nemotron-128k (Lightning 30B-A3B, vLLM) -> brain-qwen38 +# (Qwen3.8-27B NVFP4+MTP, SGLang EAGLE, in Docker) +# 2. `eye` (Cosmos 3 Edge, 14 GB) is DROPPED — the new brain is a native VLM +# and SGLang serves its vision tower in-process, so the separate eye is +# redundant. This also kills the enable_thinking:false hack the eye needed +# because vLLM 0.27.1 has no cosmos3_edge reasoning parser. +# +# brain-qwen38 56 GB Qwen3.8-27B @ 262k, sees images and video :8001 +# voice-vox 20 GB VoxCPM 2, quality narrator, capped 3600 chars :8096 +# ears 10 GB streaming ASR, self-limiting at 6m39s of audio :8006 +# voice 5 GB Chatterbox TURBO, request-path narrator :8095 +# ----- +# 91 GB committed +8 GB Governor safety margin = 99 of 100. +# +# ⚠️ ORDER IS `listed`, NOT footprint-asc, AND THE BRAIN IS FIRST. SGLang_s +# --mem-fraction-static is a fraction of TOTAL memory and is reserved as a +# static pool at boot — if ears/voice/vox are already resident it computes its +# pool against a box that looks full and dies with a mamba-state-cache error. +# This is the same failure mode the primary scene documents for `brain`. +# +# ⚠️ THE SPEED TRADE vs `stock`. Lightning is a 3B-active MoE and decodes ~46 +# tok/s class on this box; Qwen3.8-27B is DENSE and decodes ~25.6 tok/s single +# stream. You are buying a large quality jump (Terminal-Bench 2.1 63.4 -> 73.0, +# OSWorld-Verified 63.9 -> 84.3 over Qwen3.6-27B) with roughly half the +# single-stream token rate. Under concurrency the gap closes and reverses: +# SGLang EAGLE measured 123.90 tok/s at c8. +# +# ⚠️ THINKING IS OFF BY DEFAULT (added v2, 2026-08-21). Qwen3.8 thinks by +# default and SGLang's auto-detect confirms it +# (reasoning_config=ReasoningToggleConfig(toggle_param='enable_thinking', +# default_enabled=True)). That silently broke the assistant stack on first activation: +# V9's normalizeSpeech (apps/server/src/lib/llm.ts:77) sends max_tokens: 80, the +# reasoning block consumed all 80, finish_reason came back "length" and content +# was EMPTY — so news and squawk narration produced nothing, with no error. +# The registry entry now passes --default-chat-template-kwargs +# '{"enable_thinking": false}', matching what every brain-nemotron* entry +# already did. Per-request chat_template_kwargs still overrides, so an agent +# that wants thinking just asks for it. +# +# CANARIES after any change to this scene (all three must pass): +# 1. 19 x 23 -> 437 (417 means FP8 KV regressed) +# 2. max_tokens:80 request -> NON-EMPTY content (guards the above) +# 3. chat_template_kwargs {"enable_thinking":true} -> reasoning_content present +# +# ⚠️ ACTIVATION IS MEMORY-RACY. `docker stop brain-qwen38` followed immediately +# by `scene activate` fails: --mem-fraction-static is computed against TOTAL +# memory but the stopped container's pages are not released yet, so the static +# pool lands short and the container dies at boot with no useful error. The +# activate process then hangs on "waiting for exact health" holding the +# transition lock, and the next activate returns "another Scene transition is +# already running". Wait for `free -g` to show the memory back (~90 GB avail) +# before re-activating. Boot then takes ~4 min (FlashInfer autotune + CUDA graph). +# +# VERIFIED 2026-08-21 on this scene: all three canaries pass; vision reads a +# generated test image exactly (shapes, colours and the literal string); +# tool calling picks the right tool with a correct command under a 21-tool +# payload; the larger local model answered a synthetic container-count prompt correctly with +# the command cited — the same question the previous Nemotron brain got wrong +# five times running. +# +# Restore the previous default with `lumbridge-compute scene activate stock`. +models: + - brain-qwen38 + - ears + - voice-vox + - voice + +budget_gb: 100 +activation: + order: listed # brain FIRST — it reserves a static pool, see above + wait_healthy: true diff --git a/scenes/stock.scene.yaml b/scenes/stock.scene.yaml new file mode 100644 index 0000000..a30e8ca --- /dev/null +++ b/scenes/stock.scene.yaml @@ -0,0 +1,54 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: stock + version: 1 + description: "The default scene: long-context brain, eyes, ears, and both narrators." + author: karti + tags: [brain, voice, long-context] + +# Created 2026-08-16. THIS IS THE DEFAULT SCENE as of 2026-08-16 — it is what +# the box should be running unless someone is deliberately doing something else. +# Five models, chosen for capability rather than latency: +# +# brain-nemotron-128k 41 GB Lightning 30B at 128k (measured 40.1) +# ears 10 GB streaming ASR, self-limiting at 6m39s of audio +# (measured 9.9) +# voice-vox 20 GB VoxCPM 2 at cfg 3.0 / 20 steps, capped at 3600 chars +# eye 14 GB Cosmos 3 Edge, still frames only (measured 11.8) +# voice 5 GB Chatterbox TURBO on :8095 (measured 3.2 GPU) +# ⚠️ but it LEAKS HOST RAM - see the registry +# ---- +# 90 GB committed against a 100 GB budget. +# +# All footprints above were measured 2026-08-16 with +# `nvidia-smi --query-compute-apps`, NOT RSS — RSS does not see GPU memory on +# this unified-memory box (the brain reads 8 GB RSS against 40 GB actual). +# +# TWO NARRATORS, ON PURPOSE. VoxCPM (:8096) is the quality narrator but is +# SLOWER THAN REAL TIME: measured RTF 1.54-1.91 warm, 3.07 cold, against the +# 1.62 recorded at build. Chatterbox turbo (:8095) is RTF ~0.26. So: +# - anything in a request path -> :8095 +# - anything offline where quality wins -> :8096 +# Chatterbox is also here because audiobook and downstream voice services +# all hardcode :8095; dropping it on 2026-08-16 took all three down until it +# was restored here the same day. VoxCPM is NOT a drop-in for them - different +# protocol, different voice registry. +# +# The Chatterbox FULL checkpoint is sunset as of 2026-08-16; `voice` is +# turbo-only and the /voice A/B Lab's `full` option now 400s. See the registry. +# +# 256k -> 128k to make room for the eye: at 57 GB the brain plus the eye needed +# 104 GB against a 100 GB budget and the Governor refused the admit. +models: + - brain-nemotron-128k + - ears + - voice-vox + - eye + - voice + + +budget_gb: 100 +activation: + order: footprint-asc # the small services up first; the 41 GB brain last + wait_healthy: true diff --git a/scenes/studio.scene.yaml b/scenes/studio.scene.yaml new file mode 100644 index 0000000..d5590d2 --- /dev/null +++ b/scenes/studio.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: studio + version: 2 + description: "Live voice assistant — ears, brain, mouth, and music." + author: karti + tags: [assistant, voice, always-on] + +# ears 5 + brain 66 + voice 4 + music (XL 4B sft) 24 = 99 GB declared. +# Empirically validated on the reference node: all four load with ~20 GB MemAvailable free. +# budget bumped to 108 (box is 121 GB) so the Governor admits the measured-safe set. +models: + - ears + - brain + - voice + - music + +budget_gb: 108 +activation: + order: footprint-asc # small first, so brain's big load spike lands last + wait_healthy: true diff --git a/scenes/tts-eval.scene.yaml b/scenes/tts-eval.scene.yaml new file mode 100644 index 0000000..5d989d9 --- /dev/null +++ b/scenes/tts-eval.scene.yaml @@ -0,0 +1,26 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: tts-eval + version: 1 + description: "Bare box for TTS model evaluation — ears only, for scoring WER." + author: karti + tags: [eval, tts] + +# 2026-08-15, TEMPORARY. Drops brain-nemotron (38), music (26) and voice (8) so +# the candidate TTS models (VoxCPM2 ~8GB, Fish s2-pro ~17GB) have the whole box. +# 'ears' stays because transcribing each model's output back to text is how we +# measure intelligibility objectively instead of only by listening. +# +# The candidates run OUTSIDE the registry during the eval, so the governor is +# not accounting for them — that is fine on an otherwise-empty box, but do not +# leave this scene active with unmanaged models running. +# +# TO RESTORE PRODUCTION: lumbridge-compute scene activate music +models: + - ears + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/scenes/voice-gemma.scene.yaml b/scenes/voice-gemma.scene.yaml new file mode 100644 index 0000000..d66303e --- /dev/null +++ b/scenes/voice-gemma.scene.yaml @@ -0,0 +1,22 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-gemma + version: 1 + description: "Efficient voice assistant — Nemotron ASR, Gemma 4 31B, and Chatterbox." + author: karti + tags: [assistant, voice, efficient, multimodal] + +# Gemma's mostly-sliding-window attention keeps its footprint well under brain +# and brain-laguna's, leaving real headroom in the budget beyond Lumbridge Compute's 8 GB +# admission margin — useful until brain-gemma's footprint is confirmed on +# real hardware. +models: + - ears + - brain-gemma + - voice + +budget_gb: 80 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/voice-laguna.scene.yaml b/scenes/voice-laguna.scene.yaml new file mode 100644 index 0000000..37a6f2b --- /dev/null +++ b/scenes/voice-laguna.scene.yaml @@ -0,0 +1,21 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-laguna + version: 1 + description: "High-capability voice assistant — Nemotron ASR, Laguna S 2.1, and Chatterbox." + author: karti + tags: [assistant, voice, coding, reasoning] + +# Laguna replaces Qwen and intentionally excludes ACE-Step. The 108 GB scene +# budget matches the empirically safe studio ceiling on the reference node while retaining +# Lumbridge Compute's separate 8 GB admission margin. +models: + - ears + - brain-laguna + - voice + +budget_gb: 108 +activation: + order: footprint-asc + wait_healthy: true diff --git a/scenes/voice-qwen.scene.yaml b/scenes/voice-qwen.scene.yaml new file mode 100644 index 0000000..a8ad025 --- /dev/null +++ b/scenes/voice-qwen.scene.yaml @@ -0,0 +1,26 @@ +apiVersion: lumbridge/v1 +kind: Scene +metadata: + name: voice-qwen + version: 2 + description: "Superseded by a newer scene on this node. Kept only because it is still the recorded fallback." + author: karti + tags: [assistant, voice, low-latency, deprecated] + +# DEPRECATED — renamed to "downstream" on 2026-08-02. This file is retained solely +# because last_known_good still points at it; delete once the fallback rolls forward. +# +# Its ordering was made brain-first to match its successor. `brain` now runs MTP, whose +# KV-cache profiling spike is not bounded by gpu-memory-utilization; the old +# footprint-asc order would start voice+ears first and leave brain short enough +# to trip the 3GB watchdog floor. A fallback that cannot come up is worse than +# no fallback. +models: + - brain + - ears + - voice + +budget_gb: 100 +activation: + order: listed + wait_healthy: true diff --git a/src/agent.rs b/src/agent.rs new file mode 100644 index 0000000..3a5d56c --- /dev/null +++ b/src/agent.rs @@ -0,0 +1,364 @@ +//! Resident Lumbridge Compute supervisor. + +use anyhow::{Context, Result}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use crate::config::{find_scene, Registry, RestartPolicy, Supervision}; +use crate::{gateway, governor, lifecycle, mem, proc}; + +pub struct TelemetryConfig { + pub metrics_url: String, + pub model: String, + pub interval: Duration, +} + +pub fn run( + root: &Path, + listen: &str, + upstream: &str, + floor: f64, + telemetry: Option, +) -> Result<()> { + println!("Lumbridge Compute agent starting"); + lifecycle::resume(root).context("resuming persisted desired Scene")?; + + let (gateway_done_tx, gateway_done_rx) = mpsc::channel(); + let listen_owned = listen.to_string(); + let upstream_owned = upstream.to_string(); + thread::spawn(move || { + let result = gateway::run(&listen_owned, &upstream_owned); + gateway_done_tx.send(result).ok(); + }); + + if let Some(telemetry) = telemetry { + let telemetry_root = root.to_path_buf(); + println!( + "Lumbridge Compute telemetry {} every {}s", + telemetry.metrics_url, + telemetry.interval.as_secs() + ); + thread::spawn(move || { + crate::telemetry::run_collector( + telemetry_root, + telemetry.metrics_url, + telemetry.model, + telemetry.interval, + ) + }); + } else { + println!("Lumbridge Compute telemetry disabled"); + } + + println!( + "Lumbridge Compute agent supervising memory floor {:.1} GB", + floor + ); + let supervisor_started = Instant::now(); + let mut supervisor = SupervisionTracker::default(); + loop { + match gateway_done_rx.try_recv() { + Ok(result) => return result.context("stable gateway stopped"), + Err(mpsc::TryRecvError::Disconnected) => { + anyhow::bail!("stable gateway supervisor disconnected") + } + Err(mpsc::TryRecvError::Empty) => {} + } + enforce_memory_floor(root, floor)?; + if let Err(error) = supervisor.tick(root, supervisor_started.elapsed().as_secs()) { + // A failed recovery is a degraded model, not a failed resident agent. + // The policy's backoff will try again; taking down the gateway and + // memory watchdog here would turn one voice outage into a node outage. + eprintln!("agent: supervision tick failed: {error:#}"); + } + thread::sleep(Duration::from_secs(1)); + } +} + +#[derive(Debug, Default)] +struct SupervisionTracker { + models: BTreeMap, +} + +#[derive(Debug, Default)] +struct ModelTrack { + consecutive_failures: u32, + next_check_at: u64, + retry_at: u64, + next_backoff_sec: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SupervisionAction { + None, + Restart, +} + +impl SupervisionTracker { + fn tick(&mut self, root: &Path, now: u64) -> Result<()> { + let registry = Registry::load(root)?; + let state = proc::State::load_checked(root)?; + let Some(desired) = state.desired_scene.as_deref() else { + self.models.clear(); + return Ok(()); + }; + let scene = find_scene(root, desired)?; + let supervised = supervised_model_ids(®istry, &scene.models); + self.models.retain(|id, _| supervised.contains(id)); + + for id in supervised { + let Some(model) = registry.models.get(&id) else { + continue; + }; + let policy = model + .supervision + .as_ref() + .expect("selected from models with supervision") + .clone(); + if policy.restart != RestartPolicy::Always { + continue; + } + if self.observe(&id, &policy, governor::is_running(model), now) + != SupervisionAction::Restart + { + continue; + } + + eprintln!( + "agent: supervised model '{id}' failed {} consecutive health checks; restarting only that model", + policy.failure_threshold() + ); + let attempt_started = Instant::now(); + let result = lifecycle::restart_supervised_model( + root, + &id, + Duration::from_secs(policy.startup_timeout_sec()), + ); + let finished_at = now.saturating_add(attempt_started.elapsed().as_secs()); + self.complete_restart(&id, &policy, finished_at, result.is_ok()); + match result { + Ok(true) => eprintln!("agent: supervised model '{id}' recovered"), + Ok(false) => eprintln!("agent: supervised model '{id}' recovered before restart"), + Err(error) => { + let retry_in = self.models[&id].retry_at.saturating_sub(finished_at); + eprintln!( + "agent: supervised model '{id}' restart failed: {error:#}; next attempt in no less than {retry_in} seconds" + ); + } + } + } + Ok(()) + } + + fn observe( + &mut self, + id: &str, + policy: &Supervision, + healthy: bool, + now: u64, + ) -> SupervisionAction { + let track = self.models.entry(id.to_string()).or_default(); + if now < track.next_check_at { + return SupervisionAction::None; + } + track.next_check_at = now.saturating_add(policy.check_interval_sec()); + + if healthy { + track.consecutive_failures = 0; + track.retry_at = 0; + track.next_backoff_sec = policy.backoff_sec(); + return SupervisionAction::None; + } + + track.consecutive_failures = track.consecutive_failures.saturating_add(1); + if track.consecutive_failures < policy.failure_threshold() || now < track.retry_at { + return SupervisionAction::None; + } + SupervisionAction::Restart + } + + fn complete_restart(&mut self, id: &str, policy: &Supervision, now: u64, success: bool) { + let track = self.models.entry(id.to_string()).or_default(); + track.next_check_at = now.saturating_add(policy.check_interval_sec()); + if success { + track.consecutive_failures = 0; + track.retry_at = 0; + track.next_backoff_sec = policy.backoff_sec(); + return; + } + + let backoff = track.next_backoff_sec.max(policy.backoff_sec()); + track.retry_at = now.saturating_add(backoff); + track.next_backoff_sec = backoff.saturating_mul(2).min(policy.max_backoff_sec()); + // Keep the counter at the threshold so the first check after backoff may + // retry, but a healthy probe at any point still resets it above. + track.consecutive_failures = policy.failure_threshold(); + } +} + +fn supervised_model_ids(registry: &Registry, scene_models: &[String]) -> BTreeSet { + scene_models + .iter() + .filter_map(|id| { + registry + .models + .get(id) + .filter(|model| model.supervision.is_some()) + .map(|_| id.clone()) + }) + .collect() +} + +fn enforce_memory_floor(root: &Path, floor: f64) -> Result<()> { + let memory = mem::read()?; + if memory.available_gb >= floor { + return Ok(()); + } + // Take the same lock a Scene transition takes, and skip this tick if a transition + // already holds it. Without this the watchdog races activation: activation is + // deliberately a stop-all-then-start-all sequence, so mid-transition the pool is + // legitimately tight and `state.procs` is being rewritten underneath us. Firing then + // would kill a model the transition just started, and then write a stale `state` + // over the transition's own — losing track of a process that is still alive. + // + // A tick skipped here costs one second. The floor is a backstop, and the transition + // holding the lock is doing its own admission checks. + let _lock = match lifecycle::TransitionLock::acquire(root) { + Ok(lock) => lock, + Err(_) => { + eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}, but a Scene transition holds the \ + lock; deferring to it for this tick", + memory.available_gb, floor + ); + return Ok(()); + } + }; + let mut state = proc::State::load_checked(root)?; + match state.newest_alive() { + Some((id, process)) => { + eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}; stopping newest owned model '{}' (pid {})", + memory.available_gb, floor, id, process.pid + ); + proc::stop_owned(&process)?; + state.procs.remove(&id); + state.active_scene = None; + state.last_error = Some(format!( + "watchdog stopped '{id}' after MemAvailable fell to {:.1} GB", + memory.available_gb + )); + state.save(root)?; + } + None => eprintln!( + "agent: MemAvailable {:.1} GB < floor {:.1}, but no identity-owned model can be stopped", + memory.available_gb, floor + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> Supervision { + serde_yaml::from_str( + "restart: always\ncheck_interval_sec: 10\nfailure_threshold: 3\nbackoff_sec: 60\nmax_backoff_sec: 240\nstartup_timeout_sec: 5\n", + ) + .unwrap() + } + + #[test] + fn models_without_supervision_are_not_reconciled() { + let registry: Registry = serde_yaml::from_str( + "apiVersion: lumbridge/v1\nmodels:\n plain:\n name: Plain\n footprint_gb: 1\n serve:\n kind: exec\n port: 9000\n voice:\n name: Voice\n footprint_gb: 8\n supervision:\n restart: always\n serve:\n kind: exec\n port: 8095\n", + ) + .unwrap(); + let scene_models = vec!["plain".to_string(), "voice".to_string()]; + + assert_eq!( + supervised_model_ids(®istry, &scene_models), + BTreeSet::from(["voice".to_string()]) + ); + } + + #[test] + fn only_consecutive_due_health_failures_trigger_a_restart() { + let mut tracker = SupervisionTracker::default(); + let policy = policy(); + + assert_eq!( + tracker.observe("voice", &policy, false, 0), + SupervisionAction::None + ); + // A poll before the configured interval is ignored entirely. + assert_eq!( + tracker.observe("voice", &policy, false, 5), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 10), + SupervisionAction::None + ); + // One healthy observation resets the consecutive count. + assert_eq!( + tracker.observe("voice", &policy, true, 20), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 30), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 40), + SupervisionAction::None + ); + assert_eq!( + tracker.observe("voice", &policy, false, 50), + SupervisionAction::Restart + ); + } + + #[test] + fn failed_restarts_back_off_exponentially_and_health_resets_them() { + let mut tracker = SupervisionTracker::default(); + let policy = policy(); + for now in [0, 10] { + assert_eq!( + tracker.observe("voice", &policy, false, now), + SupervisionAction::None + ); + } + assert_eq!( + tracker.observe("voice", &policy, false, 20), + SupervisionAction::Restart + ); + tracker.complete_restart("voice", &policy, 20, false); + assert_eq!(tracker.models["voice"].retry_at, 80); + + for now in [30, 40, 50, 60, 70] { + assert_eq!( + tracker.observe("voice", &policy, false, now), + SupervisionAction::None + ); + } + assert_eq!( + tracker.observe("voice", &policy, false, 80), + SupervisionAction::Restart + ); + tracker.complete_restart("voice", &policy, 80, false); + assert_eq!(tracker.models["voice"].retry_at, 200); + + assert_eq!( + tracker.observe("voice", &policy, true, 90), + SupervisionAction::None + ); + assert_eq!(tracker.models["voice"].consecutive_failures, 0); + assert_eq!(tracker.models["voice"].retry_at, 0); + assert_eq!(tracker.models["voice"].next_backoff_sec, 60); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..e249d1f --- /dev/null +++ b/src/config.rs @@ -0,0 +1,263 @@ +//! Config types + loaders for the model registry and scenes. +//! +//! The registry (`registry/models.yaml`) resolves stable model *ids* to weights + +//! launch commands, and evolves as models requantize. Scenes (`scenes/*.scene.yaml`) +//! reference those ids and stay stable — the public, shareable contract. +//! +//! Several fields below are deserialized but never read by the binary +//! (`api_version`, `channel`, `entry`, `app`, `app_dir`, `tags`, `author`). +//! That is deliberate: they are the published `lumbridge/v1` manifest surface, +//! and declaring them is what makes a manifest carrying them parse rather than +//! fail. Deleting them to satisfy the lint would silently narrow the contract. +#![allow(dead_code)] + +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +#[derive(Debug, Deserialize)] +pub struct Registry { + #[serde(rename = "apiVersion")] + pub api_version: String, + pub models: BTreeMap, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Model { + pub name: String, + /// Worst-case unified memory held once serving (weights + KV/cache + encoder). + pub footprint_gb: f64, + /// `stable` pins the exact weights; `latest` may resolve a newer quant. + #[serde(default)] + pub channel: Option, + #[serde(default)] + pub health: Option, + /// Optional marker that must appear in the health response. This + /// distinguishes mutually-exclusive models that intentionally share a port. + #[serde(default)] + pub health_contains: Option, + /// Opt-in resident-agent recovery. Models without this block keep the + /// original one-shot Scene lifecycle exactly as it was. + #[serde(default)] + pub supervision: Option, + pub serve: Serve, +} + +/// Recovery policy for one model in the persisted desired Scene. +/// +/// This belongs to the local registry rather than a shareable Scene: restarting +/// a runtime is an operator policy tied to the vetted command and footprint on +/// this node, not something a downloaded Scene may turn on. +#[derive(Debug, Clone, Deserialize)] +pub struct Supervision { + pub restart: RestartPolicy, + #[serde(default = "default_check_interval_sec")] + pub check_interval_sec: u64, + #[serde(default = "default_failure_threshold")] + pub failure_threshold: u32, + #[serde(default = "default_backoff_sec")] + pub backoff_sec: u64, + #[serde(default = "default_max_backoff_sec")] + pub max_backoff_sec: u64, + #[serde(default = "default_startup_timeout_sec")] + pub startup_timeout_sec: u64, +} + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum RestartPolicy { + Always, +} + +impl Supervision { + pub fn check_interval_sec(&self) -> u64 { + self.check_interval_sec.max(1) + } + + pub fn failure_threshold(&self) -> u32 { + self.failure_threshold.max(1) + } + + pub fn backoff_sec(&self) -> u64 { + self.backoff_sec.max(1) + } + + pub fn max_backoff_sec(&self) -> u64 { + self.max_backoff_sec.max(self.backoff_sec()) + } + + pub fn startup_timeout_sec(&self) -> u64 { + self.startup_timeout_sec.max(1) + } +} + +const fn default_check_interval_sec() -> u64 { + 30 +} + +const fn default_failure_threshold() -> u32 { + 3 +} + +const fn default_backoff_sec() -> u64 { + 60 +} + +const fn default_max_backoff_sec() -> u64 { + 900 +} + +const fn default_startup_timeout_sec() -> u64 { + 180 +} + +/// How to launch a model. Fields are permissive across runtimes (vllm / diffusers / +/// python / uvicorn); only the ones a given `kind` needs are populated. +#[derive(Debug, Clone, Deserialize)] +pub struct Serve { + pub kind: String, + /// Optional runtime executable for kind-based builders (for example a + /// model-specific vLLM virtualenv). Defaults to the kind's command on PATH. + #[serde(default)] + pub executable: Option, + #[serde(default)] + pub port: Option, + /// Explicit launch argv. If set, it is used verbatim (argv[0] = program) and + /// takes precedence over any `kind`-based builder. This is where launch commands + /// live — never in a scene — so downloaded scenes can't smuggle code. + #[serde(default)] + pub command: Vec, + #[serde(default)] + pub weights: Option, + #[serde(default)] + pub entry: Option, + #[serde(default)] + pub app: Option, + #[serde(default)] + pub app_dir: Option, + #[serde(default)] + pub served_name: Vec, + #[serde(default)] + pub args: BTreeMap, + #[serde(default)] + pub env: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub struct Scene { + #[serde(rename = "apiVersion")] + pub api_version: String, + pub metadata: SceneMeta, + /// Stable model ids resolved via the registry. + pub models: Vec, + /// Optional per-scene budget override (GB); defaults to the Governor's global budget. + #[serde(default)] + pub budget_gb: Option, + #[serde(default)] + pub activation: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SceneMeta { + pub name: String, + #[serde(default)] + pub version: u32, + #[serde(default)] + pub description: String, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub author: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Activation { + /// `footprint-asc` (default) | `listed`. + #[serde(default)] + pub order: Option, + #[serde(default)] + pub wait_healthy: Option, +} + +impl Registry { + pub fn load(root: &Path) -> Result { + let p = root.join("registry/models.yaml"); + let s = + fs::read_to_string(&p).with_context(|| format!("reading registry {}", p.display()))?; + serde_yaml::from_str(&s).with_context(|| format!("parsing registry {}", p.display())) + } +} + +/// Load every `*.scene.yaml` under `/scenes`, sorted by name. +pub fn load_scenes(root: &Path) -> Result> { + let dir = root.join("scenes"); + let mut out = Vec::new(); + if !dir.exists() { + return Ok(out); + } + for entry in fs::read_dir(&dir).with_context(|| format!("reading {}", dir.display()))? { + let path = entry?.path(); + let is_scene = path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".scene.yaml")) + .unwrap_or(false); + if is_scene { + let s = fs::read_to_string(&path)?; + let scene: Scene = + serde_yaml::from_str(&s).with_context(|| format!("parsing {}", path.display()))?; + out.push(scene); + } + } + out.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name)); + Ok(out) +} + +pub fn find_scene(root: &Path, name: &str) -> Result { + load_scenes(root)? + .into_iter() + .find(|s| s.metadata.name == name) + .with_context(|| format!("no scene named '{name}' in {}/scenes", root.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supervision_is_opt_in_and_defaults_are_bounded() { + let plain: Model = serde_yaml::from_str( + "name: plain\nfootprint_gb: 1\nserve:\n kind: exec\n port: 9000\n", + ) + .unwrap(); + assert!(plain.supervision.is_none()); + + let watched: Model = serde_yaml::from_str( + "name: watched\nfootprint_gb: 1\nsupervision:\n restart: always\nserve:\n kind: exec\n port: 9001\n", + ) + .unwrap(); + let policy = watched.supervision.unwrap(); + assert_eq!(policy.restart, RestartPolicy::Always); + assert_eq!(policy.check_interval_sec(), 30); + assert_eq!(policy.failure_threshold(), 3); + assert_eq!(policy.backoff_sec(), 60); + assert_eq!(policy.max_backoff_sec(), 900); + assert_eq!(policy.startup_timeout_sec(), 180); + } + + #[test] + fn zero_intervals_cannot_create_a_hot_restart_loop() { + let watched: Model = serde_yaml::from_str( + "name: watched\nfootprint_gb: 1\nsupervision:\n restart: always\n check_interval_sec: 0\n failure_threshold: 0\n backoff_sec: 0\n max_backoff_sec: 0\n startup_timeout_sec: 0\nserve:\n kind: exec\n port: 9001\n", + ) + .unwrap(); + let policy = watched.supervision.unwrap(); + assert_eq!(policy.check_interval_sec(), 1); + assert_eq!(policy.failure_threshold(), 1); + assert_eq!(policy.backoff_sec(), 1); + assert_eq!(policy.max_backoff_sec(), 1); + assert_eq!(policy.startup_timeout_sec(), 1); + } +} diff --git a/src/eval.rs b/src/eval.rs new file mode 100644 index 0000000..c54f152 --- /dev/null +++ b/src/eval.rs @@ -0,0 +1,494 @@ +//! Reproducible evaluations for any OpenAI-compatible model server. +//! +//! Suites are declarative YAML. Results are append-only JSON artifacts suitable +//! for CI, regression comparisons, and future publication to an eval registry. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Deserialize)] +struct Suite { + #[serde(rename = "apiVersion")] + api_version: String, + kind: String, + metadata: Metadata, + #[serde(default)] + defaults: Defaults, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +struct Metadata { + name: String, + #[serde(default)] + version: u32, + #[serde(default)] + description: String, + /// Part of the published suite manifest surface; parsed so a suite + /// carrying tags loads, not read by the runner itself. + #[serde(default)] + #[allow(dead_code)] + tags: Vec, +} + +#[derive(Debug, Default, Deserialize)] +struct Defaults { + #[serde(default = "default_max_tokens")] + max_tokens: u32, + #[serde(default = "default_temperature")] + temperature: f64, + #[serde(default = "default_repeat")] + repeat: u32, + #[serde(default)] + system: String, +} + +fn default_max_tokens() -> u32 { + 128 +} +fn default_temperature() -> f64 { + 0.0 +} +fn default_repeat() -> u32 { + 1 +} + +#[derive(Debug, Deserialize)] +struct Case { + id: String, + #[serde(default)] + category: String, + prompt: String, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + assertions: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Assertion { + Exact { value: String }, + Contains { value: String }, + ContainsAny { values: Vec }, + NotContains { value: String }, + MaxWords { value: usize }, +} + +#[derive(Debug, Serialize)] +struct Artifact { + schema_version: u32, + run_id: String, + suite: String, + suite_version: u32, + model: String, + base_url: String, + started_unix_ms: u128, + summary: Summary, + samples: Vec, +} + +#[derive(Debug, Serialize)] +struct Summary { + passed: usize, + total: usize, + score: f64, + mean_ttft_ms: f64, + p50_ttft_ms: f64, + p95_ttft_ms: f64, + mean_prefill_tps: f64, + mean_decode_tps: f64, +} + +#[derive(Debug, Serialize)] +struct Sample { + case_id: String, + category: String, + repetition: u32, + passed: bool, + assertion_results: Vec, + output: String, + reasoning: String, + prompt_tokens: u64, + completion_tokens: u64, + ttft_ms: f64, + total_ms: f64, + prefill_tps: f64, + decode_tps: f64, +} + +struct Completion { + output: String, + reasoning: String, + prompt_tokens: u64, + completion_tokens: u64, + ttft_ms: f64, + total_ms: f64, +} + +/// One line of `eval ls`. Suites themselves stay private — a caller has no +/// business reaching into cases and assertions — but the catalogue is the +/// useful part and is shared by the CLI and the MCP server. +#[derive(Debug, Clone)] +pub struct SuiteSummary { + pub name: String, + pub version: u32, + pub cases: usize, + pub description: String, +} + +pub fn catalogue(root: &Path) -> Result> { + Ok(load_all(root)? + .into_iter() + .map(|s| SuiteSummary { + name: s.metadata.name, + version: s.metadata.version, + cases: s.cases.len(), + description: s.metadata.description, + }) + .collect()) +} + +pub fn list(root: &Path) -> Result<()> { + println!("{:<20} {:>4} {:>5} DESCRIPTION", "SUITE", "VER", "CASES"); + for s in catalogue(root)? { + println!( + "{:<20} {:>4} {:>5} {}", + s.name, s.version, s.cases, s.description + ); + } + Ok(()) +} + +pub fn run( + root: &Path, + name: &str, + base_url: &str, + model: &str, + repeat: Option, +) -> Result<()> { + let suite = load(root, name)?; + let known_contract = suite.api_version == "lumbridge/v1"; + if !known_contract || suite.kind != "EvalSuite" { + bail!( + "unsupported eval contract: {}/{}", + suite.api_version, + suite.kind + ); + } + let reps = repeat.unwrap_or(suite.defaults.repeat).max(1); + println!( + "eval '{}' v{} → {} ({})", + suite.metadata.name, suite.metadata.version, model, base_url + ); + let started = now_ms(); + let mut samples = Vec::new(); + for case in &suite.cases { + for repetition in 1..=reps { + print!(" {:<24} [{}/{}] ", case.id, repetition, reps); + std::io::stdout().flush().ok(); + let c = stream_completion(base_url, model, &suite.defaults, case)?; + let assertion_results: Vec = case + .assertions + .iter() + .map(|a| evaluate(a, &c.output)) + .collect(); + let passed = assertion_results.iter().all(|v| *v); + let decode_seconds = ((c.total_ms - c.ttft_ms) / 1000.0).max(0.001); + let prefill_seconds = (c.ttft_ms / 1000.0).max(0.001); + let sample = Sample { + case_id: case.id.clone(), + category: case.category.clone(), + repetition, + passed, + assertion_results, + output: c.output, + reasoning: c.reasoning, + prompt_tokens: c.prompt_tokens, + completion_tokens: c.completion_tokens, + ttft_ms: c.ttft_ms, + total_ms: c.total_ms, + prefill_tps: c.prompt_tokens as f64 / prefill_seconds, + decode_tps: c.completion_tokens as f64 / decode_seconds, + }; + println!( + "{} ttft={:.0}ms decode={:.1}tok/s", + if passed { "PASS" } else { "FAIL" }, + sample.ttft_ms, + sample.decode_tps + ); + samples.push(sample); + } + } + let summary = summarize(&samples); + let run_id = format!("{}-{}-{}", started, slug(name), slug(model)); + let artifact = Artifact { + schema_version: 1, + run_id: run_id.clone(), + suite: suite.metadata.name, + suite_version: suite.metadata.version, + model: model.to_string(), + base_url: base_url.to_string(), + started_unix_ms: started, + summary, + samples, + }; + let out_dir = root.join("eval-results"); + fs::create_dir_all(&out_dir)?; + let out = out_dir.join(format!("{run_id}.json")); + fs::write(&out, serde_json::to_string_pretty(&artifact)?)?; + println!( + "\nscore {:.1}% ({}/{}) · mean TTFT {:.0}ms · prefill≈{:.1}tok/s · decode {:.1}tok/s", + artifact.summary.score * 100.0, + artifact.summary.passed, + artifact.summary.total, + artifact.summary.mean_ttft_ms, + artifact.summary.mean_prefill_tps, + artifact.summary.mean_decode_tps + ); + println!("result {}", out.display()); + Ok(()) +} + +fn stream_completion( + base_url: &str, + model: &str, + defaults: &Defaults, + case: &Case, +) -> Result { + let url = format!("{}/chat/completions", base_url.trim_end_matches('/')); + let mut messages = Vec::new(); + if !defaults.system.is_empty() { + messages.push(json!({"role":"system","content":defaults.system})); + } + messages.push(json!({"role":"user","content":case.prompt})); + let request = json!({ + "model": model, "messages": messages, "stream": true, + "stream_options": {"include_usage": true}, + "max_tokens": case.max_tokens.unwrap_or(defaults.max_tokens), + "temperature": defaults.temperature, + "chat_template_kwargs": {"enable_thinking": false} + }); + let mut child = Command::new("curl") + .args([ + "-sS", + "-N", + "-X", + "POST", + &url, + "-H", + "Content-Type: application/json", + "--data-binary", + "@-", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("starting curl (required for eval HTTP streaming)")?; + child + .stdin + .take() + .unwrap() + .write_all(request.to_string().as_bytes())?; + let start = Instant::now(); + let mut first = None; + let mut output = String::new(); + let mut reasoning = String::new(); + let mut prompt_tokens = 0; + let mut completion_tokens = 0; + for line in BufReader::new(child.stdout.take().unwrap()).lines() { + let line = line?; + let Some(data) = line.strip_prefix("data: ") else { + continue; + }; + if data == "[DONE]" { + break; + } + let v: Value = serde_json::from_str(data).context("parsing streamed completion")?; + if let Some(usage) = v.get("usage") { + prompt_tokens = usage + .get("prompt_tokens") + .and_then(Value::as_u64) + .unwrap_or(prompt_tokens); + completion_tokens = usage + .get("completion_tokens") + .and_then(Value::as_u64) + .unwrap_or(completion_tokens); + } + let delta = &v["choices"][0]["delta"]; + let content = delta.get("content").and_then(Value::as_str).unwrap_or(""); + let thought = delta + .get("reasoning") + .or_else(|| delta.get("reasoning_content")) + .and_then(Value::as_str) + .unwrap_or(""); + if first.is_none() && (!content.is_empty() || !thought.is_empty()) { + first = Some(start.elapsed()); + } + output.push_str(content); + reasoning.push_str(thought); + } + let status = child.wait()?; + if !status.success() { + bail!("completion request failed with {status}"); + } + let total = start.elapsed().as_secs_f64() * 1000.0; + Ok(Completion { + output, + reasoning, + prompt_tokens, + completion_tokens, + ttft_ms: first.map(|d| d.as_secs_f64() * 1000.0).unwrap_or(total), + total_ms: total, + }) +} + +fn evaluate(a: &Assertion, output: &str) -> bool { + let normalized = output.trim(); + match a { + Assertion::Exact { value } => normalized.eq_ignore_ascii_case(value.trim()), + Assertion::Contains { value } => normalized.to_lowercase().contains(&value.to_lowercase()), + Assertion::ContainsAny { values } => values + .iter() + .any(|v| normalized.to_lowercase().contains(&v.to_lowercase())), + Assertion::NotContains { value } => { + !normalized.to_lowercase().contains(&value.to_lowercase()) + } + Assertion::MaxWords { value } => normalized.split_whitespace().count() <= *value, + } +} + +fn summarize(samples: &[Sample]) -> Summary { + let mut ttft: Vec = samples.iter().map(|s| s.ttft_ms).collect(); + ttft.sort_by(f64::total_cmp); + let mean = |f: fn(&Sample) -> f64| { + if samples.is_empty() { + 0.0 + } else { + samples.iter().map(f).sum::() / samples.len() as f64 + } + }; + let percentile = |p: f64| { + if ttft.is_empty() { + 0.0 + } else { + ttft[((ttft.len() - 1) as f64 * p).round() as usize] + } + }; + let passed = samples.iter().filter(|s| s.passed).count(); + Summary { + passed, + total: samples.len(), + score: if samples.is_empty() { + 0.0 + } else { + passed as f64 / samples.len() as f64 + }, + mean_ttft_ms: mean(|s| s.ttft_ms), + p50_ttft_ms: percentile(0.50), + p95_ttft_ms: percentile(0.95), + mean_prefill_tps: mean(|s| s.prefill_tps), + mean_decode_tps: mean(|s| s.decode_tps), + } +} + +fn load(root: &Path, name: &str) -> Result { + load_all(root)? + .into_iter() + .find(|s| s.metadata.name == name) + .with_context(|| format!("no eval suite named '{name}' in {}/evals", root.display())) +} + +fn load_all(root: &Path) -> Result> { + let dir = root.join("evals"); + let mut suites = Vec::new(); + if !dir.exists() { + return Ok(suites); + } + for entry in fs::read_dir(&dir)? { + let path: PathBuf = entry?.path(); + if path + .file_name() + .and_then(|n| n.to_str()) + .map(|n| n.ends_with(".eval.yaml")) + .unwrap_or(false) + { + let suite: Suite = serde_yaml::from_str(&fs::read_to_string(&path)?) + .with_context(|| format!("parsing {}", path.display()))?; + suites.push(suite); + } + } + suites.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name)); + Ok(suites) +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} +fn slug(s: &str) -> String { + s.chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assertions_are_case_insensitive_and_composable() { + let output = "Market risk remains, while company-specific risk falls."; + assert!(evaluate( + &Assertion::Contains { + value: "MARKET RISK".into() + }, + output + )); + assert!(evaluate( + &Assertion::ContainsAny { + values: vec!["idiosyncratic".into(), "company-specific".into()], + }, + output + )); + assert!(evaluate( + &Assertion::NotContains { + value: "markdown".into() + }, + output + )); + } + + #[test] + fn exact_trims_but_does_not_accept_explanation() { + assert!(evaluate(&Assertion::Exact { value: "25".into() }, " 25\n")); + assert!(!evaluate(&Assertion::Exact { value: "25".into() }, "25 GB")); + } + + #[test] + fn word_limit_counts_whitespace_tokens() { + assert!(evaluate( + &Assertion::MaxWords { value: 4 }, + "one two three four" + )); + assert!(!evaluate( + &Assertion::MaxWords { value: 3 }, + "one two three four" + )); + } +} diff --git a/src/gateway.rs b/src/gateway.rs new file mode 100644 index 0000000..d7a53f9 --- /dev/null +++ b/src/gateway.rs @@ -0,0 +1,199 @@ +//! Streaming-safe TCP gateway for OpenAI-compatible model servers. +//! +//! The gateway deliberately stays below HTTP: it forwards bytes unchanged, so +//! chunked responses and server-sent-event token streams retain their timing and +//! semantics while clients keep one stable Lumbridge Compute address. + +use anyhow::{Context, Result}; +use std::io; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::thread; +use std::time::Instant; + +/// Identifies both byte-pump halves of a proxied connection in logs. +static CONN_ID: AtomicU64 = AtomicU64::new(0); + +pub fn run(listen: &str, upstream: &str) -> Result<()> { + let listener = TcpListener::bind(listen) + .with_context(|| format!("binding Lumbridge gateway at {listen}"))?; + println!("Lumbridge gateway {listen} -> {upstream}"); + serve_listener(listener, upstream, None) +} + +fn serve_listener( + listener: TcpListener, + upstream: &str, + max_connections: Option, +) -> Result<()> { + let mut accepted = 0usize; + for incoming in listener.incoming() { + let client = match incoming { + Ok(client) => client, + Err(error) => { + eprintln!("gateway accept failed: {error}"); + continue; + } + }; + let upstream = upstream.to_string(); + let id = CONN_ID.fetch_add(1, Ordering::Relaxed); + thread::spawn(move || { + if let Err(error) = proxy(id, client, &upstream) { + eprintln!("gateway request failed: conn={id} {error:#}"); + } + }); + accepted += 1; + if max_connections.is_some_and(|limit| accepted >= limit) { + break; + } + } + Ok(()) +} + +/// Shuts both sockets down on every return path. Dropping one descriptor is not +/// sufficient because each byte-pump owns a cloned descriptor for the same +/// socket; a blocked clone otherwise keeps a failed OpenAI request alive forever. +struct ShutdownGuard { + client: TcpStream, + upstream: TcpStream, +} + +impl Drop for ShutdownGuard { + fn drop(&mut self) { + self.client.shutdown(Shutdown::Both).ok(); + self.upstream.shutdown(Shutdown::Both).ok(); + } +} + +fn proxy(id: u64, mut client: TcpStream, upstream_addr: &str) -> Result<()> { + client.set_nodelay(true).ok(); + let mut upstream = TcpStream::connect(upstream_addr) + .with_context(|| format!("connecting gateway upstream {upstream_addr}"))?; + upstream.set_nodelay(true).ok(); + + let _guard = ShutdownGuard { + client: client.try_clone()?, + upstream: upstream.try_clone()?, + }; + + let started = Instant::now(); + let mut client_reader = client.try_clone()?; + let mut upstream_writer = upstream.try_clone()?; + let request = thread::spawn(move || -> io::Result { + match io::copy(&mut client_reader, &mut upstream_writer) { + Ok(copied) => { + upstream_writer.shutdown(Shutdown::Write).ok(); + Ok(copied) + } + Err(error) => { + eprintln!("gateway request-copy failed: conn={id} {error}"); + Err(error) + } + } + }); + + let downstream = io::copy(&mut upstream, &mut client); + client.shutdown(Shutdown::Write).ok(); + let elapsed = started.elapsed().as_millis(); + let down = match downstream { + Ok(bytes) => bytes, + Err(error) => { + eprintln!("gateway response-copy failed: conn={id} after {elapsed}ms {error}"); + return Err(error.into()); + } + }; + let up = request + .join() + .map_err(|_| anyhow::anyhow!("gateway request-copy thread panicked"))??; + eprintln!("gateway conn={id} ok up={up}B down={down}B {elapsed}ms"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + + #[test] + fn gateway_forwards_bidirectional_bytes_without_buffering_protocols() { + let upstream = TcpListener::bind("127.0.0.1:0").unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + let upstream_thread = thread::spawn(move || { + let (mut socket, _) = upstream.accept().unwrap(); + let mut request = [0u8; 4]; + socket.read_exact(&mut request).unwrap(); + assert_eq!(&request, b"ping"); + socket.write_all(b"pong").unwrap(); + }); + + let gateway = TcpListener::bind("127.0.0.1:0").unwrap(); + let gateway_addr = gateway.local_addr().unwrap(); + let gateway_thread = thread::spawn(move || { + serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap(); + }); + + let mut client = TcpStream::connect(gateway_addr).unwrap(); + client.write_all(b"ping").unwrap(); + client.shutdown(Shutdown::Write).unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + assert_eq!(response, b"pong"); + + upstream_thread.join().unwrap(); + gateway_thread.join().unwrap(); + } + + #[test] + fn upstream_reset_does_not_strand_the_client() { + use std::os::fd::AsRawFd; + use std::time::Duration; + + let upstream = TcpListener::bind("127.0.0.1:0").unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + let upstream_thread = thread::spawn(move || { + let (socket, _) = upstream.accept().unwrap(); + let linger = libc::linger { + l_onoff: 1, + l_linger: 0, + }; + unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_LINGER, + &linger as *const _ as *const libc::c_void, + std::mem::size_of::() as libc::socklen_t, + ); + } + thread::sleep(Duration::from_millis(50)); + drop(socket); + }); + + let gateway = TcpListener::bind("127.0.0.1:0").unwrap(); + let gateway_addr = gateway.local_addr().unwrap(); + let gateway_thread = thread::spawn(move || { + serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap(); + }); + + let mut client = TcpStream::connect(gateway_addr).unwrap(); + client + .write_all(b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut response = Vec::new(); + if let Err(error) = client.read_to_end(&mut response) { + assert!( + !matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ), + "client was stranded after upstream reset: {error:?}" + ); + } + + upstream_thread.join().unwrap(); + gateway_thread.join().unwrap(); + } +} diff --git a/src/governor.rs b/src/governor.rs new file mode 100644 index 0000000..d1a0c0b --- /dev/null +++ b/src/governor.rs @@ -0,0 +1,223 @@ +//! The Governor — the kernel of Lumbridge Compute. +//! +//! On a unified-memory box, over-commit doesn't fail gracefully: the whole machine +//! thrashes and wedges (SSH/ping included) before the OOM killer acts. The Governor +//! makes that impossible via (1) admission control against a hard budget and +//! (2) a watchdog on `MemAvailable` that kills the newest model before thrash. +//! +//! This module currently provides the *sensing* + *admission* half. The watchdog and +//! process spawn/kill land with the `up`/`activate` commands. + +use crate::config::{Model, Registry}; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +/// Usable memory for models; the rest is reserved for the OS/desktop. +pub const DEFAULT_BUDGET_GB: f64 = 100.0; +/// Extra headroom required before admitting a new model. +pub const SAFETY_MARGIN_GB: f64 = 8.0; +/// If `MemAvailable` dips below this, the watchdog kills the newest model. +pub const WATCHDOG_FLOOR_GB: f64 = 3.0; + +/// A model is considered "running" if its serving port accepts a connection. +pub fn port_open(port: u16) -> bool { + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + TcpStream::connect_timeout(&addr, Duration::from_millis(150)).is_ok() +} + +pub fn is_running(m: &Model) -> bool { + let Some(port) = m.serve.port else { + return false; + }; + if !port_open(port) { + return false; + } + match &m.health_contains { + Some(marker) => health_response(m, port) + .map(|response| response.contains(marker)) + .unwrap_or(false), + None => true, + } +} + +/// Fetch the local HTTP health endpoint without adding an HTTP client runtime. +/// Registry health URLs are deliberately localhost-only. +fn health_response(m: &Model, port: u16) -> Option { + let path = m + .health + .as_deref() + .and_then(|url| url.split_once("localhost")) + .map(|(_, tail)| tail.trim_start_matches(|c: char| c.is_ascii_digit() || c == ':')) + .filter(|path| path.starts_with('/')) + .unwrap_or("/"); + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let mut stream = TcpStream::connect_timeout(&addr, Duration::from_millis(300)).ok()?; + stream + .set_read_timeout(Some(Duration::from_millis(500))) + .ok()?; + stream + .write_all(format!("GET {path} HTTP/1.0\r\nHost: localhost\r\n\r\n").as_bytes()) + .ok()?; + let mut response = String::new(); + stream.read_to_string(&mut response).ok()?; + Some(response) +} + +pub fn running_ids(reg: &Registry) -> Vec { + reg.models + .iter() + .filter(|(_, m)| is_running(m)) + .map(|(id, _)| id.clone()) + .collect() +} + +/// Sum of footprints of all currently-serving registered models. +pub fn committed_gb(reg: &Registry) -> f64 { + reg.models + .values() + .filter(|m| is_running(m)) + .map(|m| m.footprint_gb) + .sum() +} + +/// Admission control. Two independent ceilings, and both must hold. +/// +/// 1. **Declared** — `committed + add + margin <= budget`. +/// 2. **Observed** — `add + margin <= available`. +/// +/// The second one is what makes the promise real. `budget_gb`, and every +/// `footprint_gb` feeding `committed_gb`, are numbers a human typed into the +/// registry. If any of them is optimistic — a model that grows past its declared +/// footprint, a KV cache larger than expected, anything started outside Compute — +/// the declared check happily passes while the box is already out of memory, and +/// on unified memory that ends in a wedged machine rather than a failed malloc. +/// `available_gb` comes from `MemAvailable`, which counts reality. +/// +/// `available_gb` is `None` when sensing failed. That falls back to the declared +/// ceiling alone: refusing every start because /proc/meminfo was unreadable would +/// turn a sensing failure into a total outage. +pub fn can_admit( + add_gb: f64, + committed_gb: f64, + budget_gb: f64, + available_gb: Option, +) -> bool { + if committed_gb + add_gb + SAFETY_MARGIN_GB > budget_gb { + return false; + } + match available_gb { + Some(available) => add_gb + SAFETY_MARGIN_GB <= available, + None => true, + } +} + +/// What admission will actually enforce right now, for reporting. The lower of the +/// declared headroom and the observed headroom. +pub fn headroom_gb(committed_gb: f64, budget_gb: f64, available_gb: Option) -> f64 { + let declared = budget_gb - committed_gb - SAFETY_MARGIN_GB; + match available_gb { + Some(available) => declared.min(available - SAFETY_MARGIN_GB), + None => declared, + } +} + +/// Block until the exact registered model is healthy, or `timeout` elapses. +/// Models without a port are considered instantly ready. +pub fn wait_healthy(m: &Model, timeout: Duration) -> bool { + let Some(_) = m.serve.port else { + return true; + }; + let start = Instant::now(); + while start.elapsed() < timeout { + if is_running(m) { + return true; + } + std::thread::sleep(Duration::from_millis(400)); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + // A 108 GB Scene budget on a box with plenty free — the ordinary case. + const BUDGET: f64 = 108.0; + + #[test] + fn admits_when_both_declared_and_observed_ceilings_allow_it() { + assert!(can_admit(66.0, 9.0, BUDGET, Some(100.0))); + } + + #[test] + fn refuses_when_the_declared_budget_is_exceeded() { + // 66 + 40 + 8 margin = 114 > 108, even though the box has memory free. + assert!(!can_admit(66.0, 40.0, BUDGET, Some(100.0))); + } + + #[test] + fn refuses_when_the_box_is_out_of_memory_even_though_the_paperwork_agrees() { + // This is the case the declared check alone could never catch: nothing is + // registered as committed, so the budget says there is 100 GB of room, but + // MemAvailable says 20 GB. Something outside Compute is holding the pool. + assert!(can_admit(66.0, 0.0, BUDGET, None), "declared check passes"); + assert!(!can_admit(66.0, 0.0, BUDGET, Some(20.0))); + } + + #[test] + fn the_safety_margin_is_enforced_against_observed_memory_too() { + // 66 GB model with exactly 66 GB free is a refusal: the margin has to fit. + assert!(!can_admit(66.0, 0.0, BUDGET, Some(66.0))); + assert!(!can_admit( + 66.0, + 0.0, + BUDGET, + Some(66.0 + SAFETY_MARGIN_GB - 0.1) + )); + assert!(can_admit(66.0, 0.0, BUDGET, Some(66.0 + SAFETY_MARGIN_GB))); + } + + #[test] + fn failed_sensing_falls_back_to_the_declared_ceiling_rather_than_refusing_everything() { + assert!(can_admit(66.0, 9.0, BUDGET, None)); + // ...but it must not become a way to bypass the declared budget. + assert!(!can_admit(66.0, 40.0, BUDGET, None)); + } + + #[test] + fn a_zero_reading_refuses_everything_rather_than_admitting_everything() { + // parse_meminfo yields 0.0 for an unparseable /proc. That has to read as + // "no memory", not as "no constraint". + assert!(!can_admit(1.0, 0.0, BUDGET, Some(0.0))); + } + + #[test] + fn headroom_never_exceeds_what_the_machine_actually_has() { + // The shape of a real regression: a generous declared budget on a smaller box. Reporting + // budget-minus-committed here promises room the next admission will refuse, and both the + // MCP tool and the HTTP API serve this number to callers that cannot check it themselves. + let box_available = 55.8; + let declared_budget = 100.0; + let reported = headroom_gb(0.0, declared_budget, Some(box_available)); + assert!( + reported <= box_available, + "reported {reported} GB of headroom on a box with {box_available} GB free", + ); + assert!(can_admit( + reported, + 0.0, + declared_budget, + Some(box_available) + )); + } + + #[test] + fn headroom_reports_the_binding_constraint_not_the_generous_one() { + // Declared says 33 GB spare; the box says 12 GB spare minus margin. + assert_eq!(headroom_gb(67.0, BUDGET, None), 33.0); + assert_eq!(headroom_gb(67.0, BUDGET, Some(12.0)), 4.0); + // And the declared ceiling still wins when it is the tighter of the two. + assert_eq!(headroom_gb(100.0, BUDGET, Some(90.0)), 0.0); + } +} diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..b31c690 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,344 @@ +//! A read-only HTTP control API. +//! +//! This exists so a web UI can read a node's state without shelling out to the CLI. It is a +//! second transport over the operations the MCP server already models — `collect_status`, +//! `collect_models`, `collect_scenes`, `collect_scene`, `collect_evals` — deliberately not a +//! second implementation of them, so the two surfaces cannot drift. +//! +//! Hand-rolled on `std::net`, matching `gateway.rs`. An HTTP framework would pull a dependency +//! tree an order of magnitude larger than the whole rest of this binary, to serve a small set +//! of routes that return pre-serialised JSON. +//! +//! # What this deliberately does not do +//! +//! **It never mutates.** No activate, no stop, no eval run. A read-only surface that a browser +//! can reach is a much smaller thing to get right than one that can move a node's memory around, +//! and the read half is what a dashboard actually needs. +//! +//! # Why the defaults are what they are +//! +//! - **Loopback only.** The listen address defaults to `127.0.0.1`. There is no authentication +//! worth the name here, so a bind to `0.0.0.0` publishes your node's inventory to the network. +//! - **CORS off.** No origin is allowed unless named with `--allow-origin`. Allowing `*` would +//! let *any* page you visit read what models you run, which is a fingerprint of your machine. +//! - **The token is optional but checked in constant time.** Loopback plus an origin allowlist +//! already stops the browser attack; the token is for the case where someone puts this behind +//! a proxy anyway. + +use anyhow::{Context, Result}; +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; + +use crate::mcp::{collect_evals, collect_models, collect_scene, collect_scenes, collect_status}; + +/// Requests are tiny and come from localhost; anything slower than this is not a browser. +const READ_TIMEOUT: Duration = Duration::from_secs(5); +/// A request line plus headers. Anything larger is not a request we serve. +const MAX_HEAD_BYTES: usize = 8 * 1024; + +pub struct Config { + pub root: PathBuf, + /// Origins permitted to read this API from a browser. Empty means none. + pub allowed_origins: Vec, + /// When set, every request must carry `Authorization: Bearer `. + pub token: Option, +} + +pub fn run(listen: &str, config: Config) -> Result<()> { + let listener = TcpListener::bind(listen) + .with_context(|| format!("binding the Lumbridge Compute API at {listen}"))?; + println!( + "Lumbridge Compute API on {listen} (read-only) · origins: {} · token: {}", + if config.allowed_origins.is_empty() { + "none".to_string() + } else { + config.allowed_origins.join(", ") + }, + if config.token.is_some() { + "required" + } else { + "none" + }, + ); + if !listen.starts_with("127.") && !listen.starts_with("localhost") { + eprintln!( + "warning: {listen} is not loopback. This API has no authentication by default and \ + reveals which models this node runs." + ); + } + serve(listener, config) +} + +fn serve(listener: TcpListener, config: Config) -> Result<()> { + let config = std::sync::Arc::new(config); + for incoming in listener.incoming() { + let stream = match incoming { + Ok(stream) => stream, + Err(error) => { + eprintln!("api accept failed: {error}"); + continue; + } + }; + let config = config.clone(); + thread::spawn(move || { + if let Err(error) = handle(stream, &config) { + eprintln!("api request failed: {error:#}"); + } + }); + } + Ok(()) +} + +struct Request { + method: String, + path: String, + query: BTreeMap, + origin: Option, + authorization: Option, +} + +/// Read the request line and headers. The body is ignored: every route is a GET. +fn read_request(stream: &TcpStream) -> Result> { + let mut reader = BufReader::new(stream); + let mut head = String::new(); + let mut total = 0usize; + + loop { + let mut line = String::new(); + let n = reader.read_line(&mut line)?; + if n == 0 { + return Ok(None); // client hung up + } + total += n; + if total > MAX_HEAD_BYTES { + return Ok(None); + } + if line == "\r\n" || line == "\n" { + break; + } + head.push_str(&line); + } + + let mut lines = head.lines(); + let Some(request_line) = lines.next() else { + return Ok(None); + }; + let mut parts = request_line.split_whitespace(); + let (Some(method), Some(target)) = (parts.next(), parts.next()) else { + return Ok(None); + }; + + let mut origin = None; + let mut authorization = None; + for line in lines { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim().to_string(); + match name.trim().to_ascii_lowercase().as_str() { + "origin" => origin = Some(value), + "authorization" => authorization = Some(value), + _ => {} + } + } + + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let query = query + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| pair.split_once('=').unwrap_or((pair, ""))) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + + Ok(Some(Request { + method: method.to_string(), + path: path.to_string(), + query, + origin, + authorization, + })) +} + +/// Constant-time comparison so a token cannot be recovered a byte at a time from response timing. +fn token_ok(expected: &str, supplied: Option<&String>) -> bool { + let Some(supplied) = supplied.and_then(|v| v.strip_prefix("Bearer ")) else { + return false; + }; + let a = expected.as_bytes(); + let b = supplied.as_bytes(); + // Length is compared without branching on it beyond the final AND. + let mut diff = (a.len() ^ b.len()) as u8; + for i in 0..a.len().max(b.len()) { + diff |= a.get(i).copied().unwrap_or(0) ^ b.get(i).copied().unwrap_or(0); + } + diff == 0 +} + +fn handle(mut stream: TcpStream, config: &Config) -> Result<()> { + stream.set_read_timeout(Some(READ_TIMEOUT))?; + stream.set_write_timeout(Some(READ_TIMEOUT))?; + + let Some(request) = read_request(&stream)? else { + return Ok(()); + }; + + // Echo the origin only when it is on the allowlist. Never `*`: this API describes the + // machine it runs on, so any-origin access means any page can fingerprint the node. + let allow_origin = request + .origin + .as_ref() + .filter(|o| config.allowed_origins.iter().any(|a| a == *o)) + .cloned(); + + if request.method == "OPTIONS" { + return write_response(&mut stream, 204, "", allow_origin.as_deref(), true); + } + + if let Some(expected) = &config.token { + if !token_ok(expected, request.authorization.as_ref()) { + return write_json( + &mut stream, + 401, + r#"{"error":"unauthorized"}"#, + allow_origin.as_deref(), + ); + } + } + + if request.method != "GET" { + return write_json( + &mut stream, + 405, + r#"{"error":"this API is read-only"}"#, + allow_origin.as_deref(), + ); + } + + let (status, body) = route(&request.path, &request.query, &config.root); + write_json(&mut stream, status, &body, allow_origin.as_deref()) +} + +fn route(path: &str, query: &BTreeMap, root: &Path) -> (u16, String) { + let since = query.get("since").map(String::as_str).unwrap_or("24h"); + let rendered = match path { + "/v1/health" => Ok(r#"{"ok":true,"service":"lumbridge-compute","api":"v1"}"#.to_string()), + "/v1/status" => collect_status(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/models" => collect_models(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/scenes" => collect_scenes(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/evals" => collect_evals(root).and_then(|r| Ok(serde_json::to_string(&r)?)), + "/v1/usage" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report)?)), + "/v1/usage/agents" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report.agents)?)), + "/v1/usage/concurrency" => crate::telemetry::report(root, since) + .and_then(|report| Ok(serde_json::to_string(&report.concurrency)?)), + other => match other.strip_prefix("/v1/scenes/") { + // Exactly one segment: /v1/scenes/a/b is not a route. + Some(name) if !name.is_empty() && !name.contains('/') => { + collect_scene(root, name).and_then(|r| Ok(serde_json::to_string(&r)?)) + } + _ => return (404, r#"{"error":"no such route"}"#.to_string()), + }, + }; + + match rendered { + Ok(body) => (200, body), + // A collector fails when the thing does not exist (an unknown Scene) or when the node's + // own config is unreadable. The message is the operator's, and this is a loopback API, + // so passing it through is more useful than flattening it to "error". + Err(error) => ( + 404, + serde_json::json!({ "error": format!("{error:#}") }).to_string(), + ), + } +} + +fn write_json(stream: &mut TcpStream, status: u16, body: &str, origin: Option<&str>) -> Result<()> { + write_response(stream, status, body, origin, false) +} + +fn write_response( + stream: &mut TcpStream, + status: u16, + body: &str, + origin: Option<&str>, + preflight: bool, +) -> Result<()> { + let reason = match status { + 200 => "OK", + 204 => "No Content", + 401 => "Unauthorized", + 404 => "Not Found", + 405 => "Method Not Allowed", + _ => "Error", + }; + let mut head = format!("HTTP/1.1 {status} {reason}\r\n"); + head.push_str("Content-Type: application/json\r\n"); + head.push_str(&format!("Content-Length: {}\r\n", body.len())); + // This is live node state; a cached answer is a wrong answer. + head.push_str("Cache-Control: no-store\r\n"); + head.push_str("Connection: close\r\n"); + if let Some(origin) = origin { + head.push_str(&format!("Access-Control-Allow-Origin: {origin}\r\n")); + // Tell caches the body varies by origin, so an allowed origin's response can never be + // replayed to a disallowed one. + head.push_str("Vary: Origin\r\n"); + if preflight { + head.push_str("Access-Control-Allow-Methods: GET, OPTIONS\r\n"); + head.push_str("Access-Control-Allow-Headers: Authorization\r\n"); + head.push_str("Access-Control-Max-Age: 600\r\n"); + } + } + head.push_str("\r\n"); + stream.write_all(head.as_bytes())?; + stream.write_all(body.as_bytes())?; + stream.flush()?; + Ok(()) +} + +/// Drain and discard — kept for symmetry with future routes that accept a body. +#[allow(dead_code)] +fn discard_body(reader: &mut impl Read) { + let mut sink = Vec::new(); + let _ = reader.read_to_end(&mut sink); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_comparison_rejects_wrong_and_missing_and_prefixless() { + let t = "s3cret-token"; + assert!(token_ok(t, Some(&format!("Bearer {t}")))); + assert!(!token_ok(t, None)); + assert!(!token_ok(t, Some(&t.to_string()))); // no "Bearer " prefix + assert!(!token_ok(t, Some(&"Bearer wrong".to_string()))); + // A correct prefix must not pass — this is the bug constant-time comparison exists for. + assert!(!token_ok(t, Some(&"Bearer s3cret".to_string()))); + assert!(!token_ok(t, Some(&"Bearer s3cret-token-plus".to_string()))); + } + + #[test] + fn unknown_routes_404_and_scene_paths_take_exactly_one_segment() { + let root = Path::new("/nonexistent-root-for-routing-test"); + let query = BTreeMap::new(); + assert_eq!(route("/v1/nope", &query, root).0, 404); + assert_eq!(route("/v1/scenes/a/b", &query, root).0, 404); + assert_eq!(route("/v1/scenes/", &query, root).0, 404); + // Health needs no filesystem, so it answers even on a bogus root. + assert_eq!(route("/v1/health", &query, root).0, 200); + } + + #[test] + fn health_body_is_valid_json() { + let (status, body) = route("/v1/health", &BTreeMap::new(), Path::new("/tmp")); + assert_eq!(status, 200); + let parsed: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(parsed["ok"], true); + } +} diff --git a/src/lifecycle.rs b/src/lifecycle.rs new file mode 100644 index 0000000..95181eb --- /dev/null +++ b/src/lifecycle.rs @@ -0,0 +1,915 @@ +//! Transactional Scene lifecycle and persisted desired-state recovery. + +use anyhow::{bail, Context, Result}; +use std::collections::{BTreeMap, HashSet}; +use std::fs::{self, File, OpenOptions}; +use std::os::fd::AsRawFd; +use std::path::Path; +use std::time::Duration; + +use crate::config::{find_scene, Model, Registry, Scene}; +use crate::governor; +use crate::mem; +use crate::proc::{self, Proc, State}; + +/// What activating a Scene would do, decided before anything is mutated. +/// +/// `activate` renders this for the CLI and the MCP server returns it verbatim, +/// so a plan an operator reads and a plan an agent reads can never drift apart. +#[derive(Debug, Clone)] +pub struct Plan { + pub scene: String, + pub budget_gb: f64, + /// Registered models serving outside the target Scene; stopped first. + pub stop: Vec, + /// Scene models not yet serving, in the order they would be admitted. + pub start: Vec, +} + +pub struct TransitionLock { + file: File, +} + +impl TransitionLock { + pub fn acquire(root: &Path) -> Result { + let dir = root.join(".compute"); + fs::create_dir_all(&dir)?; + let path = dir.join("transition.lock"); + let file = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&path) + .with_context(|| format!("opening transition lock {}", path.display()))?; + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if result != 0 { + return Err(std::io::Error::last_os_error()) + .context("another Scene transition is already running; wait for it to finish"); + } + Ok(Self { file }) + } +} + +impl Drop for TransitionLock { + fn drop(&mut self) { + unsafe { + libc::flock(self.file.as_raw_fd(), libc::LOCK_UN); + } + } +} + +pub fn adopt(root: &Path, name: &str) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + + let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); + let running = governor::running_ids(®istry); + let extras: Vec = running + .iter() + .filter(|id| !target.contains(id.as_str())) + .cloned() + .collect(); + if !extras.is_empty() { + bail!( + "cannot adopt '{name}': registered model(s) outside the Scene are serving: {}", + extras.join(", ") + ); + } + + let previous = State::load_checked(root)?; + let mut adopted = BTreeMap::new(); + for id in &scene.models { + let model = ®istry.models[id]; + if !governor::is_running(model) { + bail!("cannot adopt '{name}': exact model '{id}' is not healthy"); + } + let legacy = previous.procs.get(id).with_context(|| { + format!( + "cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute" + ) + })?; + let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port) + .with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?; + adopted.insert(id.clone(), captured); + } + + let mut state = previous; + state.procs = adopted; + state.desired_scene = Some(name.to_string()); + state.active_scene = Some(name.to_string()); + state.last_known_good_scene = Some(name.to_string()); + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("adopted exact running Scene '{name}' and captured process identities"); + Ok(()) +} + +pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let prior = State::load_checked(root)?; + let prior_active = prior.active_scene.clone(); + + if dry_run { + return activate_once(root, name, true); + } + + match activate_once(root, name, false) { + Ok(()) => { + let mut state = State::load_checked(root)?; + if prior_active.as_deref() != Some(name) { + if let Some(previous) = prior_active { + state.last_known_good_scene = Some(previous); + } + } + state.desired_scene = Some(name.to_string()); + state.active_scene = Some(name.to_string()); + if state.last_known_good_scene.is_none() { + state.last_known_good_scene = Some(name.to_string()); + } + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("Scene '{name}' is active and persisted as desired"); + Ok(()) + } + Err(target_error) => { + let target_message = format!("activating '{name}' failed: {target_error:#}"); + eprintln!("{target_message}"); + let transition_started = + State::load_checked(root)?.transition_scene.as_deref() == Some(name); + // Where a failed transition would roll back to. Absent when the scene being + // activated is already the active one, which is the ordinary case for + // "start the models in this scene that are currently down". + let rollback_target = prior_active.as_deref().filter(|previous| *previous != name); + // Only tear the box down when there is somewhere to put it back. + // + // This used to run whenever a transition had begun, including when the target + // scene *was* the prior scene — so a refused admission stopped every running + // model and then skipped the rollback, because the rollback target had been + // filtered out for being the same scene. Twice on 2026-08-14 that turned one + // model failing admission into an empty box, taking brain-nemotron and music + // down with it. + // + // With no rollback target, leaving the partially-started scene running is + // strictly better: those models are healthy and they belong to the scene that + // was asked for. The refusal is reported either way. + let cleanup_error = if transition_started && rollback_target.is_some() { + stop_all_owned(root).err() + } else { + None + }; + let rollback = if transition_started && cleanup_error.is_none() { + rollback_target + .map(|previous| (previous.to_string(), activate_once(root, previous, false))) + } else { + None + }; + + let mut state = State::load_checked(root)?; + state.transition_scene = None; + state.last_error = Some(target_message.clone()); + match rollback { + Some((previous, Ok(()))) => { + state.desired_scene = Some(previous.clone()); + state.active_scene = Some(previous.clone()); + if state.last_known_good_scene.is_none() { + state.last_known_good_scene = Some(previous.clone()); + } + state.save(root)?; + eprintln!("rolled back to Scene '{previous}'"); + bail!("{target_message}; rolled back to '{previous}'") + } + Some((previous, Err(rollback_error))) => { + state.active_scene = None; + state.save(root)?; + bail!( + "{target_message}; rollback to '{previous}' also failed: {rollback_error:#}" + ) + } + None if !transition_started => { + state.save(root)?; + bail!("{target_message}") + } + None if cleanup_error.is_none() => { + state.active_scene = None; + state.save(root)?; + bail!("{target_message}") + } + None => { + state.active_scene = None; + state.save(root)?; + let cleanup_error = cleanup_error.expect("guarded by match condition"); + bail!("{target_message}; cleanup also failed: {cleanup_error:#}") + } + } + } + } +} + +fn stop_all_owned(root: &Path) -> Result<()> { + let mut state = State::load_checked(root)?; + let owned: Vec<(String, Proc)> = state + .procs + .iter() + .filter(|(_, process)| process.owned_alive()) + .map(|(id, process)| (id.clone(), process.clone())) + .collect(); + for (id, process) in owned { + proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?; + state.procs.remove(&id); + state.save(root)?; + } + Ok(()) +} + +pub fn resume(root: &Path) -> Result<()> { + let state = State::load_checked(root)?; + let desired = state + .desired_scene + .clone() + .or_else(|| state.last_known_good_scene.clone()) + .context("no desired Scene is persisted; activate or adopt one first")?; + let fallback = state.last_known_good_scene.clone(); + + match activate(root, &desired, false) { + Ok(()) => Ok(()), + Err(desired_error) => { + let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else { + return Err(desired_error).context("resuming desired Scene"); + }; + eprintln!( + "desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'" + ); + activate(root, &fallback, false).with_context(|| { + format!( + "desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed" + ) + }) + } + } +} + +/// Restart one explicitly supervised model without disturbing the other models +/// in the desired Scene. +/// +/// This is deliberately narrower than `activate`: a resident recovery attempt +/// must not turn one failed voice runtime into a stop-all rollback of an +/// otherwise healthy Scene. The same transition lock, identity ownership, +/// declared/observed admission checks and exact health gate still apply. +/// Returns `true` when a new process was started and `false` when the model +/// recovered before the lock was acquired. +pub fn restart_supervised_model(root: &Path, id: &str, timeout: Duration) -> Result { + let _lock = TransitionLock::acquire(root)?; + let result = restart_supervised_model_locked(root, id, timeout); + if let Err(error) = &result { + if let Ok(mut state) = State::load_checked(root) { + state.active_scene = None; + state.last_error = Some(format!("supervision could not restart '{id}': {error:#}")); + let _ = state.save(root); + } + } + result +} + +fn restart_supervised_model_locked(root: &Path, id: &str, timeout: Duration) -> Result { + let registry = Registry::load(root)?; + let model = registry + .models + .get(id) + .with_context(|| format!("no registered model '{id}'"))?; + model + .supervision + .as_ref() + .with_context(|| format!("model '{id}' did not opt into supervision"))?; + let port = model + .serve + .port + .with_context(|| format!("supervised model '{id}' has no serving port"))?; + + let mut state = State::load_checked(root)?; + let desired = state + .desired_scene + .clone() + .context("no desired Scene is persisted")?; + let scene = find_scene(root, &desired)?; + validate_scene(®istry, &scene)?; + if !scene.models.iter().any(|candidate| candidate == id) { + bail!("model '{id}' is not part of desired Scene '{desired}'"); + } + + // The probe that triggered supervision ran before the transition lock. A + // late request or a runtime finishing its own recovery may have repaired the + // model in that gap, so re-check before signalling anything. + if governor::is_running(model) { + return Ok(false); + } + + match state.procs.get(id).cloned() { + Some(process) if process.owned_alive() => { + // Health is bad but this is still the exact process Compute started. + // Stop only its process group; no sibling in the Scene is touched. + proc::stop_owned(&process)?; + } + Some(_) if governor::port_open(port) => { + bail!( + "port {port} is occupied but the ownership record for '{id}' is stale; refusing to signal it" + ); + } + None if governor::port_open(port) => { + bail!( + "port {port} is occupied by an unowned process; refusing to replace supervised model '{id}'" + ); + } + Some(_) | None => {} + } + + state.procs.remove(id); + state.active_scene = None; + state.last_error = Some(format!("supervision is restarting '{id}'")); + state.save(root)?; + + // Re-read both ceilings after the old process has released its memory. The + // registry number prevents paper overcommit; MemAvailable prevents an + // unregistered workload from turning a nominally valid restart into a wedged + // unified-memory node. + let committed = governor::committed_gb(®istry); + let available = mem::read().ok().map(|memory| memory.available_gb); + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + if !governor::can_admit(model.footprint_gb, committed, budget, available) { + match available { + Some(available) => bail!( + "'{id}' ({:.0} GB) restart refused: {:.0} GB committed against a {:.0} GB budget, \ + {:.1} GB actually available, {:.0} GB margin required", + model.footprint_gb, + committed, + budget, + available, + governor::SAFETY_MARGIN_GB + ), + None => bail!( + "'{id}' restart would exceed the {:.0} GB Scene budget with the safety margin", + budget + ), + } + } + + let pid = proc::spawn(root, id, model)?; + state.seq += 1; + let process = Proc::capture(pid, state.seq, model.serve.port) + .with_context(|| format!("capturing ownership for supervised '{id}'"))?; + state.procs.insert(id.to_string(), process.clone()); + state.save(root)?; + + if !governor::wait_healthy(model, timeout) { + let _ = proc::stop_owned(&process); + state.procs.remove(id); + state.last_error = Some(format!( + "supervised model '{id}' did not report exact health within {} seconds", + timeout.as_secs() + )); + state.save(root)?; + bail!( + "supervised model '{id}' did not report exact health within {} seconds", + timeout.as_secs() + ); + } + + let unhealthy: Vec = scene + .models + .iter() + .filter(|candidate| !governor::is_running(®istry.models[*candidate])) + .cloned() + .collect(); + if unhealthy.is_empty() { + state.active_scene = Some(desired); + state.last_error = None; + } else { + state.active_scene = None; + state.last_error = Some(format!( + "supervised model '{id}' recovered, but desired Scene is still unhealthy: {}", + unhealthy.join(", ") + )); + } + state.save(root)?; + Ok(true) +} + +pub fn deactivate(root: &Path) -> Result<()> { + let _lock = TransitionLock::acquire(root)?; + let mut state = State::load_checked(root)?; + let ids: Vec = state.procs.keys().cloned().collect(); + for id in ids { + let process = state.procs[&id].clone(); + if process.owned_alive() { + println!(" stopping {id} (pid {})", process.pid); + proc::stop_owned(&process)?; + } + state.procs.remove(&id); + state.save(root)?; + } + state.desired_scene = None; + state.active_scene = None; + state.transition_scene = None; + state.last_error = None; + state.save(root)?; + println!("all Compute-managed models stopped; no Scene is desired"); + Ok(()) +} + +/// Diff the target Scene against what is actually serving. Pure: it decides +/// *what* would change, never *whether* the change is allowed. +fn plan_transition(registry: &Registry, scene: &Scene) -> Plan { + let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); + let running = governor::running_ids(registry); + let stop: Vec = running + .iter() + .filter(|id| !target.contains(id.as_str())) + .cloned() + .collect(); + let mut start: Vec = scene + .models + .iter() + .filter(|id| !governor::is_running(®istry.models[*id])) + .cloned() + .collect(); + let listed = scene + .activation + .as_ref() + .and_then(|activation| activation.order.as_deref()) + .map(|order| order == "listed") + .unwrap_or(false); + if !listed { + start.sort_by(|a, b| { + registry.models[a] + .footprint_gb + .partial_cmp(®istry.models[b].footprint_gb) + .unwrap() + }); + } + Plan { + scene: scene.metadata.name.clone(), + budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB), + stop, + start, + } +} + +/// Everything that must hold before the first process is signalled. Runs to +/// completion with nothing mutated, so a rejection here leaves the currently +/// active Scene exactly as it was. +fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> { + for id in &scene.models { + if governor::is_running(®istry.models[id]) { + let process = state.procs.get(id).with_context(|| { + format!( + "'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first" + ) + })?; + if !process.owned_alive() { + bail!( + "ownership record for serving model '{id}' is stale; adopt the active Scene first" + ); + } + } + } + + for id in &plan.stop { + let process = state.procs.get(id).with_context(|| { + format!( + "'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching" + ) + })?; + if !process.owned_alive() { + bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid"); + } + } + for id in &plan.start { + let model = ®istry.models[id]; + if let Some(port) = model.serve.port { + // A model in `plan.stop` may currently hold this port. Every stop runs before + // any start, so that is a handoff, not a conflict. Without this exemption every + // same-port swap is rejected — including brain -> brain-laguna/brain-gemma, + // which share :8001 by design because the alias downstream agents call must + // survive a weight swap. + let freed_by_stop = plan.stop.iter().any(|stopping| { + registry + .models + .get(stopping) + .and_then(|stopped| stopped.serve.port) + == Some(port) + }); + if !freed_by_stop && governor::port_open(port) && !governor::is_running(model) { + bail!("port {port} is occupied by a different model; refusing to start '{id}'"); + } + } + } + Ok(()) +} + +/// `scene activate --dry-run` as data rather than as printed lines: the same +/// validation, the same plan, nothing written. Callers that need the plan +/// programmatically use this instead of scraping stdout. +pub fn plan(root: &Path, name: &str) -> Result { + let _lock = TransitionLock::acquire(root)?; + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + let plan = plan_transition(®istry, &scene); + let mut state = State::load_checked(root)?; + state.procs.retain(|_, process| process.owned_alive()); + preflight(®istry, &scene, &plan, &state)?; + Ok(plan) +} + +fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> { + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + validate_scene(®istry, &scene)?; + let plan = plan_transition(®istry, &scene); + + println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb); + println!(" stop : {}", format_ids(&plan.stop)); + println!(" start: {}", format_ids(&plan.start)); + + let mut state = State::load_checked(root)?; + state.procs.retain(|_, process| process.owned_alive()); + preflight(®istry, &scene, &plan, &state)?; + + if dry_run { + println!(" dry run: validation passed; nothing changed"); + return Ok(()); + } + + state.transition_scene = Some(name.to_string()); + state.save(root)?; + + for id in &plan.stop { + let process = state.procs[id].clone(); + println!(" stopping {id} (pid {})", process.pid); + proc::stop_owned(&process)?; + state.procs.remove(id); + state.save(root)?; + } + + let mut committed: f64 = scene + .models + .iter() + .filter(|id| governor::is_running(®istry.models[*id])) + .map(|id| registry.models[id].footprint_gb) + .sum(); + let wait_healthy = scene + .activation + .as_ref() + .and_then(|activation| activation.wait_healthy) + .unwrap_or(true); + + for id in &plan.start { + let model: &Model = ®istry.models[id]; + // Re-read the pool before every start rather than once per activation: each model + // that comes up consumes real memory, and its true appetite is only knowable after + // it has allocated. A footprint that was optimistic shows up here, on the next + // model, instead of taking the box down. + let available_gb = mem::read().ok().map(|m| m.available_gb); + if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) { + match available_gb { + Some(available) => bail!( + "'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \ + {:.1} GB actually available, {:.0} GB margin required", + model.footprint_gb, + committed, + plan.budget_gb, + available, + governor::SAFETY_MARGIN_GB + ), + None => bail!( + "'{id}' would exceed the {:.0} GB Scene budget with the safety margin", + plan.budget_gb + ), + } + } + let pid = proc::spawn(root, id, model)?; + state.seq += 1; + let process = Proc::capture(pid, state.seq, model.serve.port) + .with_context(|| format!("capturing ownership for newly started '{id}'"))?; + state.procs.insert(id.clone(), process.clone()); + state.save(root)?; + committed += model.footprint_gb; + println!(" started {id} (pid {pid}); waiting for exact health"); + if wait_healthy && !governor::wait_healthy(model, health_timeout()) { + let _ = proc::stop_owned(&process); + state.procs.remove(id); + state.save(root)?; + bail!("'{id}' did not report its exact health marker before timeout"); + } + } + + let unhealthy: Vec = scene + .models + .iter() + .filter(|id| !governor::is_running(®istry.models[*id])) + .cloned() + .collect(); + if !unhealthy.is_empty() { + bail!( + "Scene '{name}' is incomplete; exact health failed for: {}", + unhealthy.join(", ") + ); + } + Ok(()) +} + +fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> { + let missing: Vec<&String> = scene + .models + .iter() + .filter(|id| !registry.models.contains_key(*id)) + .collect(); + if !missing.is_empty() { + bail!( + "Scene '{}' references unknown model id(s): {}", + scene.metadata.name, + missing + .iter() + .map(|id| id.as_str()) + .collect::>() + .join(", ") + ); + } + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + let footprint: f64 = scene + .models + .iter() + .map(|id| registry.models[id].footprint_gb) + .sum(); + if footprint + governor::SAFETY_MARGIN_GB > budget { + bail!( + "Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget", + scene.metadata.name, + footprint + governor::SAFETY_MARGIN_GB, + budget + ); + } + Ok(()) +} + +fn health_timeout() -> Duration { + Duration::from_secs(if cfg!(test) { 5 } else { 900 }) +} + +fn format_ids(ids: &[String]) -> String { + if ids.is_empty() { + "(none)".to_string() + } else { + ids.join(", ") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn fixture_root() -> (PathBuf, u16) { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}")); + fs::create_dir_all(root.join("registry")).unwrap(); + fs::create_dir_all(root.join("scenes")).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let bad_port = bad_listener.local_addr().unwrap().port(); + drop(bad_listener); + let registry = format!( + "apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n supervision:\n restart: always\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n" + ); + fs::write(root.join("registry/models.yaml"), registry).unwrap(); + fs::write( + root.join("scenes/test.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n", + ) + .unwrap(); + (root, port) + } + + #[test] + fn activation_persists_and_deactivation_stops_owned_process() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let state = State::load_checked(&root).unwrap(); + assert_eq!(state.desired_scene.as_deref(), Some("test")); + assert_eq!(state.active_scene.as_deref(), Some("test")); + assert!(state.procs["fake"].owned_alive()); + deactivate(&root).unwrap(); + assert!(State::load_checked(&root).unwrap().procs.is_empty()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn supervision_restarts_only_the_failed_owned_model() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let before = State::load_checked(&root).unwrap(); + let old = before.procs["fake"].clone(); + proc::stop_owned(&old).unwrap(); + + assert!(restart_supervised_model(&root, "fake", Duration::from_secs(5)).unwrap()); + let after = State::load_checked(&root).unwrap(); + assert_ne!(after.procs["fake"].pid, old.pid); + assert!(after.procs["fake"].owned_alive()); + assert_eq!(after.desired_scene.as_deref(), Some("test")); + assert_eq!(after.active_scene.as_deref(), Some("test")); + + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn supervision_refuses_to_replace_an_unowned_process() { + let (root, port) = fixture_root(); + let registry = Registry::load(&root).unwrap(); + let model = ®istry.models["fake"]; + let pid = proc::spawn(&root, "fake", model).unwrap(); + assert!(governor::wait_healthy(model, Duration::from_secs(5))); + + // Make exact health fail while the unowned server continues to hold the + // port. Supervision must not infer ownership from a port number. + let registry_path = root.join("registry/models.yaml"); + let contents = fs::read_to_string(®istry_path).unwrap(); + fs::write( + ®istry_path, + contents.replace( + &format!("health: http://localhost:{port}/"), + &format!( + "health: http://localhost:{port}/\n health_contains: marker-that-is-not-served" + ), + ), + ) + .unwrap(); + fs::create_dir_all(root.join(".compute")).unwrap(); + State { + desired_scene: Some("test".to_string()), + active_scene: Some("test".to_string()), + last_known_good_scene: Some("test".to_string()), + ..State::default() + } + .save(&root) + .unwrap(); + + let error = restart_supervised_model(&root, "fake", Duration::from_millis(50)) + .unwrap_err() + .to_string(); + assert!(error.contains("unowned process")); + assert!(governor::port_open(port)); + + let owned = Proc::capture(pid, 1, Some(port)).unwrap(); + proc::stop_owned(&owned).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + /// Two models sharing one port, distinguishable by `health_contains` — the + /// brain/brain-laguna shape. Each serves its own directory so the health probe + /// can tell which one is actually up. + fn same_port_root() -> PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("lumbridge-compute-swap-{unique}")); + fs::create_dir_all(root.join("registry")).unwrap(); + fs::create_dir_all(root.join("scenes")).unwrap(); + let dir_a = root.join("srv-alpha"); + let dir_b = root.join("srv-beta"); + fs::create_dir_all(&dir_a).unwrap(); + fs::create_dir_all(&dir_b).unwrap(); + fs::write(dir_a.join("alpha-marker.txt"), "a").unwrap(); + fs::write(dir_b.join("beta-marker.txt"), "b").unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let (a, b) = (dir_a.display(), dir_b.display()); + let registry = format!( + "apiVersion: lumbridge/v1\nmodels:\n alpha:\n name: Alpha\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: alpha-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{a}']\n beta:\n name: Beta\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: beta-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{b}']\n" + ); + fs::write(root.join("registry/models.yaml"), registry).unwrap(); + fs::write( + root.join("scenes/a.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: a\n version: 1\nmodels: [alpha]\nbudget_gb: 100\n", + ) + .unwrap(); + fs::write( + root.join("scenes/b.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: b\n version: 1\nmodels: [beta]\nbudget_gb: 100\n", + ) + .unwrap(); + root + } + + #[test] + fn same_port_swap_is_allowed_when_the_occupant_is_being_stopped() { + let root = same_port_root(); + activate(&root, "a", false).unwrap(); + assert!(State::load_checked(&root) + .unwrap() + .procs + .contains_key("alpha")); + + // Regression: this used to fail with "port N is occupied by a different model; + // refusing to start 'beta'". The pre-flight port check ran over `plan.start` + // without exempting ports released by `plan.stop`, so every same-port swap was + // rejected even though stops precede starts. + activate(&root, "b", false).unwrap(); + + let state = State::load_checked(&root).unwrap(); + assert_eq!(state.active_scene.as_deref(), Some("b")); + assert!(state.procs.contains_key("beta")); + assert!(!state.procs.contains_key("alpha")); + + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn already_serving_scene_requires_explicit_adoption() { + let (root, port) = fixture_root(); + let registry = Registry::load(&root).unwrap(); + let model = ®istry.models["fake"]; + let pid = proc::spawn(&root, "fake", model).unwrap(); + assert!(governor::wait_healthy(model, Duration::from_secs(5))); + let error = activate(&root, "test", true).unwrap_err().to_string(); + assert!(error.contains("adopt the active Scene")); + let owned = Proc::capture(pid, 1, Some(port)).unwrap(); + proc::stop_owned(&owned).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn preflight_rejection_preserves_healthy_active_scene() { + let (root, _port) = fixture_root(); + activate(&root, "test", false).unwrap(); + let before = State::load_checked(&root).unwrap(); + let pid = before.procs["fake"].pid; + fs::write( + root.join("scenes/invalid.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n", + ) + .unwrap(); + assert!(activate(&root, "invalid", false).is_err()); + let after = State::load_checked(&root).unwrap(); + assert_eq!(after.active_scene.as_deref(), Some("test")); + assert_eq!(after.desired_scene.as_deref(), Some("test")); + assert_eq!(after.procs["fake"].pid, pid); + assert!(after.procs["fake"].owned_alive()); + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn failed_activation_keeps_healthy_partial_scene_without_rollback_target() { + let (root, port) = fixture_root(); + fs::write( + root.join("scenes/failing.scene.yaml"), + "apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n", + ) + .unwrap(); + assert!(activate(&root, "failing", false).is_err()); + let state = State::load_checked(&root).unwrap(); + assert!(state.procs.contains_key("fake")); + assert!(!state.procs.contains_key("bad")); + assert!(state.active_scene.is_none()); + assert!(governor::port_open(port)); + deactivate(&root).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn checked_state_load_rejects_malformed_yaml() { + let (root, _port) = fixture_root(); + fs::create_dir_all(root.join(".compute")).unwrap(); + fs::write(root.join(".compute/state.yaml"), "desired_scene: [").unwrap(); + assert!(State::load_checked(&root).is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn state_write_replaces_complete_yaml_atomically() { + let (root, _port) = fixture_root(); + let state = State { + desired_scene: Some("test".to_string()), + ..State::default() + }; + state.save(&root).unwrap(); + let loaded = State::load_checked(&root).unwrap(); + assert_eq!(loaded.desired_scene.as_deref(), Some("test")); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..fbc6806 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,561 @@ +//! Lumbridge Compute — safe AI workload orchestration for accelerator nodes. + +mod agent; +mod config; +mod eval; +mod gateway; +mod governor; +mod http; +mod lifecycle; +mod mcp; +mod mem; +mod proc; +mod telemetry; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use config::{find_scene, load_scenes, Registry, Scene}; +use proc::State; + +#[derive(Parser)] +#[command( + name = "lumbridge-compute", + version, + about = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes." +)] +struct Cli { + /// Root dir containing registry/ and scenes/ (default: $LUMBRIDGE_COMPUTE_ROOT or current dir) + #[arg(long, global = true)] + root: Option, + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Governor status: memory, budget, running set, headroom + Status, + /// Manage the model registry + Model { + #[command(subcommand)] + cmd: ModelCmd, + }, + /// Manage scenes (bundles of models) + Scene { + #[command(subcommand)] + cmd: SceneCmd, + }, + /// Run reproducible capability and serving-performance evaluations + Eval { + #[command(subcommand)] + cmd: EvalCmd, + }, + /// Inspect durable, privacy-safe model usage and concurrency history + Usage { + #[command(subcommand)] + cmd: UsageCmd, + }, + /// Run the memory watchdog in the foreground (kills the newest model before OOM-wedge) + Watchdog { + /// Kill the newest model if MemAvailable dips below this many GB + #[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)] + floor: f64, + }, + /// Run the stable streaming gateway in the foreground + Gateway { + #[arg(long, default_value = "127.0.0.1:8011")] + listen: String, + #[arg(long, default_value = "127.0.0.1:8001")] + upstream: String, + }, + /// Serve the Governor as a read-only JSON API over HTTP + /// + /// A second transport over the same operations the MCP server exposes, for a web UI. It + /// never mutates: no activation, no stop, no eval run. + Api { + /// Loopback by default. This API has no authentication unless --token is set, and it + /// reveals which models this node runs, so widening the bind is an explicit act. + #[arg(long, default_value = "127.0.0.1:8012")] + listen: String, + /// Browser origin permitted to read this API. Repeatable. Empty means no browser may + /// read it; `*` is deliberately not supported, because any page you visit would then + /// be able to fingerprint this machine. + #[arg(long = "allow-origin")] + allow_origin: Vec, + /// Require `Authorization: Bearer ` on every request. + #[arg(long)] + token: Option, + }, + /// Serve the Governor to AI agents as MCP tools over stdio + Mcp { + /// Let an agent perform a real Scene transition, not just plan one. + /// Off by default: the agent writes its own tool arguments, so the only + /// meaningful gate on a production switch is one the operator sets here. + #[arg(long)] + allow_activate: bool, + }, + /// Run the resident supervisor: resume desired Scene, gateway, and memory floor + Agent { + #[arg(long, default_value = "127.0.0.1:8011")] + listen: String, + #[arg(long, default_value = "127.0.0.1:8001")] + upstream: String, + #[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)] + floor: f64, + /// Local runtime Prometheus endpoint. SGLang must use --enable-metrics. + #[arg(long, default_value = "http://127.0.0.1:8001/metrics")] + metrics_url: String, + /// Registry model id for metrics, or auto to resolve the active Scene by port. + #[arg(long, default_value = "auto")] + metrics_model: String, + /// Sampling interval for time-weighted C0-C4 and queue history. + #[arg(long, default_value_t = 5)] + telemetry_interval_sec: u64, + /// Disable resident metrics collection while keeping supervision and the gateway. + #[arg(long)] + no_telemetry: bool, + }, +} + +#[derive(Subcommand)] +enum ModelCmd { + /// List registered models and their live state + Ls, +} + +#[derive(Subcommand)] +enum SceneCmd { + /// List scenes with total footprint + Ls, + /// Show a scene: models, footprints, and the Governor's admission verdict + Show { name: String }, + /// Activate a scene: stop what's not in it, admit + start what is + Activate { + name: String, + /// Print the plan without changing anything + #[arg(long)] + dry_run: bool, + }, + /// Adopt an already-running exact Scene and bind legacy PIDs to process identities + Adopt { name: String }, + /// Resume the persisted desired Scene, falling back to the previous known-good Scene + Resume, + /// Stop all Lumbridge Compute-managed models + Deactivate, +} + +#[derive(Subcommand)] +enum EvalCmd { + /// List available evaluation suites + Ls, + /// Run a suite against an OpenAI-compatible endpoint + Run { + suite: String, + #[arg(long, default_value = "http://127.0.0.1:8001/v1")] + base_url: String, + #[arg(long, default_value = "brain")] + model: String, + /// Override the suite's repetitions per case + #[arg(long)] + repeat: Option, + }, +} + +#[derive(Subcommand)] +enum UsageCmd { + /// Scrape the runtime once and append a durable snapshot + Collect { + #[arg(long, default_value = "http://127.0.0.1:8001/metrics")] + metrics_url: String, + #[arg(long, default_value = "auto")] + model: String, + }, + /// Report calls, tokens, vision, concurrency, queueing, and agent labels + Summary { + /// Window ending now, such as 24h, 7d, or 4w. + #[arg(long, default_value = "24h")] + since: String, + #[arg(long)] + json: bool, + }, + /// Report bounded client/agent/workload usage labels + Agents { + #[arg(long, default_value = "7d")] + since: String, + #[arg(long)] + json: bool, + }, + /// Report the time-sampled C0-C4+ distribution and queue pressure + Concurrency { + #[arg(long, default_value = "7d")] + since: String, + #[arg(long)] + json: bool, + }, +} + +fn root_dir(cli: &Cli) -> PathBuf { + cli.root + .clone() + .or_else(|| { + std::env::var("LUMBRIDGE_COMPUTE_ROOT") + .ok() + .map(PathBuf::from) + }) + .unwrap_or_else(|| PathBuf::from(".")) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let root = root_dir(&cli); + match &cli.cmd { + Cmd::Status => cmd_status(&root), + Cmd::Model { cmd } => match cmd { + ModelCmd::Ls => cmd_model_ls(&root), + }, + Cmd::Scene { cmd } => match cmd { + SceneCmd::Ls => cmd_scene_ls(&root), + SceneCmd::Show { name } => cmd_scene_show(&root, name), + SceneCmd::Activate { name, dry_run } => lifecycle::activate(&root, name, *dry_run), + SceneCmd::Adopt { name } => lifecycle::adopt(&root, name), + SceneCmd::Resume => lifecycle::resume(&root), + SceneCmd::Deactivate => lifecycle::deactivate(&root), + }, + Cmd::Eval { cmd } => match cmd { + EvalCmd::Ls => eval::list(&root), + EvalCmd::Run { + suite, + base_url, + model, + repeat, + } => eval::run(&root, suite, base_url, model, *repeat), + }, + Cmd::Usage { cmd } => cmd_usage(&root, cmd), + Cmd::Watchdog { floor } => cmd_watchdog(&root, *floor), + Cmd::Gateway { listen, upstream } => gateway::run(listen, upstream), + Cmd::Api { + listen, + allow_origin, + token, + } => http::run( + listen, + http::Config { + root: root.clone(), + allowed_origins: allow_origin.clone(), + token: token.clone(), + }, + ), + Cmd::Mcp { allow_activate } => mcp::run(&root, *allow_activate), + Cmd::Agent { + listen, + upstream, + floor, + metrics_url, + metrics_model, + telemetry_interval_sec, + no_telemetry, + } => agent::run( + &root, + listen, + upstream, + *floor, + if *no_telemetry { + None + } else { + Some(agent::TelemetryConfig { + metrics_url: metrics_url.clone(), + model: metrics_model.clone(), + interval: Duration::from_secs((*telemetry_interval_sec).max(1)), + }) + }, + ), + } +} + +fn cmd_usage(root: &Path, command: &UsageCmd) -> Result<()> { + match command { + UsageCmd::Collect { metrics_url, model } => { + let result = telemetry::collect_once(root, metrics_url, model)?; + println!("{}", serde_json::to_string_pretty(&result)?); + } + UsageCmd::Summary { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + println!("{}", telemetry::render_text(&report)); + } + } + UsageCmd::Agents { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report.agents)?); + } else if report.agents.is_empty() { + println!("no agent-labelled usage in this window"); + } else { + for row in report.agents { + println!( + "{}/{}/{} {:.0} requests · {:.0} prompt · {:.0} generated tokens", + row.client, + row.agent, + row.workload, + row.requests, + row.prompt_tokens, + row.generation_tokens + ); + } + } + } + UsageCmd::Concurrency { since, json } => { + let report = telemetry::report(root, since)?; + if *json { + println!("{}", serde_json::to_string_pretty(&report.concurrency)?); + } else if report.concurrency.samples == 0 { + println!("no concurrency samples in this window"); + } else { + for bucket in report.concurrency.distribution { + println!( + "{} {} samples · {:.1}%", + bucket.concurrency, + bucket.samples, + bucket.share * 100.0 + ); + } + println!( + "peak running {} · peak queued {} · queue observed in {} sample(s)", + report.concurrency.peak_running, + report.concurrency.peak_queued, + report.concurrency.queued_samples + ); + } + } + } + Ok(()) +} + +fn cmd_status(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + let m = mem::read()?; + let committed = governor::committed_gb(®); + let running = governor::running_ids(®); + let budget = governor::DEFAULT_BUDGET_GB; + // The binding constraint, not the generous one: admission enforces the declared + // budget AND observed memory, so reporting only the declared headroom would promise + // room the next `scene activate` is going to refuse. + let headroom = governor::headroom_gb(committed, budget, Some(m.available_gb)).max(0.0); + let declared_headroom = (budget - committed - governor::SAFETY_MARGIN_GB).max(0.0); + let managed = State::load_checked(root)?; + + println!("Lumbridge Compute · governor"); + println!( + " memory {:.1} GB total · {:.1} GB available", + m.total_gb, m.available_gb + ); + println!( + " budget {:.0} GB (safety margin {:.0} · watchdog floor {:.0})", + budget, + governor::SAFETY_MARGIN_GB, + governor::WATCHDOG_FLOOR_GB + ); + println!( + " committed {committed:.1} GB across {} model(s)", + running.len() + ); + if headroom < declared_headroom { + println!( + " headroom {headroom:.1} GB admittable (budget allows {declared_headroom:.1}; \ + MemAvailable is the tighter limit)" + ); + } else { + println!(" headroom {headroom:.1} GB admittable"); + } + println!(); + println!( + " scene active={} desired={} fallback={}", + managed.active_scene.as_deref().unwrap_or("-"), + managed.desired_scene.as_deref().unwrap_or("-"), + managed.last_known_good_scene.as_deref().unwrap_or("-") + ); + if let Some(error) = &managed.last_error { + println!(" last error {error}"); + } + + if running.is_empty() { + println!(" (no registered models currently serving)"); + } else { + println!(" running:"); + for id in &running { + let mdl = ®.models[id]; + let tag = if managed + .procs + .get(id) + .is_some_and(|process| process.owned_alive()) + { + "compute" + } else { + "ext " + }; + println!( + " ● [{tag}] {:<12} {:>5.0} GB :{:<5} {}", + id, + mdl.footprint_gb, + port_str(mdl.serve.port), + mdl.name + ); + } + } + Ok(()) +} + +fn cmd_model_ls(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + println!( + "{:<12} {:>6} {:<6} {:<6} MODEL", + "ID", "GB", "STATE", "PORT" + ); + for (id, m) in ®.models { + println!( + "{:<12} {:>6.0} {:<6} {:<6} {}", + id, + m.footprint_gb, + if governor::is_running(m) { + "up" + } else { + "down" + }, + port_str(m.serve.port), + m.name + ); + } + Ok(()) +} + +fn cmd_scene_ls(root: &Path) -> Result<()> { + let reg = Registry::load(root)?; + let scenes = load_scenes(root)?; + if scenes.is_empty() { + println!("no scenes in {}/scenes", root.display()); + return Ok(()); + } + println!("{:<12} {:>6} {:<30} MODELS", "SCENE", "GB", "DESCRIPTION"); + for s in &scenes { + let total = scene_footprint(s, ®); + println!( + "{:<12} {:>6.0} {:<30} {}", + s.metadata.name, + total, + truncate(&s.metadata.description, 30), + s.models.join(", ") + ); + } + Ok(()) +} + +fn cmd_scene_show(root: &Path, name: &str) -> Result<()> { + let reg = Registry::load(root)?; + let scene = find_scene(root, name)?; + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + + println!( + "scene {} (v{})", + scene.metadata.name, scene.metadata.version + ); + if !scene.metadata.description.is_empty() { + println!(" {}", scene.metadata.description); + } + println!(" budget {budget:.0} GB\n models:"); + + let mut total = 0.0; + let mut missing = Vec::new(); + for id in &scene.models { + match reg.models.get(id) { + Some(m) => { + total += m.footprint_gb; + let state = if governor::is_running(m) { + "up" + } else { + "down" + }; + println!( + " {:<12} {:>5.0} GB [{:<4}] {}", + id, m.footprint_gb, state, m.name + ); + } + None => { + missing.push(id.clone()); + println!(" {id:<12} ? [MISSING from registry]"); + } + } + } + + let needed = total + governor::SAFETY_MARGIN_GB; + println!( + "\n total footprint {total:.1} GB (+{:.0} safety = {needed:.1} GB)", + governor::SAFETY_MARGIN_GB + ); + let verdict = if !missing.is_empty() { + format!("✗ {} model(s) missing from registry", missing.len()) + } else if needed <= budget { + format!("✓ fits — {:.1} GB to spare", budget - needed) + } else { + format!("✗ exceeds budget by {:.1} GB", needed - budget) + }; + println!(" admission {verdict}"); + Ok(()) +} + +fn cmd_watchdog(root: &Path, floor: f64) -> Result<()> { + println!( + "Lumbridge Compute watchdog — killing the newest model if MemAvailable < {floor:.1} GB. Ctrl-C to stop." + ); + loop { + let m = mem::read()?; + if m.available_gb < floor { + let mut state = State::load_checked(root)?; + match state.newest_alive() { + Some((id, p)) => { + eprintln!( + "watchdog: MemAvailable {:.1} GB < floor {:.1} — killing newest '{id}' (pid {})", + m.available_gb, floor, p.pid + ); + proc::stop_owned(&p)?; + state.procs.remove(&id); + state.save(root)?; + } + None => eprintln!( + "watchdog: MemAvailable {:.1} GB < floor {:.1} but no Lumbridge Compute-managed model to kill!", + m.available_gb, floor + ), + } + } + std::thread::sleep(Duration::from_secs(1)); + } +} + +// ---- helpers --------------------------------------------------------------- + +fn scene_footprint(s: &Scene, reg: &Registry) -> f64 { + s.models + .iter() + .filter_map(|id| reg.models.get(id)) + .map(|m| m.footprint_gb) + .sum() +} + +fn port_str(p: Option) -> String { + p.map(|p| p.to_string()).unwrap_or_else(|| "-".to_string()) +} + +fn truncate(s: &str, n: usize) -> String { + if s.chars().count() <= n { + s.to_string() + } else { + format!( + "{}…", + s.chars().take(n.saturating_sub(1)).collect::() + ) + } +} diff --git a/src/mcp.rs b/src/mcp.rs new file mode 100644 index 0000000..ba7bd74 --- /dev/null +++ b/src/mcp.rs @@ -0,0 +1,617 @@ +//! MCP server: the Governor's surface, exposed to agents as tools. +//! +//! Agents already drive Compute by shelling out to the CLI and scraping the +//! column-aligned output. That works until a column moves. Speaking MCP means +//! the agent gets the same numbers the Governor reasons about, as data, with a +//! schema attached — and it means the *shape* of what an agent may do becomes +//! something this file decides rather than something `bash` decides. +//! +//! Two deliberate constraints shape everything below. +//! +//! **Read-first.** Every tool here except `scene_activate` is a pure read. +//! Activating a Scene stops every registered model that is not in it, which on +//! this box means taking down whatever is currently serving — a live voice +//! pipeline included. There is no undo an agent can reach for: if the target +//! Scene then fails to come up, recovery depends on rollback that may itself +//! fail. That asymmetry is why `scene adopt`, `scene resume`, and +//! `scene deactivate` are absent entirely; they are operator verbs whose +//! correctness depends on knowing what the box was doing five minutes ago. +//! `scene_activate` is exposed because planning a switch is genuinely the +//! useful thing an agent wants, and it defaults to planning only. +//! +//! **The safety boundary is the operator's, not the agent's.** A `dry_run` +//! argument defaulting to `true` documents intent but guards nothing: the agent +//! writes the arguments, so it can write `false`. The only boundary an agent +//! cannot cross is one set before it connects, so a real transition also +//! requires `lumbridge-compute mcp --allow-activate`, chosen by the human who +//! launched the server. Without that flag `dry_run: false` is refused, and the +//! refusal says so rather than silently planning instead. + +use anyhow::{Context, Result}; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::{Implementation, ServerCapabilities, ServerInfo}; +use rmcp::{tool, tool_handler, tool_router, ErrorData, Json, ServerHandler, ServiceExt}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::os::fd::FromRawFd; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::config::{find_scene, load_scenes, Registry}; +use crate::proc::State; +use crate::{eval, governor, lifecycle, mem}; + +pub fn run(root: &Path, allow_activate: bool) -> Result<()> { + let transport_stdout = hand_over_stdout()?; + // The banner has to go to stderr for the same reason everything else does; + // it doubles as confirmation to the operator that the mutating tool is off. + eprintln!( + "Lumbridge Compute MCP server on stdio · root {} · scene activation {}", + root.display(), + if allow_activate { + "ENABLED (--allow-activate)" + } else { + "disabled; plan only" + } + ); + + let server = ComputeMcp::new(root.to_path_buf(), allow_activate); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("building the MCP runtime")?; + runtime.block_on(async move { + let transport = ( + tokio::io::stdin(), + tokio::fs::File::from_std(transport_stdout), + ); + let service = server + .serve(transport) + .await + .context("negotiating the MCP stdio session")?; + service.waiting().await.context("serving MCP over stdio")?; + Ok(()) + }) +} + +/// Hand the real stdout to the transport and point fd 1 at stderr. +/// +/// On stdio transport fd 1 *is* the JSON-RPC framing, and this crate reports +/// progress with `println!` throughout — `lifecycle::activate` narrates every +/// stop and start. One such line interleaved into the framing desynchronises +/// the client mid-transition, which is the worst possible moment to lose it. +/// Rather than audit every print (and every future one), move the file +/// descriptor: library output lands on stderr, where operators already read +/// this server's logs, and the protocol gets a channel nothing else can write. +fn hand_over_stdout() -> Result { + let saved = unsafe { libc::dup(libc::STDOUT_FILENO) }; + if saved < 0 { + return Err(std::io::Error::last_os_error()) + .context("duplicating stdout for the MCP transport"); + } + if unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) } < 0 { + let error = std::io::Error::last_os_error(); + unsafe { libc::close(saved) }; + return Err(error).context("redirecting stdout to stderr"); + } + // SAFETY: `saved` is a fresh descriptor from `dup` that nothing else owns. + Ok(unsafe { std::fs::File::from_raw_fd(saved) }) +} + +#[derive(Clone)] +struct ComputeMcp { + root: Arc, + allow_activate: bool, + tool_router: ToolRouter, +} + +#[tool_router] +impl ComputeMcp { + fn new(root: PathBuf, allow_activate: bool) -> Self { + Self { + root: Arc::new(root), + allow_activate, + tool_router: Self::tool_router(), + } + } + + /// Governor status for this node: unified-memory totals, the hard budget + /// and its margins, how much is committed by models that are actually + /// serving, how much is still admittable, the persisted Scene state, and + /// the last transition or watchdog error. Read this before reasoning about + /// whether anything else will fit. + #[tool( + name = "governor_status", + annotations( + title = "Governor status", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn governor_status(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_status(&root)).await.map(Json) + } + + /// List every model in the node's registry with its declared worst-case + /// footprint, serving port, and whether it is currently serving. Footprints + /// are what admission control is decided against, not observed usage. + #[tool( + name = "model_list", + annotations( + title = "List registered models", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn model_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_models(&root)).await.map(Json) + } + + /// List the Scenes this node can activate, with each one's total footprint + /// and member models. A Scene is a named set of models brought up as a + /// single unit. + #[tool( + name = "scene_list", + annotations(title = "List scenes", read_only_hint = true, open_world_hint = false) + )] + async fn scene_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_scenes(&root)).await.map(Json) + } + + /// Show one Scene: its models, their footprints and live state, and the + /// Governor's admission verdict — whether the Scene fits its budget once + /// the safety margin is added, and by how much it fits or misses. + #[tool( + name = "scene_show", + annotations( + title = "Show a scene and its admission verdict", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn scene_show( + &self, + Parameters(params): Parameters, + ) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_scene(&root, ¶ms.name)) + .await + .map(Json) + } + + /// List the evaluation suites available on this node, with their versions + /// and case counts. Running a suite is deliberately not exposed: it drives + /// real load against a serving model for an unbounded time. + #[tool( + name = "eval_list", + annotations( + title = "List eval suites", + read_only_hint = true, + open_world_hint = false + ) + )] + async fn eval_list(&self) -> Result, ErrorData> { + let root = self.root.clone(); + offload(move || collect_evals(&root)).await.map(Json) + } + + /// Plan — or, if the operator allowed it, perform — a Scene transition. + /// + /// With `dry_run` true (the default) this runs every check that guards a + /// real transition and returns the models it would stop and start, changing + /// nothing. With `dry_run` false it actually switches the node: models + /// outside the Scene are stopped, which will interrupt anything they serve, + /// and the call blocks until every Scene model reports healthy or the + /// transition rolls back. A real transition additionally requires the + /// server to have been started with `--allow-activate`; without it, + /// `dry_run: false` is refused. Ask a human before setting it. + #[tool( + name = "scene_activate", + annotations( + title = "Activate a scene (plans by default)", + read_only_hint = false, + destructive_hint = true, + idempotent_hint = false, + open_world_hint = false + ) + )] + async fn scene_activate( + &self, + Parameters(params): Parameters, + ) -> Result, ErrorData> { + if !params.dry_run && !self.allow_activate { + return Err(ErrorData::invalid_params( + "this MCP server is running plan-only: a real Scene transition needs an \ + operator to restart it as `lumbridge-compute mcp --allow-activate`. \ + Re-run with dry_run: true to see the plan.", + None, + )); + } + let root = self.root.clone(); + offload(move || activate_scene(&root, ¶ms.name, params.dry_run)) + .await + .map(Json) + } +} + +#[tool_handler(router = self.tool_router)] +impl ServerHandler for ComputeMcp { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info(Implementation::new( + "lumbridge-compute", + env!("CARGO_PKG_VERSION"), + )) + .with_instructions( + "Lumbridge Compute runs many AI models on one unified-memory box. There is a \ + single memory pool shared by CPU and GPU, so over-committing does not fail \ + gracefully — the machine thrashes and wedges before the OOM killer acts. The \ + Governor prevents that by admitting a model only if its declared footprint \ + plus a safety margin still fits a hard budget.\n\n\ + Start from `governor_status` for what is committed and what is admittable, \ + and `scene_show` for whether a named Scene would be admitted. Everything is \ + read-only except `scene_activate`, which plans by default. Activating a Scene \ + stops every registered model outside it, so treat it as a production change \ + and get a human's agreement before asking for a non-dry run.", + ) + } +} + +// ---- tool parameters ------------------------------------------------------- + +#[derive(Debug, Deserialize, JsonSchema)] +struct SceneNameParams { + /// Scene name, as reported by `scene_list`. + name: String, +} + +#[derive(Debug, Deserialize, JsonSchema)] +struct SceneActivateParams { + /// Scene name, as reported by `scene_list`. + name: String, + /// Validate and report the plan without changing anything. Defaults to + /// true; set it to false only with a human's explicit agreement. + #[serde(default = "yes")] + dry_run: bool, +} + +fn yes() -> bool { + true +} + +// ---- tool results ---------------------------------------------------------- +// +// Lists are wrapped in objects rather than returned bare: MCP structured +// content must be a JSON object, and a named field leaves room to report +// alongside the list later without breaking a client's schema. + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct StatusReport { + memory_total_gb: f64, + memory_available_gb: f64, + budget_gb: f64, + safety_margin_gb: f64, + watchdog_floor_gb: f64, + /// Sum of the footprints of every registered model currently serving. + committed_gb: f64, + /// What a new model could still claim without breaching the safety margin. + headroom_gb: f64, + /// Scene whose exact model health was last verified. + active_scene: Option, + /// Scene the node should return to after a restart. + desired_scene: Option, + /// Previous proven Scene, used if the desired one cannot resume. + last_known_good_scene: Option, + last_error: Option, + running: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct RunningModel { + id: String, + name: String, + footprint_gb: f64, + port: Option, + /// True when Compute owns this process identity and may signal it. False + /// means the model is serving but was started outside Compute, so a Scene + /// transition will refuse to touch it until it is adopted. + compute_owned: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct ModelList { + models: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct ModelEntry { + id: String, + name: String, + /// Declared worst-case unified memory once serving, not observed usage. + footprint_gb: f64, + port: Option, + running: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct SceneList { + scenes: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct SceneEntry { + name: String, + version: u32, + description: String, + /// Total footprint of the Scene's models that exist in the registry. + footprint_gb: f64, + models: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct SceneDetail { + name: String, + version: u32, + description: String, + budget_gb: f64, + models: Vec, + /// Model ids the Scene references that the registry does not define. + missing_models: Vec, + footprint_gb: f64, + /// Footprint plus the Governor's safety margin — the figure compared + /// against the budget. + required_gb: f64, + /// The Governor's verdict: would this Scene be admitted? + admits: bool, + verdict: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct SceneModel { + id: String, + /// Absent when the id is missing from the registry. + name: Option, + footprint_gb: Option, + running: bool, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub(crate) struct EvalList { + suites: Vec, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct EvalSuite { + name: String, + version: u32, + cases: usize, + description: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +struct ActivationReport { + scene: String, + budget_gb: f64, + /// Serving models outside the Scene, which the transition stops first. + stop: Vec, + /// Scene models not yet serving, in the order they would be admitted. + start: Vec, + /// False when this was a plan and the node was left untouched. + applied: bool, +} + +// ---- collectors ------------------------------------------------------------ +// +// These compose the same primitives the CLI prints from — `governor`, `mem`, +// `config`, `proc::State` — rather than parsing the CLI's output. The numbers +// an agent sees are therefore the numbers admission control uses, by +// construction. + +pub(crate) fn collect_status(root: &Path) -> Result { + let registry = Registry::load(root)?; + let memory = mem::read()?; + let committed = governor::committed_gb(®istry); + let managed = State::load_checked(root)?; + let budget = governor::DEFAULT_BUDGET_GB; + + let running = governor::running_ids(®istry) + .into_iter() + .map(|id| { + let model = ®istry.models[&id]; + RunningModel { + name: model.name.clone(), + footprint_gb: model.footprint_gb, + port: model.serve.port, + compute_owned: managed + .procs + .get(&id) + .is_some_and(|process| process.owned_alive()), + id, + } + }) + .collect(); + + Ok(StatusReport { + memory_total_gb: memory.total_gb, + memory_available_gb: memory.available_gb, + budget_gb: budget, + safety_margin_gb: governor::SAFETY_MARGIN_GB, + watchdog_floor_gb: governor::WATCHDOG_FLOOR_GB, + // `+ 0.0` normalises the negative zero an empty sum can produce, which serialises as + // "-0.0" and reads as a bug to anyone consuming this JSON. + committed_gb: committed + 0.0, + // The BINDING constraint, not the declared one. This used to be + // `budget - committed - margin`, which ignores how much memory the box actually has — + // so on a 62 GB machine with a 100 GB budget it reported 92 GB of headroom that the + // next admission would refuse. cmd_status was fixed when admission started consulting + // MemAvailable; this path was missed, which meant every agent driving the node over + // MCP got the optimistic number. + headroom_gb: governor::headroom_gb(committed, budget, Some(memory.available_gb)).max(0.0), + active_scene: managed.active_scene, + desired_scene: managed.desired_scene, + last_known_good_scene: managed.last_known_good_scene, + last_error: managed.last_error, + running, + }) +} + +pub(crate) fn collect_models(root: &Path) -> Result { + let registry = Registry::load(root)?; + Ok(ModelList { + models: registry + .models + .iter() + .map(|(id, model)| ModelEntry { + id: id.clone(), + name: model.name.clone(), + footprint_gb: model.footprint_gb, + port: model.serve.port, + running: governor::is_running(model), + }) + .collect(), + }) +} + +pub(crate) fn collect_scenes(root: &Path) -> Result { + let registry = Registry::load(root)?; + Ok(SceneList { + scenes: load_scenes(root)? + .into_iter() + .map(|scene| SceneEntry { + name: scene.metadata.name, + version: scene.metadata.version, + description: scene.metadata.description, + footprint_gb: scene + .models + .iter() + .filter_map(|id| registry.models.get(id)) + .map(|model| model.footprint_gb) + .sum(), + models: scene.models, + }) + .collect(), + }) +} + +pub(crate) fn collect_scene(root: &Path, name: &str) -> Result { + let registry = Registry::load(root)?; + let scene = find_scene(root, name)?; + let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); + + let mut footprint = 0.0; + let mut missing = Vec::new(); + let mut models = Vec::new(); + for id in &scene.models { + match registry.models.get(id) { + Some(model) => { + footprint += model.footprint_gb; + models.push(SceneModel { + id: id.clone(), + name: Some(model.name.clone()), + footprint_gb: Some(model.footprint_gb), + running: governor::is_running(model), + }); + } + None => { + missing.push(id.clone()); + models.push(SceneModel { + id: id.clone(), + name: None, + footprint_gb: None, + running: false, + }); + } + } + } + + // Same three-way verdict the CLI prints: a missing id is fatal regardless + // of arithmetic, because the Scene cannot be resolved at all. + let required = footprint + governor::SAFETY_MARGIN_GB; + let (admits, verdict) = if !missing.is_empty() { + ( + false, + format!("{} model(s) missing from the registry", missing.len()), + ) + } else if required <= budget { + ( + true, + format!("fits with {:.1} GB to spare", budget - required), + ) + } else { + ( + false, + format!("exceeds the budget by {:.1} GB", required - budget), + ) + }; + + Ok(SceneDetail { + name: scene.metadata.name, + version: scene.metadata.version, + description: scene.metadata.description, + budget_gb: budget, + models, + missing_models: missing, + footprint_gb: footprint, + required_gb: required, + admits, + verdict, + }) +} + +pub(crate) fn collect_evals(root: &Path) -> Result { + Ok(EvalList { + suites: eval::catalogue(root)? + .into_iter() + .map(|suite| EvalSuite { + name: suite.name, + version: suite.version, + cases: suite.cases, + description: suite.description, + }) + .collect(), + }) +} + +fn activate_scene(root: &Path, name: &str, dry_run: bool) -> Result { + // Plan first either way. On a real transition this is the plan we report; + // `activate` re-derives and re-validates its own under the transition lock, + // so the report describes intent and the lock still owns the truth. + let plan = lifecycle::plan(root, name)?; + if !dry_run { + lifecycle::activate(root, name, false)?; + } + Ok(ActivationReport { + scene: plan.scene, + budget_gb: plan.budget_gb, + stop: plan.stop, + start: plan.start, + applied: !dry_run, + }) +} + +/// Run a synchronous Governor call off the transport's thread. +/// +/// Every read here touches `/proc` and probes serving ports with blocking +/// socket timeouts, and a real transition can sit for minutes waiting on model +/// health. Doing that inline would stall the JSON-RPC reader for the duration, +/// so the synchronous core stays on the blocking pool where it belongs. +async fn offload(work: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(work).await { + Ok(Ok(value)) => Ok(value), + // `{:#}` keeps anyhow's context chain, which is where the actionable + // half of a Compute error lives ("adopt the active Scene first"). + Ok(Err(error)) => Err(ErrorData::internal_error(format!("{error:#}"), None)), + Err(join) => Err(ErrorData::internal_error( + format!("Compute worker task failed: {join}"), + None, + )), + } +} diff --git a/src/mem.rs b/src/mem.rs new file mode 100644 index 0000000..840c646 --- /dev/null +++ b/src/mem.rs @@ -0,0 +1,89 @@ +//! Unified-memory sensing. On a shared-memory box there's one pool, so `MemAvailable` +//! from /proc/meminfo is the single source of truth the Governor guards. + +use anyhow::{Context, Result}; +use std::fs; + +pub struct MemInfo { + pub total_gb: f64, + pub available_gb: f64, +} + +pub fn read() -> Result { + let s = fs::read_to_string("/proc/meminfo").context("reading /proc/meminfo")?; + Ok(parse_meminfo(&s)) +} + +/// Split out from `read` so the parser is testable without a real /proc. +pub fn parse_meminfo(s: &str) -> MemInfo { + let mut total = 0.0; + let mut available = 0.0; + for line in s.lines() { + if let Some(v) = line.strip_prefix("MemTotal:") { + total = kb_to_gb(v); + } else if let Some(v) = line.strip_prefix("MemAvailable:") { + available = kb_to_gb(v); + } + } + MemInfo { + total_gb: total, + available_gb: available, + } +} + +/// Parse a " 123456 kB" meminfo value into GB. +fn kb_to_gb(v: &str) -> f64 { + v.split_whitespace() + .next() + .and_then(|n| n.parse::().ok()) + .unwrap_or(0.0) + / 1024.0 + / 1024.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPARK: &str = "\ +MemTotal: 127512345 kB +MemFree: 2048000 kB +MemAvailable: 104857600 kB +Buffers: 123456 kB +"; + + #[test] + fn reads_total_and_available_and_ignores_other_fields() { + let m = parse_meminfo(SPARK); + assert!((m.total_gb - 121.6).abs() < 0.1, "total was {}", m.total_gb); + assert!( + (m.available_gb - 100.0).abs() < 0.1, + "available was {}", + m.available_gb + ); + } + + #[test] + fn missing_fields_read_as_zero_rather_than_panicking() { + // A zero here is what makes admission refuse, so an unparseable /proc must not + // look like an empty box with room to spare. + let m = parse_meminfo("SomethingElse: 1 kB\n"); + assert_eq!(m.total_gb, 0.0); + assert_eq!(m.available_gb, 0.0); + } + + #[test] + fn malformed_values_do_not_panic() { + let m = parse_meminfo("MemTotal: not-a-number kB\nMemAvailable:\n"); + assert_eq!(m.total_gb, 0.0); + assert_eq!(m.available_gb, 0.0); + } + + #[test] + fn memavailable_is_not_confused_with_memfree() { + // MemFree is much smaller than MemAvailable on a box with page cache; picking the + // wrong one would make the watchdog fire constantly. + let m = parse_meminfo(SPARK); + assert!(m.available_gb > 50.0); + } +} diff --git a/src/proc.rs b/src/proc.rs new file mode 100644 index 0000000..de9a24f --- /dev/null +++ b/src/proc.rs @@ -0,0 +1,470 @@ +//! Process lifecycle: spawn a model in its own process group, track it in a state +//! file, and stop it (SIGTERM → SIGKILL to the whole group). The Governor decides +//! *whether* to start; this module *how*. + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{self, Command, Stdio}; +use std::time::Duration; + +use crate::config::Model; + +/// One Lumbridge Compute-managed process. `seq` is a monotonic launch counter so the watchdog +/// can always find the *newest* model to sacrifice first. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Proc { + pub pid: i32, + pub seq: u64, + pub port: Option, + /// Linux boot id plus process start ticks bind ownership to one process + /// incarnation, preventing a reused PID from ever being signalled. + #[serde(default)] + pub boot_id: Option, + #[serde(default)] + pub start_time_ticks: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct State { + #[serde(default)] + pub seq: u64, + #[serde(default)] + pub procs: BTreeMap, + #[serde(default)] + pub desired_scene: Option, + #[serde(default)] + pub active_scene: Option, + /// Previous proven scene used if the desired scene cannot resume. + #[serde(default)] + pub last_known_good_scene: Option, + #[serde(default)] + pub transition_scene: Option, + #[serde(default)] + pub last_error: Option, +} + +fn state_dir(root: &Path) -> PathBuf { + root.join(".compute") +} + +impl State { + pub fn load_checked(root: &Path) -> Result { + let p = state_dir(root).join("state.yaml"); + match fs::read_to_string(&p) { + Ok(s) => serde_yaml::from_str(&s) + .with_context(|| format!("parsing persisted state {}", p.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(State::default()), + Err(error) => Err(error).with_context(|| format!("reading {}", p.display())), + } + } + + pub fn save(&self, root: &Path) -> Result<()> { + let dir = state_dir(root); + fs::create_dir_all(&dir)?; + let p = dir.join("state.yaml"); + let tmp = dir.join(format!(".state.yaml.{}.tmp", process::id())); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp) + .with_context(|| format!("creating {}", tmp.display()))?; + file.write_all(serde_yaml::to_string(self)?.as_bytes())?; + file.sync_all()?; + fs::rename(&tmp, &p).with_context(|| format!("atomically replacing {}", p.display()))?; + File::open(&dir) + .with_context(|| format!("opening state directory {}", dir.display()))? + .sync_all() + .with_context(|| format!("syncing state directory {}", dir.display()))?; + Ok(()) + } + + /// Newest still-alive managed model id, if any. + pub fn newest_alive(&self) -> Option<(String, Proc)> { + self.procs + .iter() + .filter(|(_, p)| p.owned_alive()) + .max_by_key(|(_, p)| p.seq) + .map(|(id, p)| (id.clone(), p.clone())) + } +} + +#[derive(Debug)] +struct ProcessIdentity { + boot_id: String, + start_time_ticks: u64, + process_group_id: i32, + state: char, +} + +impl Proc { + /// Capture the identity of a process that Lumbridge Compute just spawned or + /// that an operator explicitly adopted from legacy state. + pub fn capture(pid: i32, seq: u64, port: Option) -> Result { + let identity = process_identity(pid)?; + if identity.process_group_id != pid { + bail!( + "refusing process {pid}: process group {} does not equal pid", + identity.process_group_id + ); + } + Ok(Proc { + pid, + seq, + port, + boot_id: Some(identity.boot_id), + start_time_ticks: Some(identity.start_time_ticks), + }) + } + + pub fn owned_alive(&self) -> bool { + self.validate_owned().is_ok() + } + + fn validate_owned(&self) -> Result<()> { + let expected_boot = self + .boot_id + .as_deref() + .context("legacy process record has no boot identity; adopt the active scene first")?; + let expected_start = self + .start_time_ticks + .context("legacy process record has no start identity; adopt the active scene first")?; + let current = process_identity(self.pid)?; + if current.boot_id != expected_boot || current.start_time_ticks != expected_start { + bail!( + "pid {} no longer identifies the process Compute started", + self.pid + ); + } + if current.state == 'Z' { + bail!("pid {} is a zombie awaiting reap", self.pid); + } + if current.process_group_id != self.pid { + bail!( + "pid {} now belongs to process group {}; refusing group signal", + self.pid, + current.process_group_id + ); + } + Ok(()) + } +} + +fn process_identity(pid: i32) -> Result { + let boot_id = fs::read_to_string("/proc/sys/kernel/random/boot_id") + .context("reading Linux boot id")? + .trim() + .to_string(); + let stat_path = format!("/proc/{pid}/stat"); + let stat = fs::read_to_string(&stat_path) + .with_context(|| format!("reading process identity {stat_path}"))?; + let (state, process_group_id, start_time_ticks) = parse_stat_identity(&stat)?; + Ok(ProcessIdentity { + boot_id, + start_time_ticks, + process_group_id, + state, + }) +} + +fn parse_stat_identity(stat: &str) -> Result<(char, i32, u64)> { + // `comm` is parenthesized and may contain spaces, so locate its final `)` + // before indexing the fields that follow it. The remainder starts at field 3. + let close = stat + .rfind(')') + .context("malformed /proc stat: missing process name terminator")?; + let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect(); + let state = fields + .first() + .and_then(|value| value.chars().next()) + .context("malformed /proc stat: missing process state")?; + let process_group_id = fields + .get(2) + .context("malformed /proc stat: missing process group")? + .parse::() + .context("parsing process group id")?; + let start_time_ticks = fields + .get(19) + .context("malformed /proc stat: missing start time")? + .parse::() + .context("parsing process start time")?; + Ok((state, process_group_id, start_time_ticks)) +} + +/// Expand a leading `~/` to `$HOME`. +fn expand(s: &str) -> String { + if let Some(rest) = s.strip_prefix("~/") { + if let Ok(home) = std::env::var("HOME") { + return format!("{home}/{rest}"); + } + } + s.to_string() +} + +/// The fraction of the pool a vLLM model may reserve, derived from its declared +/// footprint rather than typed separately. +/// +/// `footprint_gb` is what the Governor budgets against; `--gpu-memory-utilization` is +/// what actually reserves the memory. Keeping them as two hand-copied numbers means +/// admission control can be arithmetically correct and still wrong about the machine — +/// the registry says a model takes 66 GB while the flag lets it reserve 0.55 of a +/// 121.6 GB pool, which is 67. Deriving one from the other makes the declared footprint +/// the single source of truth. +/// +/// An explicit `gpu-memory-utilization` in `serve.args` always wins; this only fills in +/// the gap. Returns `None` when the pool size is unknown, in which case vLLM's own +/// default applies exactly as before. +fn derived_gpu_memory_utilization(m: &Model, mem_total_gb: Option) -> Option { + if m.serve.args.contains_key("gpu-memory-utilization") { + return None; + } + let total = mem_total_gb?; + if total <= 0.0 || m.footprint_gb <= 0.0 { + return None; + } + let fraction = m.footprint_gb / total; + // Never hand vLLM a fraction that would reserve the whole box. + (fraction > 0.0 && fraction < 0.95).then_some((fraction * 1000.0).round() / 1000.0) +} + +/// `kind`-based builder. argv[0] is the program. +pub fn build_argv(m: &Model, mem_total_gb: Option) -> Result> { + if !m.serve.command.is_empty() { + return Ok(m.serve.command.iter().map(|s| expand(s)).collect()); + } + match m.serve.kind.as_str() { + "vllm" => { + let executable = m + .serve + .executable + .as_deref() + .map(expand) + .unwrap_or_else(|| "vllm".to_string()); + let mut a = vec![executable, "serve".to_string()]; + if let Some(w) = &m.serve.weights { + a.push(expand(w)); + } + if !m.serve.served_name.is_empty() { + a.push("--served-model-name".into()); + a.extend(m.serve.served_name.iter().cloned()); + } + a.push("--host".into()); + a.push("0.0.0.0".into()); + if let Some(p) = m.serve.port { + a.push("--port".into()); + a.push(p.to_string()); + } + if let Some(fraction) = derived_gpu_memory_utilization(m, mem_total_gb) { + a.push("--gpu-memory-utilization".into()); + a.push(format!("{fraction}")); + } + for (k, v) in &m.serve.args { + render_arg(&mut a, k, v); + } + Ok(a) + } + other => bail!( + "model '{}' has no explicit `command` and kind '{}' has no builder yet — \ + add a `command: [...]` to the registry entry", + m.name, + other + ), + } +} + +fn render_arg(out: &mut Vec, k: &str, v: &serde_yaml::Value) { + use serde_yaml::Value; + let flag = format!("--{k}"); + match v { + Value::Bool(true) => out.push(flag), + Value::Bool(false) => {} + Value::String(s) => { + out.push(flag); + out.push(s.clone()); + } + Value::Number(n) => { + out.push(flag); + out.push(n.to_string()); + } + _ => { + out.push(flag); + out.push( + serde_yaml::to_string(v) + .unwrap_or_default() + .trim() + .to_string(), + ); + } + } +} + +/// Spawn `id`'s model detached in its own process group, logs → `.compute/logs/.log`. +/// Returns the child pid (also its process-group id). +pub fn spawn(root: &Path, id: &str, m: &Model) -> Result { + let argv = build_argv(m, crate::mem::read().ok().map(|info| info.total_gb))?; + let logdir = state_dir(root).join("logs"); + fs::create_dir_all(&logdir)?; + let log = File::create(logdir.join(format!("{id}.log")))?; + let errlog = log.try_clone()?; + + let mut cmd = Command::new(&argv[0]); + cmd.args(&argv[1..]) + .stdin(Stdio::null()) + .stdout(Stdio::from(log)) + .stderr(Stdio::from(errlog)) + .process_group(0); // own group → clean group-kill, immune to CLI's signals + for (k, v) in &m.serve.env { + cmd.env(k, expand(v)); + } + let mut child = cmd + .spawn() + .with_context(|| format!("spawning '{id}': {}", argv.join(" ")))?; + let pid = child.id() as i32; + // A resident agent may outlive many model processes. Reap each child when + // it exits so failed runtimes cannot accumulate as zombies under the agent. + std::thread::spawn(move || { + let _ = child.wait(); + }); + Ok(pid) +} + +/// Stop only the exact process group captured in this ownership record. +pub fn stop_owned(proc: &Proc) -> Result<()> { + if !proc.owned_alive() { + return Ok(()); + } + proc.validate_owned()?; + let term = unsafe { libc::kill(-proc.pid, libc::SIGTERM) }; + if term != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("sending SIGTERM to owned process group {}", proc.pid)); + } + for _ in 0..60 { + if !proc.owned_alive() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + proc.validate_owned()?; + let kill = unsafe { libc::kill(-proc.pid, libc::SIGKILL) }; + if kill != 0 { + return Err(std::io::Error::last_os_error()) + .with_context(|| format!("sending SIGKILL to owned process group {}", proc.pid)); + } + for _ in 0..20 { + if !proc.owned_alive() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + bail!("owned process group {} survived SIGKILL", proc.pid) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a Model the way the registry does — through serde — so these tests exercise + /// the real deserialization path rather than a hand-assembled struct. + fn vllm_model(footprint_gb: f64, extra_args: &str) -> Model { + let yaml = format!( + "name: brain\nfootprint_gb: {footprint_gb}\nserve:\n kind: vllm\n port: 8001\n args:\n max-model-len: 8192\n{extra_args}" + ); + serde_yaml::from_str(&yaml).expect("test model yaml") + } + + #[test] + fn gpu_memory_utilization_is_derived_from_the_declared_footprint() { + // 66 GB of a 121.6 GB pool is 0.543 — not the 0.55 that used to be typed in + // separately, which is the whole point: one number, not two that can drift. + let m = vllm_model(66.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), Some(0.543)); + let argv = build_argv(&m, Some(121.6)).unwrap(); + let i = argv + .iter() + .position(|a| a == "--gpu-memory-utilization") + .unwrap(); + assert_eq!(argv[i + 1], "0.543"); + } + + #[test] + fn an_explicit_flag_in_the_registry_always_wins() { + let m = vllm_model(66.0, " gpu-memory-utilization: 0.8\n"); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None); + // ...and it is rendered exactly once, from serve.args. + let argv = build_argv(&m, Some(121.6)).unwrap(); + assert_eq!( + argv.iter() + .filter(|a| *a == "--gpu-memory-utilization") + .count(), + 1 + ); + let i = argv + .iter() + .position(|a| a == "--gpu-memory-utilization") + .unwrap(); + assert_eq!(argv[i + 1], "0.8"); + } + + #[test] + fn unknown_pool_size_leaves_vllms_own_default_alone() { + let m = vllm_model(66.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, None), None); + assert!(!build_argv(&m, None) + .unwrap() + .iter() + .any(|a| a == "--gpu-memory-utilization")); + } + + #[test] + fn a_footprint_that_would_claim_the_whole_box_is_not_derived() { + // Better to let vLLM apply its own default than to hand it 0.99 and wedge the box. + let m = vllm_model(120.0, ""); + assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None); + } + + #[test] + fn nonsense_inputs_do_not_produce_a_flag() { + assert_eq!( + derived_gpu_memory_utilization(&vllm_model(0.0, ""), Some(121.6)), + None + ); + assert_eq!( + derived_gpu_memory_utilization(&vllm_model(66.0, ""), Some(0.0)), + None + ); + } + + #[test] + fn an_explicit_command_bypasses_the_builder_entirely() { + let mut m = vllm_model(66.0, ""); + m.serve.command = vec!["python".into(), "-m".into(), "server".into()]; + assert_eq!(build_argv(&m, Some(121.6)).unwrap(), m.serve.command); + } + + #[test] + fn proc_stat_parser_handles_spaces_in_process_name() { + let stat = "123 (worker process) S 1 123 123 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 4567"; + let (state, pgrp, start) = parse_stat_identity(stat).unwrap(); + assert_eq!(state, 'S'); + assert_eq!(pgrp, 123); + assert_eq!(start, 4567); + } + + #[test] + fn legacy_records_are_never_treated_as_owned() { + let proc = Proc { + pid: std::process::id() as i32, + seq: 1, + port: None, + boot_id: None, + start_time_ticks: None, + }; + assert!(!proc.owned_alive()); + } +} diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 0000000..14afedb --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,1158 @@ +//! Privacy-safe, durable model-serving telemetry. +//! +//! The model runtime remains the authority for scheduling, token, latency, and +//! multimodal encoder metrics. Compute samples its Prometheus endpoint and stores +//! only numeric measurements plus bounded labels. Prompts, outputs, image data, +//! URLs, and arbitrary HTTP headers never enter this database. + +use anyhow::{bail, Context, Result}; +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::io::{Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::config::{find_scene, Registry}; +use crate::proc::State; + +const DB_RELATIVE_PATH: &str = ".compute/usage/telemetry.sqlite3"; +const MAX_METRICS_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +const COUNTER_SNAPSHOT_INTERVAL_MS: i64 = 60_000; +const ALLOWED_CUSTOM_LABELS: &[&str] = &["client", "agent", "workload"]; + +const COUNTERS: &[&str] = &[ + "sglang:prompt_tokens_total", + "sglang:generation_tokens_total", + "sglang:cached_tokens_total", + "sglang:num_requests_total", + "sglang:num_aborted_requests_total", + "sglang:encoder_requests_received_total", + "sglang:encoder_cache_total_tokens_total", + "sglang:encoder_cache_hit_tokens_total", + "sglang:encoder_mm_items_per_request_sum", + "sglang:encoder_mm_items_per_request_count", + "sglang:time_to_first_token_seconds_sum", + "sglang:time_to_first_token_seconds_count", + "sglang:e2e_request_latency_seconds_sum", + "sglang:e2e_request_latency_seconds_count", +]; + +#[derive(Debug, Clone, PartialEq)] +struct Metric { + name: String, + labels: BTreeMap, + value: f64, +} + +#[derive(Debug, Clone)] +pub struct Target { + pub model: String, + pub scene: String, + pub max_concurrency: u32, + pub config_fingerprint: String, +} + +#[derive(Debug, Serialize)] +pub struct CollectResult { + pub timestamp_ms: i64, + pub model: String, + pub scene: String, + pub max_concurrency: u32, + pub metrics_recorded: usize, + pub running: u32, + pub queued: u32, +} + +#[derive(Debug, Clone, Serialize, Default)] +pub struct UsageTotals { + pub requests: f64, + pub aborted_requests: f64, + pub prompt_tokens: f64, + pub generation_tokens: f64, + pub cached_tokens: f64, + pub vision_requests: f64, + pub vision_request_share: f64, + pub vision_request_source: String, + pub vision_prompt_tokens: f64, + pub vision_items: f64, + pub vision_encoder_tokens: f64, + pub vision_encoder_cached_tokens: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencyBucket { + pub concurrency: String, + pub samples: u64, + pub share: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencySummary { + pub samples: u64, + pub queued_samples: u64, + pub peak_running: u32, + pub peak_queued: u32, + /// A time-sampled distribution, not an admission-count distribution. + pub distribution: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AgentUsage { + pub client: String, + pub agent: String, + pub workload: String, + pub requests: f64, + pub prompt_tokens: f64, + pub generation_tokens: f64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct UsageSummary { + pub since_ms: i64, + pub until_ms: i64, + pub database: String, + pub totals: UsageTotals, + pub concurrency: ConcurrencySummary, + pub agents: Vec, +} + +pub struct Store { + path: PathBuf, + conn: Connection, +} + +impl Store { + pub fn open(root: &Path) -> Result { + let path = root.join(DB_RELATIVE_PATH); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("creating telemetry directory {}", parent.display()))?; + } + let conn = Connection::open(&path) + .with_context(|| format!("opening telemetry database {}", path.display()))?; + conn.busy_timeout(Duration::from_secs(5))?; + conn.execute_batch( + "PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS telemetry_schema ( + version INTEGER PRIMARY KEY, + applied_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS config_snapshots ( + fingerprint TEXT PRIMARY KEY, + first_seen_ms INTEGER NOT NULL, + model TEXT NOT NULL, + scene TEXT NOT NULL, + max_concurrency INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS engine_samples ( + timestamp_ms INTEGER NOT NULL, + model TEXT NOT NULL, + scene TEXT NOT NULL, + config_fingerprint TEXT NOT NULL, + max_concurrency INTEGER NOT NULL, + running REAL NOT NULL, + queued REAL NOT NULL, + generation_tps REAL, + cache_hit_rate REAL, + PRIMARY KEY (timestamp_ms, model) + ); + CREATE INDEX IF NOT EXISTS engine_samples_time + ON engine_samples(timestamp_ms); + CREATE TABLE IF NOT EXISTS counter_samples ( + timestamp_ms INTEGER NOT NULL, + model TEXT NOT NULL, + metric TEXT NOT NULL, + labels_json TEXT NOT NULL, + value REAL NOT NULL, + PRIMARY KEY (timestamp_ms, model, metric, labels_json) + ); + CREATE INDEX IF NOT EXISTS counter_samples_metric_time + ON counter_samples(metric, timestamp_ms); + INSERT OR IGNORE INTO telemetry_schema(version, applied_at_ms) + VALUES (1, CAST(strftime('%s','now') AS INTEGER) * 1000);", + )?; + Ok(Self { path, conn }) + } + + fn record(&mut self, target: &Target, metrics: &[Metric]) -> Result { + self.record_with_counter_interval(target, metrics, COUNTER_SNAPSHOT_INTERVAL_MS) + } + + fn record_with_counter_interval( + &mut self, + target: &Target, + metrics: &[Metric], + counter_interval_ms: i64, + ) -> Result { + let timestamp_ms = unix_time_ms()?; + let running = default_gauge(metrics, "sglang:num_running_reqs").unwrap_or(0.0); + let queued = default_gauge(metrics, "sglang:num_queue_reqs").unwrap_or(0.0); + let generation_tps = default_gauge(metrics, "sglang:gen_throughput"); + let cache_hit_rate = default_gauge(metrics, "sglang:cache_hit_rate"); + + let last_counter_ms: Option = self.conn.query_row( + "SELECT MAX(timestamp_ms) FROM counter_samples WHERE model = ?1", + params![target.model], + |row| row.get(0), + )?; + let counter_due = + last_counter_ms.is_none_or(|previous| timestamp_ms - previous >= counter_interval_ms); + + let transaction = self.conn.transaction()?; + transaction.execute( + "INSERT OR IGNORE INTO config_snapshots + (fingerprint, first_seen_ms, model, scene, max_concurrency) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + target.config_fingerprint, + timestamp_ms, + target.model, + target.scene, + target.max_concurrency + ], + )?; + transaction.execute( + "INSERT INTO engine_samples + (timestamp_ms, model, scene, config_fingerprint, max_concurrency, + running, queued, generation_tps, cache_hit_rate) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + timestamp_ms, + target.model, + target.scene, + target.config_fingerprint, + target.max_concurrency, + running, + queued, + generation_tps, + cache_hit_rate + ], + )?; + + let mut recorded = 0usize; + if counter_due { + for metric in metrics + .iter() + .filter(|metric| COUNTERS.contains(&metric.name.as_str())) + { + let labels = retained_labels(&metric.labels); + transaction.execute( + "INSERT INTO counter_samples + (timestamp_ms, model, metric, labels_json, value) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + timestamp_ms, + target.model, + metric.name, + serde_json::to_string(&labels)?, + metric.value + ], + )?; + recorded += 1; + } + } + transaction.commit()?; + + Ok(CollectResult { + timestamp_ms, + model: target.model.clone(), + scene: target.scene.clone(), + max_concurrency: target.max_concurrency, + metrics_recorded: recorded, + running: running.max(0.0).round() as u32, + queued: queued.max(0.0).round() as u32, + }) + } + + pub fn summary(&self, since_ms: i64, until_ms: i64) -> Result { + let image = |labels: &BTreeMap| { + labels.get("modality").is_some_and(|value| value == "image") + }; + let vision_workload = |labels: &BTreeMap| { + labels + .get("workload") + .is_some_and(|value| value == "vision") + }; + let has_encoder_request_metrics = + self.has_metric_samples("sglang:encoder_requests_received_total", since_ms, until_ms)?; + let vision_requests = if has_encoder_request_metrics { + self.metric_delta( + "sglang:encoder_requests_received_total", + since_ms, + until_ms, + image, + )? + } else { + self.metric_delta( + "sglang:num_requests_total", + since_ms, + until_ms, + vision_workload, + )? + }; + let mut totals = UsageTotals { + requests: self + .metric_delta("sglang:num_requests_total", since_ms, until_ms, |_| true)?, + aborted_requests: self.metric_delta( + "sglang:num_aborted_requests_total", + since_ms, + until_ms, + |_| true, + )?, + prompt_tokens: self.metric_delta( + "sglang:prompt_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + generation_tokens: self.metric_delta( + "sglang:generation_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + cached_tokens: self.metric_delta( + "sglang:cached_tokens_total", + since_ms, + until_ms, + |_| true, + )?, + vision_requests, + vision_request_share: 0.0, + vision_request_source: if has_encoder_request_metrics { + "encoder_metrics".to_string() + } else { + "workload_label".to_string() + }, + vision_prompt_tokens: self.metric_delta( + "sglang:prompt_tokens_total", + since_ms, + until_ms, + vision_workload, + )?, + vision_items: self.metric_delta( + "sglang:encoder_mm_items_per_request_sum", + since_ms, + until_ms, + image, + )?, + vision_encoder_tokens: self.metric_delta( + "sglang:encoder_cache_total_tokens_total", + since_ms, + until_ms, + image, + )?, + vision_encoder_cached_tokens: self.metric_delta( + "sglang:encoder_cache_hit_tokens_total", + since_ms, + until_ms, + image, + )?, + }; + totals.vision_request_share = if totals.requests > 0.0 { + (totals.vision_requests / totals.requests).clamp(0.0, 1.0) + } else { + 0.0 + }; + + Ok(UsageSummary { + since_ms, + until_ms, + database: self.path.display().to_string(), + totals, + concurrency: self.concurrency(since_ms, until_ms)?, + agents: self.agent_usage(since_ms, until_ms)?, + }) + } + + fn metric_delta(&self, metric: &str, since_ms: i64, until_ms: i64, keep: F) -> Result + where + F: Fn(&BTreeMap) -> bool, + { + let total: f64 = self + .series_deltas(metric, since_ms, until_ms)? + .into_iter() + .filter(|(labels, _)| keep(labels)) + .map(|(_, delta)| delta) + .sum(); + Ok(if total == 0.0 { 0.0 } else { total }) + } + + fn has_metric_samples(&self, metric: &str, since_ms: i64, until_ms: i64) -> Result { + self.conn + .query_row( + "SELECT EXISTS( + SELECT 1 FROM counter_samples + WHERE metric = ?1 AND timestamp_ms >= ?2 AND timestamp_ms <= ?3 + )", + params![metric, since_ms, until_ms], + |row| row.get(0), + ) + .context("checking telemetry metric availability") + } + + fn series_deltas( + &self, + metric: &str, + since_ms: i64, + until_ms: i64, + ) -> Result, f64)>> { + let mut baseline_statement = self.conn.prepare( + "SELECT model, MIN(timestamp_ms) + FROM counter_samples + WHERE metric = ?1 AND timestamp_ms <= ?2 + GROUP BY model", + )?; + let baseline_rows = baseline_statement.query_map(params![metric, until_ms], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })?; + let mut model_baselines = HashMap::new(); + for row in baseline_rows { + let (model, timestamp_ms) = row?; + model_baselines.insert(model, timestamp_ms); + } + + let mut statement = self.conn.prepare( + "SELECT model, timestamp_ms, labels_json, value + FROM counter_samples + WHERE metric = ?1 AND timestamp_ms <= ?2 + ORDER BY model, labels_json, timestamp_ms", + )?; + let rows = statement.query_map(params![metric, until_ms], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, f64>(3)?, + )) + })?; + + let mut state: HashMap<(String, String), (Option, f64)> = HashMap::new(); + for row in rows { + let (model, timestamp_ms, labels_json, value) = row?; + let model_baseline = model_baselines.get(&model).copied().unwrap_or(timestamp_ms); + let entry = state.entry((model, labels_json)).or_insert((None, 0.0)); + if timestamp_ms >= since_ms { + if let Some(previous) = entry.0 { + entry.1 += if value >= previous { + value - previous + } else { + value + }; + } else if timestamp_ms > model_baseline { + // Prometheus creates a new labeled series on first use. The + // model-wide baseline predates it, so its initial value is + // usage in this window rather than unknown prehistory. + entry.1 += value; + } + } + entry.0 = Some(value); + } + + state + .into_iter() + .map(|((_model, labels), (_previous, delta))| { + let labels = serde_json::from_str(&labels) + .context("parsing labels stored in telemetry database")?; + Ok((labels, delta)) + }) + .collect() + } + + fn concurrency(&self, since_ms: i64, until_ms: i64) -> Result { + let mut statement = self.conn.prepare( + "SELECT running, queued FROM engine_samples + WHERE timestamp_ms >= ?1 AND timestamp_ms <= ?2 + ORDER BY timestamp_ms", + )?; + let rows = statement.query_map(params![since_ms, until_ms], |row| { + Ok((row.get::<_, f64>(0)?, row.get::<_, f64>(1)?)) + })?; + let mut counts: BTreeMap = BTreeMap::new(); + let mut samples = 0u64; + let mut queued_samples = 0u64; + let mut peak_running = 0u32; + let mut peak_queued = 0u32; + for row in rows { + let (running, queued) = row?; + let running = running.max(0.0).round() as u32; + let queued = queued.max(0.0).round() as u32; + let bucket = if running >= 4 { + "C4+".to_string() + } else { + format!("C{running}") + }; + *counts.entry(bucket).or_default() += 1; + samples += 1; + queued_samples += u64::from(queued > 0); + peak_running = peak_running.max(running); + peak_queued = peak_queued.max(queued); + } + let distribution = counts + .into_iter() + .map(|(concurrency, count)| ConcurrencyBucket { + concurrency, + samples: count, + share: if samples == 0 { + 0.0 + } else { + count as f64 / samples as f64 + }, + }) + .collect(); + Ok(ConcurrencySummary { + samples, + queued_samples, + peak_running, + peak_queued, + distribution, + }) + } + + fn agent_usage(&self, since_ms: i64, until_ms: i64) -> Result> { + #[derive(Default)] + struct Row { + requests: f64, + prompt_tokens: f64, + generation_tokens: f64, + } + let mut grouped: BTreeMap<(String, String, String), Row> = BTreeMap::new(); + for (metric, field) in [ + ("sglang:num_requests_total", 0), + ("sglang:prompt_tokens_total", 1), + ("sglang:generation_tokens_total", 2), + ] { + for (labels, delta) in self.series_deltas(metric, since_ms, until_ms)? { + let key = ( + label_or_unknown(&labels, "client"), + label_or_unknown(&labels, "agent"), + label_or_unknown(&labels, "workload"), + ); + let row = grouped.entry(key).or_default(); + match field { + 0 => row.requests += delta, + 1 => row.prompt_tokens += delta, + _ => row.generation_tokens += delta, + } + } + } + let mut rows: Vec<_> = grouped + .into_iter() + .filter(|(_, row)| { + row.requests != 0.0 || row.prompt_tokens != 0.0 || row.generation_tokens != 0.0 + }) + .map(|((client, agent, workload), row)| AgentUsage { + client, + agent, + workload, + requests: row.requests, + prompt_tokens: row.prompt_tokens, + generation_tokens: row.generation_tokens, + }) + .collect(); + rows.sort_by(|a, b| b.requests.total_cmp(&a.requests)); + Ok(rows) + } +} + +pub fn collect_once(root: &Path, metrics_url: &str, model_hint: &str) -> Result { + let target = resolve_target(root, metrics_url, model_hint)?; + let body = fetch_http(metrics_url)?; + let metrics = parse_prometheus(&body); + if !metrics + .iter() + .any(|metric| metric.name == "sglang:num_running_reqs") + { + bail!("{metrics_url} returned no SGLang scheduler metrics; start SGLang with --enable-metrics"); + } + Store::open(root)?.record(&target, &metrics) +} + +pub fn run_collector(root: PathBuf, metrics_url: String, model_hint: String, interval: Duration) { + let mut failures = 0u64; + loop { + match collect_once(&root, &metrics_url, &model_hint) { + Ok(result) => { + if failures > 0 { + eprintln!( + "telemetry: recovered after {failures} failed scrape(s); model={} scene={}", + result.model, result.scene + ); + } + failures = 0; + } + Err(error) => { + failures += 1; + if failures == 1 || failures.is_multiple_of(60) { + eprintln!("telemetry: scrape failed ({failures} consecutive): {error:#}"); + } + } + } + thread::sleep(interval.max(Duration::from_secs(1))); + } +} + +pub fn report(root: &Path, since: &str) -> Result { + let duration = parse_duration(since)?; + let until_ms = unix_time_ms()?; + let since_ms = until_ms.saturating_sub(duration.as_millis().min(i64::MAX as u128) as i64); + Store::open(root)?.summary(since_ms, until_ms) +} + +pub fn render_text(summary: &UsageSummary) -> String { + let totals = &summary.totals; + let mut lines = vec![ + "Lumbridge Compute · usage".to_string(), + format!(" requests {:>12.0}", totals.requests), + format!(" prompt tok {:>12.0}", totals.prompt_tokens), + format!(" output tok {:>12.0}", totals.generation_tokens), + format!(" cached tok {:>12.0}", totals.cached_tokens), + format!(" aborted {:>12.0}", totals.aborted_requests), + format!(" vision calls {:>12.0}", totals.vision_requests), + format!( + " vision share {:>11.1}%", + totals.vision_request_share * 100.0 + ), + format!(" vision tok {:>12.0}", totals.vision_prompt_tokens), + format!(" image items {:>12.0}", totals.vision_items), + String::new(), + " time-sampled concurrency:".to_string(), + ]; + if summary.concurrency.samples == 0 { + lines.push(" (no samples in this window)".to_string()); + } else { + for bucket in &summary.concurrency.distribution { + lines.push(format!( + " {:>3} {:>8} samples {:>6.1}%", + bucket.concurrency, + bucket.samples, + bucket.share * 100.0 + )); + } + lines.push(format!( + " peak running {} · peak queued {} · queue observed in {} sample(s)", + summary.concurrency.peak_running, + summary.concurrency.peak_queued, + summary.concurrency.queued_samples + )); + } + if !summary.agents.is_empty() { + lines.push(String::new()); + lines.push(" agents:".to_string()); + for row in &summary.agents { + lines.push(format!( + " {}/{}/{} {:.0} req · {:.0} in · {:.0} out tok", + row.client, + row.agent, + row.workload, + row.requests, + row.prompt_tokens, + row.generation_tokens + )); + } + } + lines.join("\n") +} + +fn resolve_target(root: &Path, metrics_url: &str, model_hint: &str) -> Result { + let registry = Registry::load(root)?; + let state = State::load_checked(root)?; + let scene_name = state + .active_scene + .or(state.desired_scene) + .unwrap_or_else(|| "unknown".to_string()); + let port = parse_http_url(metrics_url)?.1; + let model = if model_hint != "auto" { + model_hint.to_string() + } else if scene_name != "unknown" { + find_scene(root, &scene_name)? + .models + .into_iter() + .find(|id| registry.models.get(id).and_then(|m| m.serve.port) == Some(port)) + .with_context(|| { + format!("active Scene '{scene_name}' has no model on metrics port {port}") + })? + } else { + registry + .models + .iter() + .find(|(_, model)| model.serve.port == Some(port)) + .map(|(id, _)| id.clone()) + .with_context(|| format!("registry has no model on metrics port {port}"))? + }; + let model_config = registry + .models + .get(&model) + .with_context(|| format!("no registry model named '{model}'"))?; + let max_concurrency = configured_concurrency(model_config); + let fingerprint_input = format!( + "{model}\n{scene_name}\n{max_concurrency}\n{:?}\n{:?}", + model_config.serve.command, model_config.serve.args + ); + Ok(Target { + model, + scene: scene_name, + max_concurrency, + config_fingerprint: format!("fnv1a64:{:016x}", fnv1a64(fingerprint_input.as_bytes())), + }) +} + +fn configured_concurrency(model: &crate::config::Model) -> u32 { + for flag in ["--max-running-requests", "--max-num-seqs"] { + if let Some(index) = model.serve.command.iter().position(|arg| arg == flag) { + if let Some(value) = model + .serve + .command + .get(index + 1) + .and_then(|value| value.parse().ok()) + { + return value; + } + } + } + for key in ["max-running-requests", "max-num-seqs"] { + if let Some(value) = model + .serve + .args + .get(key) + .and_then(serde_yaml::Value::as_u64) + { + return value.min(u32::MAX as u64) as u32; + } + } + 0 +} + +fn retained_labels(labels: &BTreeMap) -> BTreeMap { + labels + .iter() + .filter(|(key, _)| { + matches!( + key.as_str(), + "model_name" | "stream" | "reason" | "modality" | "status" | "priority" + ) || ALLOWED_CUSTOM_LABELS.contains(&key.as_str()) + }) + .map(|(key, value)| { + let value = if ALLOWED_CUSTOM_LABELS.contains(&key.as_str()) { + sanitize_custom_label(value) + } else { + value.chars().take(128).collect() + }; + (key.clone(), value) + }) + .collect() +} + +fn sanitize_custom_label(value: &str) -> String { + if !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + value.to_string() + } else { + "invalid".to_string() + } +} + +fn label_or_unknown(labels: &BTreeMap, key: &str) -> String { + labels + .get(key) + // SGLang emits empty custom labels on unlabeled traffic. They are + // deliberately collapsed to the same bounded sentinel as malformed + // values at ingestion, and both render as unattributed usage. + .filter(|value| !value.is_empty() && value.as_str() != "invalid") + .cloned() + .unwrap_or_else(|| "unknown".to_string()) +} + +fn default_gauge(metrics: &[Metric], name: &str) -> Option { + metrics + .iter() + .filter(|metric| metric.name == name) + .find(|metric| metric.labels.get("priority").is_none_or(String::is_empty)) + .map(|metric| metric.value) +} + +fn parse_prometheus(text: &str) -> Vec { + text.lines().filter_map(parse_metric_line).collect() +} + +fn parse_metric_line(line: &str) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let split = line.rfind(char::is_whitespace)?; + let head = line[..split].trim(); + let value = line[split..].trim().parse::().ok()?; + if !value.is_finite() { + return None; + } + let (name, labels) = if let Some(open) = head.find('{') { + if !head.ends_with('}') { + return None; + } + ( + &head[..open], + parse_labels(&head[open + 1..head.len() - 1])?, + ) + } else { + (head, BTreeMap::new()) + }; + Some(Metric { + name: name.to_string(), + labels, + value, + }) +} + +fn parse_labels(input: &str) -> Option> { + let bytes = input.as_bytes(); + let mut labels = BTreeMap::new(); + let mut index = 0usize; + while index < bytes.len() { + while index < bytes.len() && (bytes[index] == b',' || bytes[index].is_ascii_whitespace()) { + index += 1; + } + if index == bytes.len() { + break; + } + let key_start = index; + while index < bytes.len() && bytes[index] != b'=' { + index += 1; + } + if index == bytes.len() { + return None; + } + let key = input[key_start..index].trim().to_string(); + index += 1; + if bytes.get(index) != Some(&b'"') { + return None; + } + index += 1; + let mut value = String::new(); + while index < bytes.len() { + match bytes[index] { + b'"' => { + index += 1; + break; + } + b'\\' => { + index += 1; + let escaped = *bytes.get(index)?; + value.push(match escaped { + b'n' => '\n', + b'\\' => '\\', + b'"' => '"', + other => other as char, + }); + index += 1; + } + byte => { + value.push(byte as char); + index += 1; + } + } + } + labels.insert(key, value); + } + Some(labels) +} + +fn fetch_http(url: &str) -> Result { + let (host, port, path) = parse_http_url(url)?; + let address = (host.as_str(), port) + .to_socket_addrs()? + .next() + .with_context(|| format!("resolving {host}:{port}"))?; + let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(3)) + .with_context(|| format!("connecting to {url}"))?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(3)))?; + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: {host}:{port}\r\nAccept: text/plain\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n" + )?; + let mut response = Vec::new(); + stream + .take((MAX_METRICS_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut response)?; + if response.len() > MAX_METRICS_RESPONSE_BYTES { + bail!("metrics response exceeded {MAX_METRICS_RESPONSE_BYTES} bytes"); + } + let head_end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .context("metrics endpoint returned no HTTP header terminator")?; + let head = String::from_utf8_lossy(&response[..head_end]); + let status = head.lines().next().unwrap_or_default(); + if !status + .split_whitespace() + .nth(1) + .is_some_and(|code| code == "200") + { + bail!("metrics endpoint returned {status}"); + } + let body = &response[head_end + 4..]; + let body = if head + .lines() + .any(|line| line.eq_ignore_ascii_case("transfer-encoding: chunked")) + { + decode_chunked(body)? + } else { + body.to_vec() + }; + String::from_utf8(body).context("metrics response was not UTF-8") +} + +fn decode_chunked(mut body: &[u8]) -> Result> { + let mut decoded = Vec::new(); + loop { + let line_end = body + .windows(2) + .position(|window| window == b"\r\n") + .context("invalid chunked metrics response")?; + let size_text = std::str::from_utf8(&body[..line_end])? + .split(';') + .next() + .unwrap_or_default(); + let size = usize::from_str_radix(size_text.trim(), 16)?; + body = &body[line_end + 2..]; + if size == 0 { + break; + } + if body.len() < size + 2 { + bail!("truncated chunked metrics response"); + } + decoded.extend_from_slice(&body[..size]); + body = &body[size + 2..]; + } + Ok(decoded) +} + +fn parse_http_url(url: &str) -> Result<(String, u16, String)> { + let rest = url + .strip_prefix("http://") + .with_context(|| format!("telemetry metrics URL must use local http://, got '{url}'"))?; + let (authority, path) = rest.split_once('/').unwrap_or((rest, "")); + let (host, port) = authority + .rsplit_once(':') + .with_context(|| format!("metrics URL must include a port: '{url}'"))?; + let port = port.parse::().context("parsing metrics URL port")?; + if host.is_empty() { + bail!("metrics URL has an empty host"); + } + Ok((host.to_string(), port, format!("/{path}"))) +} + +fn parse_duration(value: &str) -> Result { + let split = value + .find(|ch: char| !ch.is_ascii_digit()) + .unwrap_or(value.len()); + let amount = value[..split] + .parse::() + .with_context(|| format!("invalid duration '{value}'"))?; + let unit = &value[split..]; + let seconds = match unit { + "s" => amount, + "m" => amount.saturating_mul(60), + "h" => amount.saturating_mul(60 * 60), + "d" => amount.saturating_mul(24 * 60 * 60), + "w" => amount.saturating_mul(7 * 24 * 60 * 60), + _ => bail!("duration must end in s, m, h, d, or w (for example 24h or 7d)"), + }; + Ok(Duration::from_secs(seconds)) +} + +fn unix_time_ms() -> Result { + let milliseconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock is before Unix epoch")? + .as_millis(); + Ok(milliseconds.min(i64::MAX as u128) as i64) +} + +fn fnv1a64(input: &[u8]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for byte in input { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_root() -> PathBuf { + let suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("lumbridge-telemetry-{suffix}")); + fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn prometheus_parser_keeps_escaped_bounded_labels() { + let metrics = parse_prometheus( + "# HELP ignored\nsglang:num_requests_total{model_name=\"brain\",agent=\"prime\\\"one\",stream=\"true\"} 12\nsglang:num_running_reqs 3\n", + ); + assert_eq!(metrics.len(), 2); + assert_eq!(metrics[0].name, "sglang:num_requests_total"); + assert_eq!(metrics[0].labels["agent"], "prime\"one"); + assert_eq!(metrics[1].value, 3.0); + } + + #[test] + fn counter_deltas_survive_runtime_resets() { + let root = temp_root(); + let mut store = Store::open(&root).unwrap(); + let target = Target { + model: "brain".to_string(), + scene: "solo".to_string(), + max_concurrency: 4, + config_fingerprint: "test".to_string(), + }; + let labels = BTreeMap::from([ + ("agent".to_string(), "a".to_string()), + ("client".to_string(), "c".to_string()), + ]); + let mut record = |value| { + store + .record_with_counter_interval( + &target, + &[ + Metric { + name: "sglang:num_running_reqs".to_string(), + labels: BTreeMap::new(), + value: 1.0, + }, + Metric { + name: "sglang:num_requests_total".to_string(), + labels: labels.clone(), + value, + }, + ], + 0, + ) + .unwrap(); + thread::sleep(Duration::from_millis(2)); + }; + record(10.0); + let since = unix_time_ms().unwrap() - 1; + record(14.0); + record(2.0); // runtime restarted + let summary = store.summary(since, unix_time_ms().unwrap()).unwrap(); + assert_eq!(summary.totals.requests, 6.0); + assert_eq!(summary.agents[0].requests, 6.0); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn new_labeled_series_count_from_first_use_and_identify_vision() { + let root = temp_root(); + let mut store = Store::open(&root).unwrap(); + let target = Target { + model: "brain".to_string(), + scene: "solo".to_string(), + max_concurrency: 4, + config_fingerprint: "test".to_string(), + }; + let gauge = Metric { + name: "sglang:num_running_reqs".to_string(), + labels: BTreeMap::new(), + value: 0.0, + }; + let baseline_labels = BTreeMap::from([ + ("agent".to_string(), "".to_string()), + ("client".to_string(), "".to_string()), + ("workload".to_string(), "".to_string()), + ]); + store + .record_with_counter_interval( + &target, + &[ + gauge.clone(), + Metric { + name: "sglang:num_requests_total".to_string(), + labels: baseline_labels.clone(), + value: 10.0, + }, + Metric { + name: "sglang:prompt_tokens_total".to_string(), + labels: baseline_labels, + value: 100.0, + }, + ], + 0, + ) + .unwrap(); + thread::sleep(Duration::from_millis(2)); + let since = unix_time_ms().unwrap() - 1; + let vision_labels = BTreeMap::from([ + ("agent".to_string(), "eye-1".to_string()), + ("client".to_string(), "cloud-agent".to_string()), + ("workload".to_string(), "vision".to_string()), + ]); + store + .record_with_counter_interval( + &target, + &[ + gauge, + Metric { + name: "sglang:num_requests_total".to_string(), + labels: vision_labels.clone(), + value: 1.0, + }, + Metric { + name: "sglang:prompt_tokens_total".to_string(), + labels: vision_labels, + value: 334.0, + }, + ], + 0, + ) + .unwrap(); + + let summary = store.summary(since, unix_time_ms().unwrap()).unwrap(); + assert_eq!(summary.totals.requests, 1.0); + assert_eq!(summary.totals.vision_requests, 1.0); + assert_eq!(summary.totals.vision_prompt_tokens, 334.0); + assert_eq!(summary.totals.vision_request_source, "workload_label"); + assert_eq!(summary.agents[0].agent, "eye-1"); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn duration_parser_is_explicit() { + assert_eq!(parse_duration("24h").unwrap(), Duration::from_secs(86_400)); + assert_eq!(parse_duration("7d").unwrap(), Duration::from_secs(604_800)); + assert!(parse_duration("24").is_err()); + } + + #[test] + fn caller_labels_are_bounded_and_cannot_carry_structured_data() { + assert_eq!(sanitize_custom_label(""), "invalid"); + assert_eq!(sanitize_custom_label("worker-1"), "worker-1"); + assert_eq!(sanitize_custom_label("person@example.com"), "invalid"); + assert_eq!(sanitize_custom_label(&"x".repeat(65)), "invalid"); + assert_eq!( + label_or_unknown( + &BTreeMap::from([("agent".to_string(), "invalid".to_string())]), + "agent" + ), + "unknown" + ); + } +} diff --git a/systemd/lumbridge-compute-agent-user.service b/systemd/lumbridge-compute-agent-user.service new file mode 100644 index 0000000..571630a --- /dev/null +++ b/systemd/lumbridge-compute-agent-user.service @@ -0,0 +1,19 @@ +[Unit] +Description=Lumbridge Compute resident Scene supervisor, gateway, and telemetry collector +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +Environment=HOME=%h +Environment=LUMBRIDGE_COMPUTE_ROOT=%h/lumbridge-prod/compute +ExecStart=%h/lumbridge-prod/target/release/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 --metrics-url http://127.0.0.1:8001/metrics --metrics-model auto --telemetry-interval-sec 5 +Restart=always +RestartSec=2 +# Model process groups are lifecycle-owned by Compute and must survive an agent +# binary restart; the new agent revalidates boot id, PID, and process start time. +KillMode=process +TimeoutStopSec=15 + +[Install] +WantedBy=default.target diff --git a/systemd/lumbridge-compute-agent.service b/systemd/lumbridge-compute-agent.service new file mode 100644 index 0000000..9bb2843 --- /dev/null +++ b/systemd/lumbridge-compute-agent.service @@ -0,0 +1,20 @@ +[Unit] +Description=Lumbridge Compute resident Scene supervisor and model gateway +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=lumbridge +Environment=HOME=/var/lib/lumbridge +Environment=LUMBRIDGE_COMPUTE_ROOT=$LUMBRIDGE_COMPUTE_ROOT +ExecStart=/usr/local/bin/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3 +Restart=always +RestartSec=2 +# Model process groups are lifecycle-owned by Compute and must survive an agent +# binary restart; the agent will revalidate their boot/start identities on resume. +KillMode=process +TimeoutStopSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/tools/backfill_voice.py b/tools/backfill_voice.py new file mode 100644 index 0000000..aab770f --- /dev/null +++ b/tools/backfill_voice.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""One-time backfill: seed voice.jsonl from WAV files that already existed in +~/chatterbox/generations before per-request logging was added to voice_api.py. +No chars/voice/variant metadata exists for these -- marked backfill:true. +Safe to re-run: skips gen_ids already present in voice.jsonl. +""" +import contextlib +import json +import os +import wave + +GEN_DIR = os.path.expanduser("~/chatterbox/generations") +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") +VOICE_LOG = os.path.join(USAGE_DIR, "voice.jsonl") + + +def already_logged(): + seen = set() + if os.path.exists(VOICE_LOG): + with open(VOICE_LOG) as f: + for line in f: + try: + seen.add(json.loads(line)["gen_id"]) + except Exception: + continue + return seen + + +def main(): + os.makedirs(USAGE_DIR, exist_ok=True) + seen = already_logged() + added = 0 + skipped = 0 + errors = 0 + with open(VOICE_LOG, "a") as out: + for fname in os.listdir(GEN_DIR): + if not fname.endswith(".wav"): + continue + gen_id = fname[:-4] + if gen_id in seen: + skipped += 1 + continue + path = os.path.join(GEN_DIR, fname) + try: + with contextlib.closing(wave.open(path, "rb")) as wf: + frames = wf.getnframes() + rate = wf.getframerate() + duration = frames / float(rate) if rate else 0.0 + except Exception: + errors += 1 + continue + record = { + "ts": os.path.getmtime(path), + "gen_id": gen_id, + "duration_s": duration, + "sr": rate, + "chars": None, + "voice": None, + "variant_used": None, + "gen_ms": None, + "backfill": True, + } + out.write(json.dumps(record) + "\n") + added += 1 + print(f"added={added} skipped(already logged)={skipped} errors={errors}") + + +if __name__ == "__main__": + main() diff --git a/tools/scrape_vllm_metrics.py b/tools/scrape_vllm_metrics.py new file mode 100755 index 0000000..8c1347d --- /dev/null +++ b/tools/scrape_vllm_metrics.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Scrape vLLM's own Prometheus /metrics on each locally-served model and append +one JSON snapshot line per reachable model to vllm_snapshots.jsonl. + +vLLM counters (prompt_tokens_total, generation_tokens_total, request_success_total) +reset to zero on every process restart -- this is what turns them into a durable +history. A model that's down is skipped for this tick, not an error: run this +every few minutes from cron and it just accumulates whatever was actually up. +""" +import json +import os +import re +import time +import urllib.request + +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") +SNAPSHOT_FILE = os.path.join(USAGE_DIR, "vllm_snapshots.jsonl") + +# model_name -> port, per registry/models.yaml +MODELS = { + "brain": 8001, + "embed": 8012, + "ocr": 8013, +} + +# Prometheus exposition line: metric{labels} value +# vLLM metric names contain a colon (vllm:prompt_tokens_total), which \w does not match. +LINE_RE = re.compile(r'^([\w:]+)(\{[^}]*\})?\s+([0-9eE+\-.]+)\s*$') + +COUNTERS = ( + "vllm:prompt_tokens_total", + "vllm:generation_tokens_total", + "vllm:num_requests_running", +) +# request_success_total is split by finished_reason -- sum all reasons. +SUCCESS_METRIC = "vllm:request_success_total" + + +def _fetch(port): + url = f"http://127.0.0.1:{port}/metrics" + with urllib.request.urlopen(url, timeout=3) as resp: + return resp.read().decode() + + +def _parse(text): + values = {name: 0.0 for name in COUNTERS} + success_total = 0.0 + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + m = LINE_RE.match(line) + if not m: + continue + name, value = m.group(1), m.group(3) + try: + value = float(value) + except ValueError: + continue + if name in values: + values[name] = value + elif name == SUCCESS_METRIC: + success_total += value + values[SUCCESS_METRIC] = success_total + return values + + +def main(): + os.makedirs(USAGE_DIR, exist_ok=True) + ts = time.time() + lines = [] + for model, port in MODELS.items(): + try: + text = _fetch(port) + except Exception: + continue # model is down -- skip silently, not an error + parsed = _parse(text) + record = { + "ts": ts, + "model": model, + "prompt_tokens_total": parsed["vllm:prompt_tokens_total"], + "generation_tokens_total": parsed["vllm:generation_tokens_total"], + "request_success_total": parsed[SUCCESS_METRIC], + "num_requests_running": parsed["vllm:num_requests_running"], + } + lines.append(json.dumps(record)) + if lines: + with open(SNAPSHOT_FILE, "a") as f: + for line in lines: + f.write(line + "\n") + print(f"scraped {len(lines)}/{len(MODELS)} models reachable") + + +if __name__ == "__main__": + main() diff --git a/tools/usage_report.py b/tools/usage_report.py new file mode 100755 index 0000000..f3ff942 --- /dev/null +++ b/tools/usage_report.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""the reference node usage report: rolls up the JSONL usage logs (voice/stt/image/music +generations) and the scraped vLLM token-counter snapshots into real numbers. + +Usage: + usage_report.py # human-readable report, last 14 days + usage_report.py --days 7 # narrower daily window + usage_report.py --json # machine-readable, same data +""" +import argparse +import datetime +import json +import os + +USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage") + +EVENT_LOGS = { + "voice": ("voice.jsonl", "duration_s"), + "stt": ("stt.jsonl", "audio_duration_s"), + "image": ("image.jsonl", None), + "music": ("music.jsonl", "duration_s"), +} + +VLLM_MODELS = ("brain", "embed", "ocr") +VLLM_COUNTERS = ("prompt_tokens_total", "generation_tokens_total", "request_success_total") + + +def _read_jsonl(path): + rows = [] + if not os.path.exists(path): + return rows + with open(path) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def _day(ts): + return datetime.datetime.fromtimestamp(ts).date().isoformat() + + +def summarize_events(name, filename, duration_field, days): + rows = _read_jsonl(os.path.join(USAGE_DIR, filename)) + cutoff = datetime.datetime.now().timestamp() - days * 86400 + by_day = {} + total_count = len(rows) + total_duration = 0.0 + for r in rows: + ts = r.get("ts") + if ts is None: + continue + dur = r.get(duration_field) or 0.0 if duration_field else 0.0 + total_duration += dur + day = _day(ts) + entry = by_day.setdefault(day, {"count": 0, "duration_s": 0.0}) + entry["count"] += 1 + entry["duration_s"] += dur + recent_days = {d: v for d, v in by_day.items() + if datetime.datetime.fromisoformat(d).timestamp() >= cutoff - 86400} + return { + "service": name, + "total_count": total_count, + "total_duration_s": round(total_duration, 1), + "total_duration_hours": round(total_duration / 3600, 2), + "has_duration": duration_field is not None, + "daily": dict(sorted(recent_days.items())), + } + + +def summarize_vllm(days): + rows = _read_jsonl(os.path.join(USAGE_DIR, "vllm_snapshots.jsonl")) + cutoff = datetime.datetime.now().timestamp() - days * 86400 + by_model = {m: [] for m in VLLM_MODELS} + for r in rows: + model = r.get("model") + if model in by_model: + by_model[model].append(r) + + result = {} + for model, snaps in by_model.items(): + snaps.sort(key=lambda r: r["ts"]) + current = snaps[-1] if snaps else None + tracked = {c: 0.0 for c in VLLM_COUNTERS} + prev = None + for s in snaps: + if s["ts"] < cutoff: + prev = s + continue + if prev is not None: + for c in VLLM_COUNTERS: + delta = s.get(c, 0) - prev.get(c, 0) + if delta > 0: + tracked[c] += delta + prev = s + result[model] = { + "reachable_now": current is not None and current["ts"] >= cutoff, + "snapshots_recorded": len(snaps), + "current": { + "prompt_tokens_total": current["prompt_tokens_total"], + "generation_tokens_total": current["generation_tokens_total"], + "request_success_total": current["request_success_total"], + } if current else None, + "tracked_since_monitoring_started": tracked, + } + return result + + +def render_text(events, vllm, days): + lines = [] + lines.append(f"=== the reference node usage report (last {days} days) ===\n") + + for e in events: + lines.append(f"-- {e['service']} --") + if e["has_duration"]: + lines.append(f" total: {e['total_count']} generations, " + f"{e['total_duration_hours']}h ({e['total_duration_s']}s)") + else: + lines.append(f" total: {e['total_count']} generations") + if not e["daily"]: + lines.append(" (no activity in this window)") + else: + for day, v in e["daily"].items(): + if e["has_duration"]: + lines.append(f" {day}: {v['count']:>5} gens, {round(v['duration_s']/60, 1):>7} min") + else: + lines.append(f" {day}: {v['count']:>5} gens") + lines.append("") + + lines.append("-- local LLM / vLLM token throughput --") + for model, v in vllm.items(): + if not v["current"]: + lines.append(f" {model}: not reachable (no snapshot ever recorded)") + continue + status = "up" if v["reachable_now"] else f"down (last seen in a prior snapshot)" + c = v["current"] + t = v["tracked_since_monitoring_started"] + lines.append(f" {model}: {status}") + lines.append(f" current counters (since last process restart): " + f"{int(c['prompt_tokens_total']):,} prompt tok, " + f"{int(c['generation_tokens_total']):,} gen tok, " + f"{int(c['request_success_total']):,} requests") + lines.append(f" tracked in window ({v['snapshots_recorded']} scrapes): " + f"{int(t['prompt_tokens_total']):,} prompt tok, " + f"{int(t['generation_tokens_total']):,} gen tok, " + f"{int(t['request_success_total']):,} requests") + lines.append("") + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--days", type=int, default=14) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + events = [summarize_events(name, filename, duration_field, args.days) + for name, (filename, duration_field) in EVENT_LOGS.items()] + vllm = summarize_vllm(args.days) + + if args.json: + print(json.dumps({"events": events, "vllm": vllm, "days": args.days}, indent=2)) + else: + print(render_text(events, vllm, args.days)) + + +if __name__ == "__main__": + main()