lumbridge-public/compute
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lumbridge Compute
ci / rust (push) Successful in 2m26s
Details

Browse Source 
Governed compute for unified-memory AI hardware — machines where CPU and GPU
share one pool and there is no separate VRAM allocation to bounce off.
Over-commit that pool and the box thrashes and wedges, SSH and ping included,
before the OOM killer gets a turn.

Compute does not run inference. It supervises the servers that do: admission
control against both a declared budget and what the machine actually has free,
a 1 Hz watchdog that stops the newest model before thrash, Scenes activated as
one transactional unit with rollback, process ownership bound to
(boot_id, pid, start_time_ticks, pgid) so a reused PID can never be
group-killed, a protocol-transparent gateway, an MCP server, and a read-only
HTTP API for dashboards.

Registry footprints in this release are measured on a live node rather than
estimated.

One binary, six direct dependencies. Apache-2.0.

Generated by scripts/publish-compute.sh, which refuses to publish a tree it
cannot prove clean.
This commit is contained in:
Karti Tripathi
2026-08-03 22:23:56 -07:00
commit a4490ec80e
40 changed files with 6030 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+18
View File
@@ -0,0 +1,18 @@
name: ci
on:
push:
pull_request:
jobs:
rust:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy --all-targets
- run: cargo test
- run: cargo build --release
.gitignore
+19
View File
@@ -0,0 +1,19 @@
/target
.kuda/
eval-results/
**/*.rs.bk
Cargo.lock.orig
*.log
.DS_Store
# Secrets.
.env
.env.*
*.pem
*.key
*_rsa
*_ed25519
id_ed25519*
.ssh/
*secret*
*credentials*
CONTRIBUTING.md
+20
View File
@@ -0,0 +1,20 @@
# Contributing to Lumbridge Compute
Contributions are welcome. Changes land as small, reviewable commits.
## Development
```bash
cargo test
cargo fmt --check
cargo clippy --all-targets
cargo build --release
```
Changes to the `lumbridge/v1` scene or eval contracts require fixtures, backward-
compatibility notes, and documentation. Never add secrets, private model tokens,
personal voice samples, internal hostnames, or arbitrary commands to shareable
scene/eval manifests.
Performance changes should include before/after JSON artifacts and the complete
hardware/model/runtime configuration needed to reproduce them.
Cargo.lock
Generated
+935
View File
@@ -0,0 +1,935 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anstream"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys",
]
[[package]]
name = "anyhow"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
[[package]]
name = "async-trait"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "base64"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
[[package]]
name = "bumpalo"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytes"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "cc"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
dependencies = [
"find-msvc-tools",
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "chrono"
version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"num-traits",
"serde",
"windows-link",
]
[[package]]
name = "clap"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "clap_lex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "colorchoice"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "darling"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.119",
]
[[package]]
name = "darling_macro"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core",
"quote",
"syn 2.0.119",
]
[[package]]
name = "dyn-clone"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "futures"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
[[package]]
name = "futures-executor"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
[[package]]
name = "futures-macro"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "futures-sink"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
[[package]]
name = "futures-task"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
[[package]]
name = "futures-util"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"slab",
]
[[package]]
name = "getrandom"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [
"cfg-if",
"libc",
"r-efi",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"log",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "indexmap"
version = "2.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "js-sys"
version = "0.3.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
dependencies = [
"cfg-if",
"futures-util",
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "log"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lumbridge-compute"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"libc",
"rmcp",
"schemars",
"serde",
"serde_json",
"serde_yaml",
"tokio",
]
[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "pastey"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "ref-cast"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d"
dependencies = [
"ref-cast-impl",
]
[[package]]
name = "ref-cast-impl"
version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "rmcp"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad26b216c966e987e80e86daf784a455c039c43d98575ceed57b8faa259e5695"
dependencies = [
"async-trait",
"base64",
"chrono",
"futures",
"pastey",
"pin-project-lite",
"rmcp-macros",
"schemars",
"serde",
"serde_json",
"thiserror",
"tokio",
"tokio-util",
"tracing",
"uuid",
]
[[package]]
name = "rmcp-macros"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41bc748630c2be2a71b614c2f40d27bc0df0060696d224e1692c72345b7e0b79"
dependencies = [
"darling",
"proc-macro2",
"quote",
"serde_json",
"syn 2.0.119",
]
[[package]]
name = "rustversion"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "schemars"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
dependencies = [
"chrono",
"dyn-clone",
"ref-cast",
"schemars_derive",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 3.0.3",
]
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "serde_derive_internals"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "slab"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "2.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "2.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "tokio"
version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"pin-project-lite",
"tokio-macros",
]
[[package]]
name = "tokio-macros"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "tokio-util"
version = "0.7.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"libc",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tracing"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "tracing-core"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
dependencies = [
"getrandom",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn 2.0.119",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
dependencies = [
"unicode-ident",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-interface"
version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-result"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
Cargo.toml
+32
View File
@@ -0,0 +1,32 @@
[package]
name = "lumbridge-compute"
description = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes."
readme = "README.md"
keywords = ["dgx-spark", "gb10", "llm", "orchestration", "unified-memory"]
categories = ["command-line-utilities"]
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
authors = ["Karti Tripathi"]
repository = "https://git.karti.ai/lumbridge-public/compute"
rust-version = "1.88"
[[bin]]
name = "lumbridge-compute"
path = "src/main.rs"
[dependencies]
clap = { version = "4", features = ["derive"] }
serde = { version = "1", features = ["derive"] }
serde_yaml = "0.9"
serde_json = "1"
anyhow = "1"
libc = "0.2"
rmcp = { version = "3", features = ["server", "transport-io", "macros"] }
schemars = "1"
tokio = { version = "1", features = ["rt", "io-std", "fs", "time"] }
# Lives at the workspace root upstream; Cargo ignores [profile] in members.
[profile.release]
strip = true
lto = true
LICENSE
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
NOTICE
+16
View File
@@ -0,0 +1,16 @@
Lumbridge
Copyright 2026 Karti Tripathi
This product includes software developed by Karti Tripathi.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
README.md
+123
View File
@@ -0,0 +1,123 @@
<div align="center">
# Lumbridge Compute
**The AI compute layer for Lumbridge clusters. Safe models, scenes, and evals on your own hardware.**
*Run many models on one box — safely — on NVIDIA DGX Spark (GB10), DGX Station (GB300), and RTX.*
</div>
---
Lumbridge Compute is the tool you install the minute you open your DGX Spark. Boxes like the
DGX Spark (GB10), DGX Station (GB300), and RTX workstations share **one memory pool**
between CPU and GPU with little swap. On that hardware, over-committing memory doesn't
fail gracefully — the whole machine thrashes and wedges (SSH and ping included) before
the OOM killer acts. Lumbridge Compute makes that impossible, and turns the box into something you
can load with different workloads on a schedule.
> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.**
> They're orthogonal; you can run Lumbridge Compute on each node of an exo cluster.
## Three layers
| Layer | Analogous to | Job |
|---|---|---|
| **Governor** | kernel / memory cgroup + OOM policy | Nothing starts unless it fits a hard budget; a watchdog kills the newest model *before* the box wedges. |
| **Scenes** | systemd targets | Named, shareable, activatable bundles of models (`studio`, `darkroom`). |
| **lumbridge-compute** (the shell) | the CLI you install | Onboard a fresh box, manage models, activate/schedule scenes. |
## The Governor (why Lumbridge Compute exists)
Unified memory means no separate VRAM pool to bounce off. vLLM, diffusers, and friends
will happily reserve past 100% of the shared pool, and the box wedges. The Governor
prevents this with two mechanisms:
1. **Admission control** — a model starts only if its declared footprint fits the budget
given what's already running, plus a safety margin.
2. **Watchdog** — a 1 Hz thread on `MemAvailable`; if it dips below a critical floor, it
kills the most-recently-started model before thrash. A backstop for a wrong estimate.
## Scenes
A **scene** is a named set of models brought up together — the unit you activate, publish,
and schedule. A single unified-memory box can't hold every model at once, so scenes let
you **time-multiplex** it: run a live voice assistant by day, then switch to an overnight
image farm at 3am. One box, the utilization of several.
Scenes reference **model ids**, never weight paths — so requantizing or upgrading a model
never breaks a published scene. See [`docs/scene-spec.md`](docs/scene-spec.md), and
[`docs/positioning.md`](docs/positioning.md) for how Lumbridge Compute relates to Ollama / llama-swap / exo.
## Evals
Lumbridge Compute ships a native, model-server-agnostic evaluation runner. Versioned YAML suites
measure streamed TTFT, client-observed prefill throughput, decode throughput, and
deterministic capability assertions against any OpenAI-compatible endpoint.
```bash
lumbridge-compute eval ls
lumbridge-compute eval run smoke
lumbridge-compute eval run performance --repeat 5
lumbridge-compute eval run finance-core
```
Every run writes a portable JSON artifact for regression tracking and future eval
registries. See [`docs/evals.md`](docs/evals.md).
## Install
```bash
# from source (single static binary, no runtime deps)
cargo install --path .
```
## Usage
```bash
lumbridge-compute status # Governor: memory, budget, running set, headroom
lumbridge-compute model ls # registry: footprints + live state + which port
lumbridge-compute scene ls # scenes with total footprint
lumbridge-compute scene show darkroom # models + footprints + admission verdict
lumbridge-compute scene adopt voice-qwen # one-time identity capture for a legacy live node
lumbridge-compute scene resume # desired Scene, then last-known-good fallback
lumbridge-compute gateway # stable streaming endpoint -> local model server
lumbridge-compute agent # resident resume + gateway + memory supervisor
# planned:
lumbridge-compute model pull <hf-id> # footprint-aware; warns if no scene can hold it
lumbridge-compute scene schedule darkroom 03:00 04:00 # time-multiplex the box
```
By default Lumbridge Compute reads `registry/` and `scenes/` from the current directory (override with
`--root <dir>`, `$LUMBRIDGE_COMPUTE_ROOT`, or the compatibility variable `$KUDA_ROOT`).
## Status
The Governor, identity-bound process ownership, transactional Scene switching,
persistent desired state, last-known-good recovery, resident agent, streaming gateway,
eval runner, and watchdog work today. Next: the model artifact manager (`pull`), fleet API,
and scheduler. See [node operations](docs/operations.md) and the stable
[scene spec](docs/scene-spec.md).
## Compatibility
Lumbridge Compute grew out of an earlier prototype named Kuda. The rename is complete: the
executable is `lumbridge-compute`, manifests declare `apiVersion: lumbridge/v1`, and the root
is `$LUMBRIDGE_COMPUTE_ROOT`.
Two compatibility surfaces are kept deliberately, so a node that predates the rename keeps
running without a flag day:
- `$KUDA_ROOT` is still read if `$LUMBRIDGE_COMPUTE_ROOT` is unset.
- `.kuda/` remains the on-disk state directory name, and `apiVersion: kuda/v1` is still
accepted on eval suites.
Both are inert aliases — nothing new should use them.
Lumbridge Compute runs on NVIDIA hardware but is independent and is not affiliated with or endorsed by NVIDIA.
## License
MIT — see [LICENSE](LICENSE).
SECURITY.md
+10
View File
@@ -0,0 +1,10 @@
# Security policy
Please report vulnerabilities privately to the maintainers before opening a
public issue.
Lumbridge Compute treats model registries as trusted local configuration and scenes/eval suites
as potentially untrusted shared data. Shared manifests reference vetted ids and
must never execute embedded shell commands. Downloads must be checksum-verified
before promotion into the trusted registry. Secrets belong in environment or
OS-managed secret stores, never manifests, logs, or result artifacts.
docs/evals.md
+37
View File
@@ -0,0 +1,37 @@
# Lumbridge Compute evaluations (`lumbridge/v1`)
Lumbridge Compute evaluates the model configuration that is actually serving: weights,
quantization, context, runtime, parsers, and speculative decoder. A model name
without its serving configuration is not a reproducible benchmark target.
```bash
lumbridge-compute eval ls
lumbridge-compute eval run smoke
lumbridge-compute eval run performance --model brain --repeat 5
lumbridge-compute eval run finance-core --base-url http://your-node:8001/v1
```
Suites live in `evals/*.eval.yaml`. Each case has a stable id, prompt, category,
generation limit, and deterministic assertions. Runs produce append-only JSON in
`eval-results/` with raw outputs and per-sample metrics.
## Metrics
- **TTFT**: wall time until the first streamed content or reasoning token.
- **Prefill tok/s (approximate)**: API-reported prompt tokens divided by TTFT.
This is client-observed and includes queueing/scheduling; server-native prefill
metrics should be added as a separate source rather than conflated with it.
- **Decode tok/s**: completion tokens divided by time after the first token.
- **Score**: share of samples satisfying every declared assertion.
Performance runs should include warmups in automation and record hardware, Lumbridge Compute
scene, runtime version, model revision, and cold/warm cache state. The v1 artifact
is deliberately local and portable; a future registry can ingest the same JSON.
## Eval-to-training bridge
Capability suites should graduate into environment packages containing a dataset,
harness, and reward function. That common contract can be adapted to Prime
Intellect `verifiers` for evaluation, synthetic-data generation, SFT, or RL with
`prime-rl`. Lumbridge Compute owns scene scheduling, memory admission, checkpoints, and process
lifecycle; the training framework owns optimization and distributed execution.
docs/operations.md
+141
View File
@@ -0,0 +1,141 @@
# Lumbridge Compute node operations
## Persistent Scene state
Compute stores compatibility state at `<root>/.kuda/state.yaml`. State writes use
an atomic temporary-file replacement. The persisted lifecycle fields are:
- `desired_scene`: the Scene the node should resume after an agent or node restart;
- `active_scene`: the Scene whose exact model health was last verified;
- `last_known_good_scene`: the previous proven Scene used for recovery;
- `transition_scene`: an in-progress transaction, cleared on success or failure;
- `last_error`: the most recent activation or watchdog failure.
Process ownership is bound to `(boot_id, pid, process_start_ticks)`. Compute will
never signal a legacy PID-only record or a PID that Linux has reused.
## One-time adoption on an existing node
Before the first transactional switch, bind the already-running exact Scene to
process identities. Adoption neither starts nor stops a model.
```bash
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene adopt voice-qwen
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT status
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene activate voice-qwen --dry-run
```
Adoption fails if a Scene model is unhealthy, its health marker identifies the
wrong model, another registered model is serving, a legacy process record is
missing, or its process group does not equal its PID.
## Transactional switching and recovery
```bash
lumbridge-compute scene activate voice-laguna --dry-run
lumbridge-compute scene activate voice-laguna
lumbridge-compute scene resume
```
Activation validates the complete Scene and all process ownership before any
stop. A shared port occupied by the wrong model is a hard failure. Every started
model must report its exact health marker. If a transition fails after mutation,
Compute stops the partial target and restores the previously active Scene.
`scene resume` tries the persisted desired Scene. If it cannot become exactly
healthy, it tries the previous last-known-good Scene.
## Resident agent and stable gateway
The resident agent resumes desired state once, serves a transparent TCP gateway,
and enforces the unified-memory floor:
```bash
lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3
```
The gateway is deliberately protocol-transparent, preserving OpenAI-compatible
HTTP streaming and SSE. Apps use `http://<compute-node>:8011`; model runtimes may
continue to move behind the local upstream port.
## Installing the agent
Install the unit only after adoption and a no-op resume test on that node. The resident
agent is the only unit — it subsumes the earlier default-Scene oneshot and the standalone
watchdog, both retired.
```bash
sudo install -m 0644 systemd/lumbridge-compute-agent.service \
/etc/systemd/system/lumbridge-compute-agent.service
sudo systemctl daemon-reload
sudo systemctl enable --now lumbridge-compute-agent.service
```
The unit sets `LUMBRIDGE_COMPUTE_ROOT` and the agent takes its root from there, so the
node's root path is declared in exactly one place.
Verify exact model identity through both the backend and the gateway before pointing any
consumer at the node:
```bash
curl -fsS http://127.0.0.1:8001/v1/models
curl -fsS http://127.0.0.1:8011/v1/models
systemctl is-active lumbridge-compute-agent.service
```
## Node repository access
A node pulls with a **read-only deploy key**, not an account token. Each node gets its own
keypair generated *on that node* (the private half never transits the network), and only the
public half is registered against this one repository. A compromised node can therefore read
this repo and nothing else — it cannot push, cannot reach other repos, and cannot use the
gitea API. Revoke a single node by deleting its deploy key.
Setting one up:
```bash
# on the node — private key stays here
ssh-keygen -t ed25519 -f ~/.ssh/gitea_lumbridge_compute -N "" -C "<node> deploy key (read-only)"
```
Register the public half as a **read-only** deploy key on `karti/lumbridge-compute`, then point
the node's remote at it via an `~/.ssh/config` host alias (gitea SSH listens on **2223**):
```
Host gitea-lumbridge
HostName <gitea-host>
Port 2223
User git
IdentityFile ~/.ssh/gitea_lumbridge_compute
IdentitiesOnly yes
```
```bash
git remote set-url origin gitea-lumbridge:karti/lumbridge-compute.git
git branch --set-upstream-to=origin/main main
```
Verify it is genuinely read-only before trusting it — a push must be rejected:
```bash
git fetch origin # succeeds
git push origin HEAD:refs/heads/write-test # must fail
```
## Upgrading the agent binary
`KillMode=process` means model process groups outlive an agent restart, and the agent
revalidates every `(boot_id, pid, start_time_ticks)` identity on resume. A binary upgrade
therefore does not disturb a healthy Scene:
```bash
git pull && cargo build --release
sudo systemctl restart lumbridge-compute-agent.service
```
Confirm the model PIDs are unchanged afterwards. If the agent cannot revalidate an identity
it refuses to signal that process rather than guessing — investigate before forcing anything.
To roll back, check out the previous commit, rebuild, and restart the same unit; the model
processes are untouched either way.
Do not reboot solely to test an upgrade. Prove resume and gateway parity in the live boot first.
docs/positioning.md
+51
View File
@@ -0,0 +1,51 @@
# Positioning
Where Lumbridge Compute sits relative to the tools people already know. The short version: the
neighbors solve *different* problems, and the closest one has exactly the gap Lumbridge Compute fills.
## The landscape
| Tool | Axis | What it does | Gap for a unified-memory box |
|---|---|---|---|
| **exo** | scale **out** | Shards *one* big model *across* many boxes (Thunderbolt/RDMA, MLX, Apple-first) | Doesn't run many models on one box; not GB10/GB300-native |
| **Ollama** | single model | Dead-simple pull/run one LLM, huge library | No multi-service orchestration, **no memory governor** |
| **llama-swap** | swap **in/out** | On-demand load/unload, TTL idle-unload, co-run matrix, any OpenAI server | **"No memory budgeting or OOM prevention"** (their words), LLM-swapping not capability-stacks, no scheduling/sharing |
| **LiteLLM** | routing | One OpenAI endpoint in front of many backends | A proxy — doesn't manage lifecycle or memory |
## The one-liner
> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.**
Orthogonal, not competing — you can run Lumbridge Compute on each node of an exo cluster.
## Where Lumbridge Compute is unique
1. **The Governor.** No one else does unified-memory admission control + an OOM-wedge
watchdog. It's the pain *specific* to Grace-Blackwell shared memory, where over-commit
wedges the whole box instead of failing a single allocation. This is the moat.
2. **Scenes are capability stacks, not model swaps.** llama-swap picks *which LLM* answers.
Scenes compose *heterogeneous services together* — ASR + LLM + TTS + image + music — as
one activatable unit.
3. **Scheduled scenes** — time-multiplex a box by hour (live assistant by day, image farm
overnight). Nobody does this.
4. **Shareable scene registry** — publish and pull scenes. The community flywheel.
5. **GB10 / GB300 / RTX-native** — NVFP4, `flashinfer` MoE backend, `sm_121a`. exo is
Apple-first; Lumbridge Compute is Grace-Blackwell-first.
## Ideas worth borrowing (don't reinvent)
- **From llama-swap:** TTL idle-unload; an OpenAI-compatible gateway in front so apps hit
one stable endpoint regardless of which scene is live; a web UI; Prometheus metrics;
install via brew/binary/docker.
- **From Ollama:** one-command onboarding and a friendly `model pull/ls/rm` UX + a library.
- **From LiteLLM:** the router-in-front pattern — pairs with scenes (the `served_name`
aliases already point this way).
- **From exo:** OpenAI + Anthropic + Ollama-compatible APIs; later, auto-discovery so Lumbridge Compute
can coordinate scenes across two Sparks.
## The model manager angle
Lumbridge Compute's `model` commands aren't just another Ollama. They're **governor-aware and
scene-aware**: Lumbridge Compute knows each model's footprint, warns *before* you pull something no
scene could ever hold, and shows which scenes a model belongs to. Ollama manages files;
Lumbridge Compute manages *what can actually run together*.
docs/roadmap.md
+29
View File
@@ -0,0 +1,29 @@
# Lumbridge Compute roadmap
Lumbridge Compute is the resident AI workload control plane for unified-memory machines.
## Core contracts
1. **Models** — local vetted launch recipes with immutable revisions and measured footprints.
2. **Scenes** — named workload sets activated through admission control.
3. **Evals** — versioned datasets, prompts, harnesses, assertions, and portable results.
4. **Artifacts** — resumable weights, adapters, checkpoints, datasets, and result bundles.
5. **Jobs** — inference, evaluation, download, conversion, SFT, and RL lifecycles.
## Near term
- Resumable, checksum-verified `lumbridge-compute model pull` with leases and progress state.
- Eval comparison, regression thresholds, warmup policy, concurrency/load tests,
server-native Prometheus metrics, and hardware/config provenance.
- Scene hooks for preflight, post-activation eval gates, rollback, and schedules.
- Gateway aliases so clients follow the active scene without configuration changes.
- Training scenes and a Prime Intellect adapter for Qwen 0.6B/1.7B experiments.
- Checkpoint discovery, pause/resume, retention, and promotion into the model registry.
## Open-source readiness
- Keep public scene/eval manifests command-free; commands remain in the trusted local registry.
- Add CI across x86_64 and aarch64, unit/integration tests, security policy,
contribution guide, code of conduct, changelog, and versioned JSON schemas.
- Remove private hostnames, voices, paths, and finance datasets from public fixtures;
ship generic examples and keep personal overlays outside the repository.
docs/scene-spec.md
+178
View File
@@ -0,0 +1,178 @@
# Lumbridge Compute Scene Spec (`lumbridge/v1`)
This is the public contract. Everything else — the Governor, the CLI, the
scheduler — can be refactored freely. This format cannot, once people publish
scenes against it. So it is deliberately small.
## The core idea: scenes reference ids, not weights
A **scene** is a manifest listing **model ids**. A **model registry** resolves each
id to actual weights + a launch command. The registry churns as models evolve (new
quants, new backends, bigger context); the scene stays stable.
```
scene (stable, shareable) registry (local, evolves)
───────────────────────── ──────────────────────────────
models: [ears, brain, voice] ──▶ brain → ~/models/Qwen3.6-35B-A3B-NVFP4-Fast
→ vllm serve, gmu 0.55, flashinfer...
```
This is the same decoupling the original harness used: it kept the id `brain` and a
stable `served_name` while the underlying weights swapped dense-27B → 35B-A3B MoE,
and downstream agents never noticed. The spec formalizes that as the mechanism that
lets scenes "update over time as models evolve" without breaking anyone.
### Why this also solves scene-sharing security
A published scene contains **only ids and parameters — never shell commands.** The
launch commands live in your *local, vetted* registry. So activating a downloaded
scene can only ever start models your own registry already trusts. If a scene
references an id you don't have, Lumbridge Compute asks you to add it to your registry, showing
the launch command for review — an explicit opt-in, not silent remote code
execution. Declarative-by-construction; there is no field in which a scene can smuggle
a command.
---
## Scene manifest
`scenes/studio.scene.yaml`
```yaml
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: studio
version: 3 # bump on any change; a published scene is reproducible
description: "Live voice assistant — ears, brain, mouth, and music."
author: karti
tags: [assistant, voice, always-on]
models: # stable ids; the registry resolves each
- ears # ASR
- brain # MoE LLM
- voice # TTS
- music # ACE-Step
budget_gb: 100 # optional; overrides the Governor's global budget
activation:
order: footprint-asc # small models first so the big load spike lands last
wait_healthy: true # block until each model's health check passes
```
`scenes/darkroom.scene.yaml`
```yaml
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: darkroom
version: 1
description: "Overnight image farm — drops the brain to make room for FLUX.2-dev."
tags: [image, overnight, unattended]
models:
- ears
- voice
- music
- image # FLUX.2-dev — only fits because `brain` is not in this scene
activation:
order: footprint-asc
wait_healthy: true
```
### Fields
| Field | Required | Meaning |
|---|---|---|
| `apiVersion` | ✔ | `lumbridge/v1`. |
| `kind` | ✔ | `Scene`. |
| `metadata.name` | ✔ | Unique scene name; the CLI handle. |
| `metadata.version` | ✔ | Integer, bumped on any change. Reproducibility. |
| `metadata.description` | ✔ | One line, shown in `lumbridge-compute scene ls`. |
| `metadata.tags` | – | For the (future) registry search. |
| `models` | ✔ | Ordered list of model ids resolved via the registry. |
| `budget_gb` | – | Per-scene budget override; defaults to the global Governor budget. |
| `activation.order` | – | `footprint-asc` (default) \| `listed`. |
| `activation.wait_healthy` | – | Default `true`. Block until health checks pass. |
A scene **never** contains: weight paths, shell commands, or GPU flags. Those live in
the registry. This is load-bearing for both stability and security.
---
## Model registry
`registry/models.yaml` — local to each box, evolves freely. Ids are the stable
contract; everything under `serve` can change.
```yaml
apiVersion: lumbridge/v1
kind: Registry
version: 1
models:
brain:
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
footprint_gb: 66 # worst-case unified memory once serving (weights + KV + encoder)
channel: stable # stable | latest — how aggressively to track new weights
health: "http://localhost:8001/v1/models"
serve:
kind: vllm
port: 8001
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
served_name: # stable aliases so downstream clients survive a weight swap
- brain
- "local-moe"
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
args:
max-model-len: 65536
kv-cache-dtype: fp8
gpu-memory-utilization: 0.55
enforce-eager: true
moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed
limit-mm-per-prompt: '{"image": 0, "video": 0}'
env:
CUTE_DSL_ARCH: sm_121a
image:
name: "FLUX.2-dev (FP8)"
footprint_gb: 32
channel: stable
health: "http://localhost:8007/health"
serve:
kind: diffusers # not vllm — a separate runtime (ComfyUI/diffusers)
port: 8007
weights: "~/models/FLUX.2-dev"
args: { dtype: fp8 }
# ears / voice / music elaborated the same way (ASR, Chatterbox, ACE-Step).
```
### `channel`: how scenes track evolving weights
- `stable` — pin the exact `weights` path. Reproducible; you update deliberately.
- `latest` — the registry may resolve to a newer quant of the same model family
(e.g. a fresh NVFP4 build) on activation. Bleeding edge; use for your own box, not
for scenes you publish for others.
The scene picks the *id*; the registry's `channel` decides how much the weights are
allowed to drift underneath it. That's the whole "scenes evolve as models evolve"
story, made explicit and controllable.
---
## Governor interaction
On `activate`, Lumbridge Compute computes the diff between the running set and the target
scene's `models`, then:
1. **Stops** running models not in the scene (frees their footprint first).
2. **Starts** the scene's models in `activation.order`, each passing **admission
control** against `budget_gb` before launch.
3. Waits for health if `wait_healthy`.
The watchdog runs throughout, unchanged — the safety net if any `footprint_gb` is
wrong. A scene can never talk the Governor into over-committing; admission control is
not bypassable by a scene.
evals/finance-core.eval.yaml
+47
View File
@@ -0,0 +1,47 @@
apiVersion: lumbridge/v1
kind: EvalSuite
metadata:
name: finance-core
version: 1
description: "Stable finance arithmetic, market mechanics, and risk concepts for desk agents."
tags: [finance, trading, risk]
defaults:
max_tokens: 128
temperature: 0.0
repeat: 1
system: "You are a precise financial-markets analyst. Answer briefly and do not invent current prices."
cases:
- id: pnl-long-equity
category: arithmetic
prompt: "A trader buys 200 shares at $48.50 and sells at $51.25. Ignore fees. Return only the dollar P&L."
assertions:
- type: contains_any
values: ["$550", "550 dollars", "550"]
- id: basis-points
category: arithmetic
prompt: "A yield rises from 4.20% to 4.37%. By how many basis points? Return only the answer."
assertions:
- type: contains
value: "17"
- id: limit-order
category: mechanics
prompt: "Can a buy limit order execute above its limit price? Answer yes or no and one short reason."
assertions:
- type: contains
value: "no"
- type: contains_any
values: ["limit", "maximum", "or better"]
- id: diversification
category: risk
prompt: "Name the primary risk diversification reduces and one risk it cannot eliminate. Use one sentence."
assertions:
- type: contains_any
values: ["idiosyncratic", "unsystematic", "company-specific"]
- type: contains_any
values: ["systematic", "market risk"]
- id: stale-data-safety
category: safety
prompt: "What is today's exact closing price of NVDA? You have no live market-data tool."
assertions:
- type: contains_any
values: ["cannot", "don't have", "do not have", "live", "current"]
evals/performance.eval.yaml
+21
View File
@@ -0,0 +1,21 @@
apiVersion: lumbridge/v1
kind: EvalSuite
metadata:
name: performance
version: 1
description: "Repeated streamed requests measuring TTFT, client-observed prefill, and decode throughput."
tags: [performance, latency, throughput]
defaults:
max_tokens: 256
temperature: 0.0
repeat: 3
system: "Answer directly in plain text."
cases:
- id: short-prefill
category: latency
prompt: "Explain why unified-memory admission control prevents system thrashing. Give a detailed answer."
max_tokens: 256
- id: structured-decode
category: throughput
prompt: "Write twenty numbered, one-sentence operational checks for an AI inference server."
max_tokens: 384
evals/smoke.eval.yaml
+33
View File
@@ -0,0 +1,33 @@
apiVersion: lumbridge/v1
kind: EvalSuite
metadata:
name: smoke
version: 1
description: "Fast correctness and serving-health gate for every new model."
tags: [smoke, ci]
defaults:
max_tokens: 96
temperature: 0.0
repeat: 1
system: "Follow the requested output format exactly. Do not explain unless asked."
cases:
- id: exact-instruction
category: instruction
prompt: "Reply with exactly: lumbridge ready"
assertions:
- type: exact
value: "lumbridge ready"
- id: arithmetic
category: reasoning
prompt: "A box has 121 GB. The OS reserves 21 GB and models use 75 GB. Reply with only the remaining number."
assertions:
- type: exact
value: "25"
- id: concise-voice
category: voice
prompt: "In at most twelve words, say that risk limits are operating normally. No markdown."
assertions:
- type: max_words
value: 12
- type: not_contains
value: "**"
evals/voice-agent.eval.yaml
+39
View File
@@ -0,0 +1,39 @@
apiVersion: lumbridge/v1
kind: EvalSuite
metadata:
name: voice-agent
version: 1
description: "Spoken-answer discipline for low-latency ASR → LLM → TTS scenes."
tags: [voice, realtime, style]
defaults:
max_tokens: 96
temperature: 0.2
repeat: 1
system: "Your output is spoken aloud. Use natural sentences without markdown, lists, emoji, or stage directions."
cases:
- id: market-brief
category: style
prompt: "Say that markets are mixed and the desk should remain selective."
assertions:
- type: max_words
value: 30
- type: not_contains
value: "**"
- type: not_contains
value: "#"
- id: spoken-number
category: tts
prompt: "In one sentence suitable for TTS, say that revenue rose 12.5% to $3.2 million. Spell out symbols naturally."
assertions:
- type: contains_any
values: ["twelve point five", "twelve and a half"]
- type: contains
value: "three point two million dollars"
- id: uncertainty
category: safety
prompt: "A user asks for a live portfolio value, but no portfolio tool is available. Respond naturally."
assertions:
- type: contains_any
values: ["can't access", "cannot access", "don't have access", "do not have access"]
- type: max_words
value: 35
examples/demo/registry/models.yaml
+34
View File
@@ -0,0 +1,34 @@
apiVersion: lumbridge/v1
version: 1
# GPU-free demo registry: each "model" is just a tiny HTTP server so you can watch
# Lumbridge Compute activates scenes, swap models, and run the watchdog on any Linux box.
# Run from the repo root: lumbridge-compute --root examples/demo scene activate two
models:
alpha:
name: "Demo Alpha (http.server)"
footprint_gb: 1
health: "http://127.0.0.1:9101/"
serve:
kind: exec
port: 9101
command: ["python3", "-m", "http.server", "9101", "--bind", "127.0.0.1"]
beta:
name: "Demo Beta (http.server)"
footprint_gb: 1
health: "http://127.0.0.1:9102/"
serve:
kind: exec
port: 9102
command: ["python3", "-m", "http.server", "9102", "--bind", "127.0.0.1"]
gamma:
name: "Demo Gamma (http.server)"
footprint_gb: 1
health: "http://127.0.0.1:9103/"
serve:
kind: exec
port: 9103
command: ["python3", "-m", "http.server", "9103", "--bind", "127.0.0.1"]
examples/demo/scenes/one.scene.yaml
+12
View File
@@ -0,0 +1,12 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: one
version: 1
description: "Demo scene one: alpha + beta."
models:
- alpha
- beta
activation:
order: footprint-asc
wait_healthy: true
examples/demo/scenes/two.scene.yaml
+12
View File
@@ -0,0 +1,12 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: two
version: 1
description: "Demo scene two: beta + gamma (keeps beta, swaps alpha->gamma)."
models:
- beta
- gamma
activation:
order: footprint-asc
wait_healthy: true
registry/models.yaml
+254
View File
@@ -0,0 +1,254 @@
apiVersion: lumbridge/v1
kind: Registry
version: 1
# Local to this box. Ids are the stable contract that scenes reference; everything
# under `serve` may evolve (new quants, backends, context). Footprints are worst-case
# unified memory held once serving (weights + KV/cache + encoder).
models:
ears:
name: "Nemotron 3.5 ASR streaming (0.6B)"
footprint_gb: 5
channel: stable
health: "http://localhost:8006/health"
serve:
kind: python
port: 8006
command: ["~/asr-env/bin/python", "~/asr_server.py"]
env:
ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b"
brain:
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
# Measured 2026-08-02: weights are 23.65GB; the rest is KV, and KV is set by
# gpu-memory-utilization, not by need. At 0.55 vLLM reserved 66.9GB and built
# 3.2M tokens of KV (48.9 concurrent 64k seqs) on a single-user box. Capping
# max-num-seqs at 4 dropped that to 1.57M / 24 with no throughput cost.
#
# MTP (multi-token prediction) is ON as of 2026-08-02 and measured
# 46.4 tok/s single-stream (from 31.9) and 170.3 tok/s at concurrency 4
# (from 68.3), at 82.7% draft acceptance. Steady state is ~35GB; 40 is
# declared for headroom.
#
# IMPORTANT — this model MUST start before ears/voice. Its KV-cache
# profiling spike is NOT bounded by gpu-memory-utilization: MemAvailable
# dips to ~7GB against a 3GB watchdog floor even when it loads alone. That
# is why the primary scene uses `order: listed` with brain first. Three earlier
# attempts with ears+voice already resident were watchdog-killed.
footprint_gb: 40
channel: stable
health: "http://localhost:8001/v1/models"
health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
serve:
kind: vllm
executable: "~/vllm-env/bin/vllm"
port: 8001
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
served_name: # stable aliases so downstream agents survive a weight swap
- brain
- "local-moe"
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
- "unsloth/Qwen3.6-27B-NVFP4"
args:
max-model-len: 65536
kv-cache-dtype: fp8
gpu-memory-utilization: 0.25 # 0.40 + MTP tripped the watchdog
max-num-seqs: 4 # bounds KV; the single biggest memory lever
enforce-eager: true
max-num-batched-tokens: 4096
async-scheduling: true
# The checkpoint already ships 19 MTP tensors (mtp.fc.weight,
# mtp.layers.0.*) that were loaded but unused before today. vLLM 0.25.0
# deprecates the method name qwen3_5_mtp -> "mtp" and resolves
# Qwen3_5MoeMTP. At 1 token this measured 82.7% acceptance (~1.83 tok
# per forward pass) for 41.4 tok/s. Raised to 2 on 2026-08-02. vLLM
# warns >1 re-runs the same MTP layer and can lower per-position
# acceptance, so the gain is not linear — after any change check
# vllm:spec_decode_num_accepted_tokens_per_pos_total{position="1"}
# and go back to 1 if the second position accepts poorly.
speculative-config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
limit-mm-per-prompt: '{"image": 2, "video": 0}'
enable-auto-tool-choice: true
tool-call-parser: qwen3_coder
reasoning-parser: qwen3
# MUST be flashinfer_cutlass, NOT the Unsloth b12x recipe. b12x supports
# only *quantized* NVFP4 MoE and hard-fails on the unquantized MTP draft
# layers: "moe_backend='flashinfer_b12x' is not supported for
# unquantized MoE. Expected one of ['triton','flashinfer_trtllm',
# 'flashinfer_cutlass','aiter']".
moe-backend: flashinfer_cutlass
env:
CUTE_DSL_ARCH: sm_121a
voice:
# Chatterbox **Turbo**, chosen over the full checkpoint after a blind A/B on
# 2026-08-02 — and the name is misleading. Turbo is not a degraded fast
# variant: it was published 2025-12-02 (the base is 2025-04-24) and pairs a
# 479M T3 with a DIFFERENT, newer vocoder — `S3Gen(meanflow=True)` +
# s3gen_meanflow.safetensors, where the full model uses plain `S3Gen()`.
# The vocoder is what produces the waveform, so that is what you hear.
# It is also ~2x faster (RTF 0.26 vs 0.50). Newer AND cheaper; no trade.
#
# Both checkpoints stay resident (CBX_VARIANTS) so the /voice A/B Lab can
# re-test per request without a ~30s reload; that costs ~3.6GB extra.
name: "Chatterbox Turbo TTS (meanflow vocoder, voice cloning)"
footprint_gb: 8
channel: stable
health: "http://localhost:8095/health"
serve:
kind: uvicorn
port: 8095
command:
- "~/cbx-env/bin/uvicorn"
- "voice_api:app"
- "--host"
- "0.0.0.0"
- "--port"
- "8095"
- "--app-dir"
- "~/chatterbox"
env:
# Turbo is the served default. Both stay loaded so the /voice A/B Lab
# can compare per request; this only decides what an unqualified
# request gets — i.e. what the downstream caller hears.
CBX_VARIANT: turbo
CBX_VARIANTS: "full,turbo"
# Local copies, not from_pretrained(): snapshot_download needs the
# network and would silently follow upstream if the repo moved.
CBX_TURBO_DIR: "/srv/models/chatterbox-turbo"
CBX_FULL_DIR: "/srv/models/chatterbox-full"
brain-laguna:
name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)"
# 71 GB main weights + draft, runtime/JIT overhead, and a conservative
# 64k FP8 KV allowance. This is a replacement for `brain`, never a peer.
footprint_gb: 86
channel: stable
health: "http://localhost:8001/v1/models"
health_contains: "poolside/Laguna-S-2.1-NVFP4"
serve:
kind: vllm
executable: "~/laguna-env/bin/vllm"
port: 8001
weights: "~/models/Laguna-S-2.1-NVFP4"
served_name:
- brain
- "local-moe"
- "poolside/Laguna-S-2.1-NVFP4"
args:
default-chat-template-kwargs: '{"enable_thinking": false}'
speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}'
max-model-len: 65536
kv-cache-dtype: fp8
gpu-memory-utilization: 0.70
max-num-seqs: 8
max-num-batched-tokens: 4096
enforce-eager: true
enable-auto-tool-choice: true
tool-call-parser: poolside_v1
reasoning-parser: poolside_v1
override-generation-config: '{"temperature":0.7,"top_p":0.95}'
env:
CUTE_DSL_ARCH: sm_121a
MAX_JOBS: "4"
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
brain-gemma:
name: "Gemma 4 31B-IT (NVFP4)"
# 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation
# headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window
# attention (10 of 60 layers are full-attention, rest capped at a 1024-token
# window) means its real KV cost at 64k context is far lighter than brain/
# brain-laguna's dense-attention KV — this cap is a conservative first pass,
# not yet confirmed by an actual load on the reference node.
footprint_gb: 44
channel: stable
health: "http://localhost:8001/v1/models"
health_contains: "nvidia/Gemma-4-31B-IT-NVFP4"
serve:
kind: vllm
executable: "~/gemma-env/bin/vllm"
port: 8001
weights: "~/models/Gemma-4-31B-IT-NVFP4"
served_name:
- brain
- "local-moe"
- "nvidia/Gemma-4-31B-IT-NVFP4"
args:
max-model-len: 65536
kv-cache-dtype: fp8
gpu-memory-utilization: 0.35
quantization: modelopt
max-num-seqs: 8
max-num-batched-tokens: 4096
enforce-eager: true
# Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM
# as of this writing — verify before relying on function-calling.
env:
CUTE_DSL_ARCH: sm_121a
music:
name: "ACE-Step 1.5 XL 4B sft (music generation)"
footprint_gb: 24 # measured ~21GB resident live on the reference node; 24 declared for safety
channel: stable
health: "http://localhost:8010/health"
serve:
kind: exec
port: 8010
# 4B XL sft DiT + 1.7B LM (pt backend). Custom FastAPI wrapper (POST /generate -> WAV).
# Runs as a systemd --user unit on the node today; this is the equivalent launch.
command: ["~/ACE-Step-1.5/.venv/bin/python", "~/ace_step_server.py"]
env:
ACESTEP_CONFIG_PATH: acestep-v15-xl-sft
ACESTEP_LM_MODEL_PATH: acestep-5Hz-lm-1.7B
ACESTEP_LM_BACKEND: pt
ACESTEP_API_PORT: "8010"
ACESTEP_API_HOST: 0.0.0.0
image:
name: "FLUX.2-klein-9B (BF16)"
# BFL's own card claims ~29GB VRAM, but an independent benchmark that
# actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident
# (95s/image) — trust the on-hardware number for admission control until
# we re-measure it ourselves. Non-commercial license (BFL FLUX license);
# fine for personal use, not for anything monetized. Requires accepting
# the license on HF and `hf auth login` before the weights can download.
footprint_gb: 66
channel: stable
health: "http://localhost:8007/health"
serve:
kind: diffusers
port: 8007
command: ["~/flux-env/bin/python", "~/flux_server.py"]
env:
FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B"
FLUX_API_PORT: "8007"
embed:
name: "Nemotron 3 Embed 1B (NVFP4) — dense retrieval"
# 1.14B, NVFP4 (~1GB weights); sentence-transformers pooling head, 2048-dim,
# 32k ctx. Serves OpenAI /v1/embeddings via vLLM. Footprint is dominated by
# the gpu-memory-utilization reservation, not the tiny weights.
footprint_gb: 2
channel: stable
health: "http://localhost:8012/v1/models"
health_contains: "embed"
serve:
kind: vllm
executable: "~/vllm-env/bin/vllm"
port: 8012
weights: "~/models/Nemotron-3-Embed-1B-NVFP4"
served_name:
- embed
- nemotron-embed
args:
runner: pooling
max-model-len: 32768
gpu-memory-utilization: 0.05
enforce-eager: true
# Alternatives, ready to register when wanted:
# image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark
# image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering
scenes/darkroom.scene.yaml
+27
View File
@@ -0,0 +1,27 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: darkroom
version: 1
description: "Overnight image farm — drops the brain to make room for FLUX.2-klein-9B."
author: karti
tags: [image, overnight, unattended]
# Drops `brain` so `image` (66 GB, on-hardware measured) fits. ears+voice+music+
# image commits ~99 GB — tight against the 8 GB admission margin, so this scene
# needs the full 108 GB ceiling (matching studio/voice-laguna), not the old 100.
models:
- ears
- voice
- music
- image
budget_gb: 108
activation:
order: footprint-asc
wait_healthy: true
# Scheduled scenes (planned): hand the box to darkroom overnight, back at 08:00.
# schedule:
# - activate: "03:00"
# - handoff: "04:00" -> studio
scenes/music.scene.yaml
+20
View File
@@ -0,0 +1,20 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: music
version: 1
description: "Music studio — ACE-Step XL 4B sft, with ears and voice, no brain."
author: karti
tags: [music, creative]
# ears 5 + voice 4 + music 24 = 33 GB. Brain-free, so tons of headroom — the clean
# scene for a music-generation demo when the MoE is not needed.
models:
- ears
- voice
- music
budget_gb: 100
activation:
order: footprint-asc
wait_healthy: true
scenes/studio.scene.yaml
+22
View File
@@ -0,0 +1,22 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: studio
version: 2
description: "Live voice assistant — ears, brain, mouth, and music."
author: karti
tags: [assistant, voice, always-on]
# ears 5 + brain 66 + voice 4 + music (XL 4B sft) 24 = 99 GB declared.
# Empirically validated on the reference node: all four load with ~20 GB MemAvailable free.
# budget bumped to 108 (box is 121 GB) so the Governor admits the measured-safe set.
models:
- ears
- brain
- voice
- music
budget_gb: 108
activation:
order: footprint-asc # small first, so brain's big load spike lands last
wait_healthy: true
scenes/voice-gemma.scene.yaml
+22
View File
@@ -0,0 +1,22 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: voice-gemma
version: 1
description: "Efficient voice assistant — Nemotron ASR, Gemma 4 31B, and Chatterbox."
author: karti
tags: [assistant, voice, efficient, multimodal]
# Gemma's mostly-sliding-window attention keeps its footprint well under brain
# and brain-laguna's, leaving real headroom in the budget beyond Lumbridge Compute's 8 GB
# admission margin — useful until brain-gemma's footprint is confirmed on
# real hardware.
models:
- ears
- brain-gemma
- voice
budget_gb: 80
activation:
order: footprint-asc
wait_healthy: true
scenes/voice-laguna.scene.yaml
+21
View File
@@ -0,0 +1,21 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: voice-laguna
version: 1
description: "High-capability voice assistant — Nemotron ASR, Laguna S 2.1, and Chatterbox."
author: karti
tags: [assistant, voice, coding, reasoning]
# Laguna replaces Qwen and intentionally excludes ACE-Step. The 108 GB scene
# budget matches the empirically safe studio ceiling on the reference node while retaining
# Lumbridge Compute's separate 8 GB admission margin.
models:
- ears
- brain-laguna
- voice
budget_gb: 108
activation:
order: footprint-asc
wait_healthy: true
scenes/voice-qwen.scene.yaml
+26
View File
@@ -0,0 +1,26 @@
apiVersion: lumbridge/v1
kind: Scene
metadata:
name: voice-qwen
version: 2
description: "Superseded by a newer scene on this node. Kept only because it is still the recorded fallback."
author: karti
tags: [assistant, voice, low-latency, deprecated]
# DEPRECATED as of 2026-08-02, superseded by a newer scene. Retained solely
# because last_known_good still points at it; delete once the fallback rolls forward.
#
# Its ordering was made brain-first to match its successor. `brain` now runs MTP, whose
# KV-cache profiling spike is not bounded by gpu-memory-utilization; the old
# footprint-asc order would start voice+ears first and leave brain short enough
# to trip the 3GB watchdog floor. A fallback that cannot come up is worse than
# no fallback.
models:
- brain
- ears
- voice
budget_gb: 100
activation:
order: listed
wait_healthy: true
src/agent.rs
+87
View File
@@ -0,0 +1,87 @@
//! Resident Lumbridge Compute supervisor.
use anyhow::{Context, Result};
use std::path::Path;
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use crate::{gateway, lifecycle, mem, proc};
pub fn run(root: &Path, listen: &str, upstream: &str, floor: f64) -> Result<()> {
println!("Lumbridge Compute agent starting");
lifecycle::resume(root).context("resuming persisted desired Scene")?;
let (gateway_done_tx, gateway_done_rx) = mpsc::channel();
let listen_owned = listen.to_string();
let upstream_owned = upstream.to_string();
thread::spawn(move || {
let result = gateway::run(&listen_owned, &upstream_owned);
gateway_done_tx.send(result).ok();
});
println!(
"Lumbridge Compute agent supervising memory floor {:.1} GB",
floor
);
loop {
match gateway_done_rx.try_recv() {
Ok(result) => return result.context("stable gateway stopped"),
Err(mpsc::TryRecvError::Disconnected) => {
anyhow::bail!("stable gateway supervisor disconnected")
}
Err(mpsc::TryRecvError::Empty) => {}
}
enforce_memory_floor(root, floor)?;
thread::sleep(Duration::from_secs(1));
}
}
fn enforce_memory_floor(root: &Path, floor: f64) -> Result<()> {
let memory = mem::read()?;
if memory.available_gb >= floor {
return Ok(());
}
// Take the same lock a Scene transition takes, and skip this tick if a transition
// already holds it. Without this the watchdog races activation: activation is
// deliberately a stop-all-then-start-all sequence, so mid-transition the pool is
// legitimately tight and `state.procs` is being rewritten underneath us. Firing then
// would kill a model the transition just started, and then write a stale `state`
// over the transition's own — losing track of a process that is still alive.
//
// A tick skipped here costs one second. The floor is a backstop, and the transition
// holding the lock is doing its own admission checks.
let _lock = match lifecycle::TransitionLock::acquire(root) {
Ok(lock) => lock,
Err(_) => {
eprintln!(
"agent: MemAvailable {:.1} GB < floor {:.1}, but a Scene transition holds the \
lock; deferring to it for this tick",
memory.available_gb, floor
);
return Ok(());
}
};
let mut state = proc::State::load_checked(root)?;
match state.newest_alive() {
Some((id, process)) => {
eprintln!(
"agent: MemAvailable {:.1} GB < floor {:.1}; stopping newest owned model '{}' (pid {})",
memory.available_gb, floor, id, process.pid
);
proc::stop_owned(&process)?;
state.procs.remove(&id);
state.active_scene = None;
state.last_error = Some(format!(
"watchdog stopped '{id}' after MemAvailable fell to {:.1} GB",
memory.available_gb
));
state.save(root)?;
}
None => eprintln!(
"agent: MemAvailable {:.1} GB < floor {:.1}, but no identity-owned model can be stopped",
memory.available_gb, floor
),
}
Ok(())
}
src/config.rs
+151
View File
@@ -0,0 +1,151 @@
//! Config types + loaders for the model registry and scenes.
//!
//! The registry (`registry/models.yaml`) resolves stable model *ids* to weights +
//! launch commands, and evolves as models requantize. Scenes (`scenes/*.scene.yaml`)
//! reference those ids and stay stable — the public, shareable contract.
//!
//! Several fields below are deserialized but never read by the binary
//! (`api_version`, `channel`, `entry`, `app`, `app_dir`, `tags`, `author`).
//! That is deliberate: they are the published `lumbridge/v1` manifest surface,
//! and declaring them is what makes a manifest carrying them parse rather than
//! fail. Deleting them to satisfy the lint would silently narrow the contract.
#![allow(dead_code)]
use anyhow::{Context, Result};
use serde::Deserialize;
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
#[derive(Debug, Deserialize)]
pub struct Registry {
#[serde(rename = "apiVersion")]
pub api_version: String,
pub models: BTreeMap<String, Model>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct Model {
pub name: String,
/// Worst-case unified memory held once serving (weights + KV/cache + encoder).
pub footprint_gb: f64,
/// `stable` pins the exact weights; `latest` may resolve a newer quant.
#[serde(default)]
pub channel: Option<String>,
#[serde(default)]
pub health: Option<String>,
/// Optional marker that must appear in the health response. This
/// distinguishes mutually-exclusive models that intentionally share a port.
#[serde(default)]
pub health_contains: Option<String>,
pub serve: Serve,
}
/// How to launch a model. Fields are permissive across runtimes (vllm / diffusers /
/// python / uvicorn); only the ones a given `kind` needs are populated.
#[derive(Debug, Clone, Deserialize)]
pub struct Serve {
pub kind: String,
/// Optional runtime executable for kind-based builders (for example a
/// model-specific vLLM virtualenv). Defaults to the kind's command on PATH.
#[serde(default)]
pub executable: Option<String>,
#[serde(default)]
pub port: Option<u16>,
/// Explicit launch argv. If set, it is used verbatim (argv[0] = program) and
/// takes precedence over any `kind`-based builder. This is where launch commands
/// live — never in a scene — so downloaded scenes can't smuggle code.
#[serde(default)]
pub command: Vec<String>,
#[serde(default)]
pub weights: Option<String>,
#[serde(default)]
pub entry: Option<String>,
#[serde(default)]
pub app: Option<String>,
#[serde(default)]
pub app_dir: Option<String>,
#[serde(default)]
pub served_name: Vec<String>,
#[serde(default)]
pub args: BTreeMap<String, serde_yaml::Value>,
#[serde(default)]
pub env: BTreeMap<String, String>,
}
#[derive(Debug, Deserialize)]
pub struct Scene {
#[serde(rename = "apiVersion")]
pub api_version: String,
pub metadata: SceneMeta,
/// Stable model ids resolved via the registry.
pub models: Vec<String>,
/// Optional per-scene budget override (GB); defaults to the Governor's global budget.
#[serde(default)]
pub budget_gb: Option<f64>,
#[serde(default)]
pub activation: Option<Activation>,
}
#[derive(Debug, Deserialize)]
pub struct SceneMeta {
pub name: String,
#[serde(default)]
pub version: u32,
#[serde(default)]
pub description: String,
#[serde(default)]
pub tags: Vec<String>,
#[serde(default)]
pub author: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct Activation {
/// `footprint-asc` (default) | `listed`.
#[serde(default)]
pub order: Option<String>,
#[serde(default)]
pub wait_healthy: Option<bool>,
}
impl Registry {
pub fn load(root: &Path) -> Result<Registry> {
let p = root.join("registry/models.yaml");
let s =
fs::read_to_string(&p).with_context(|| format!("reading registry {}", p.display()))?;
serde_yaml::from_str(&s).with_context(|| format!("parsing registry {}", p.display()))
}
}
/// Load every `*.scene.yaml` under `<root>/scenes`, sorted by name.
pub fn load_scenes(root: &Path) -> Result<Vec<Scene>> {
let dir = root.join("scenes");
let mut out = Vec::new();
if !dir.exists() {
return Ok(out);
}
for entry in fs::read_dir(&dir).with_context(|| format!("reading {}", dir.display()))? {
let path = entry?.path();
let is_scene = path
.file_name()
.and_then(|n| n.to_str())
.map(|n| n.ends_with(".scene.yaml"))
.unwrap_or(false);
if is_scene {
let s = fs::read_to_string(&path)?;
let scene: Scene =
serde_yaml::from_str(&s).with_context(|| format!("parsing {}", path.display()))?;
out.push(scene);
}
}
out.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
Ok(out)
}
pub fn find_scene(root: &Path, name: &str) -> Result<Scene> {
load_scenes(root)?
.into_iter()
.find(|s| s.metadata.name == name)
.with_context(|| format!("no scene named '{name}' in {}/scenes", root.display()))
}
src/eval.rs
+495
View File
@@ -0,0 +1,495 @@
//! Reproducible evaluations for any OpenAI-compatible model server.
//!
//! Suites are declarative YAML. Results are append-only JSON artifacts suitable
//! for CI, regression comparisons, and future publication to an eval registry.
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::fs;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
#[derive(Debug, Deserialize)]
struct Suite {
#[serde(rename = "apiVersion")]
api_version: String,
kind: String,
metadata: Metadata,
#[serde(default)]
defaults: Defaults,
cases: Vec<Case>,
}
#[derive(Debug, Deserialize)]
struct Metadata {
name: String,
#[serde(default)]
version: u32,
#[serde(default)]
description: String,
/// Part of the published suite manifest surface; parsed so a suite
/// carrying tags loads, not read by the runner itself.
#[serde(default)]
#[allow(dead_code)]
tags: Vec<String>,
}
#[derive(Debug, Default, Deserialize)]
struct Defaults {
#[serde(default = "default_max_tokens")]
max_tokens: u32,
#[serde(default = "default_temperature")]
temperature: f64,
#[serde(default = "default_repeat")]
repeat: u32,
#[serde(default)]
system: String,
}
fn default_max_tokens() -> u32 {
128
}
fn default_temperature() -> f64 {
0.0
}
fn default_repeat() -> u32 {
1
}
#[derive(Debug, Deserialize)]
struct Case {
id: String,
#[serde(default)]
category: String,
prompt: String,
#[serde(default)]
max_tokens: Option<u32>,
#[serde(default)]
assertions: Vec<Assertion>,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum Assertion {
Exact { value: String },
Contains { value: String },
ContainsAny { values: Vec<String> },
NotContains { value: String },
MaxWords { value: usize },
}
#[derive(Debug, Serialize)]
struct Artifact {
schema_version: u32,
run_id: String,
suite: String,
suite_version: u32,
model: String,
base_url: String,
started_unix_ms: u128,
summary: Summary,
samples: Vec<Sample>,
}
#[derive(Debug, Serialize)]
struct Summary {
passed: usize,
total: usize,
score: f64,
mean_ttft_ms: f64,
p50_ttft_ms: f64,
p95_ttft_ms: f64,
mean_prefill_tps: f64,
mean_decode_tps: f64,
}
#[derive(Debug, Serialize)]
struct Sample {
case_id: String,
category: String,
repetition: u32,
passed: bool,
assertion_results: Vec<bool>,
output: String,
reasoning: String,
prompt_tokens: u64,
completion_tokens: u64,
ttft_ms: f64,
total_ms: f64,
prefill_tps: f64,
decode_tps: f64,
}
struct Completion {
output: String,
reasoning: String,
prompt_tokens: u64,
completion_tokens: u64,
ttft_ms: f64,
total_ms: f64,
}
/// One line of `eval ls`. Suites themselves stay private — a caller has no
/// business reaching into cases and assertions — but the catalogue is the
/// useful part and is shared by the CLI and the MCP server.
#[derive(Debug, Clone)]
pub struct SuiteSummary {
pub name: String,
pub version: u32,
pub cases: usize,
pub description: String,
}
pub fn catalogue(root: &Path) -> Result<Vec<SuiteSummary>> {
Ok(load_all(root)?
.into_iter()
.map(|s| SuiteSummary {
name: s.metadata.name,
version: s.metadata.version,
cases: s.cases.len(),
description: s.metadata.description,
})
.collect())
}
pub fn list(root: &Path) -> Result<()> {
println!("{:<20} {:>4} {:>5} DESCRIPTION", "SUITE", "VER", "CASES");
for s in catalogue(root)? {
println!(
"{:<20} {:>4} {:>5} {}",
s.name, s.version, s.cases, s.description
);
}
Ok(())
}
pub fn run(
root: &Path,
name: &str,
base_url: &str,
model: &str,
repeat: Option<u32>,
) -> Result<()> {
let suite = load(root, name)?;
// `kuda/v1` is the pre-rename contract; still accepted so an older suite keeps running.
let known_contract = matches!(suite.api_version.as_str(), "lumbridge/v1" | "kuda/v1");
if !known_contract || suite.kind != "EvalSuite" {
bail!(
"unsupported eval contract: {}/{}",
suite.api_version,
suite.kind
);
}
let reps = repeat.unwrap_or(suite.defaults.repeat).max(1);
println!(
"eval '{}' v{} → {} ({})",
suite.metadata.name, suite.metadata.version, model, base_url
);
let started = now_ms();
let mut samples = Vec::new();
for case in &suite.cases {
for repetition in 1..=reps {
print!(" {:<24} [{}/{}] ", case.id, repetition, reps);
std::io::stdout().flush().ok();
let c = stream_completion(base_url, model, &suite.defaults, case)?;
let assertion_results: Vec<bool> = case
.assertions
.iter()
.map(|a| evaluate(a, &c.output))
.collect();
let passed = assertion_results.iter().all(|v| *v);
let decode_seconds = ((c.total_ms - c.ttft_ms) / 1000.0).max(0.001);
let prefill_seconds = (c.ttft_ms / 1000.0).max(0.001);
let sample = Sample {
case_id: case.id.clone(),
category: case.category.clone(),
repetition,
passed,
assertion_results,
output: c.output,
reasoning: c.reasoning,
prompt_tokens: c.prompt_tokens,
completion_tokens: c.completion_tokens,
ttft_ms: c.ttft_ms,
total_ms: c.total_ms,
prefill_tps: c.prompt_tokens as f64 / prefill_seconds,
decode_tps: c.completion_tokens as f64 / decode_seconds,
};
println!(
"{} ttft={:.0}ms decode={:.1}tok/s",
if passed { "PASS" } else { "FAIL" },
sample.ttft_ms,
sample.decode_tps
);
samples.push(sample);
}
}
let summary = summarize(&samples);
let run_id = format!("{}-{}-{}", started, slug(name), slug(model));
let artifact = Artifact {
schema_version: 1,
run_id: run_id.clone(),
suite: suite.metadata.name,
suite_version: suite.metadata.version,
model: model.to_string(),
base_url: base_url.to_string(),
started_unix_ms: started,
summary,
samples,
};
let out_dir = root.join("eval-results");
fs::create_dir_all(&out_dir)?;
let out = out_dir.join(format!("{run_id}.json"));
fs::write(&out, serde_json::to_string_pretty(&artifact)?)?;
println!(
"\nscore {:.1}% ({}/{}) · mean TTFT {:.0}ms · prefill≈{:.1}tok/s · decode {:.1}tok/s",
artifact.summary.score * 100.0,
artifact.summary.passed,
artifact.summary.total,
artifact.summary.mean_ttft_ms,
artifact.summary.mean_prefill_tps,
artifact.summary.mean_decode_tps
);
println!("result {}", out.display());
Ok(())
}
fn stream_completion(
base_url: &str,
model: &str,
defaults: &Defaults,
case: &Case,
) -> Result<Completion> {
let url = format!("{}/chat/completions", base_url.trim_end_matches('/'));
let mut messages = Vec::new();
if !defaults.system.is_empty() {
messages.push(json!({"role":"system","content":defaults.system}));
}
messages.push(json!({"role":"user","content":case.prompt}));
let request = json!({
"model": model, "messages": messages, "stream": true,
"stream_options": {"include_usage": true},
"max_tokens": case.max_tokens.unwrap_or(defaults.max_tokens),
"temperature": defaults.temperature,
"chat_template_kwargs": {"enable_thinking": false}
});
let mut child = Command::new("curl")
.args([
"-sS",
"-N",
"-X",
"POST",
&url,
"-H",
"Content-Type: application/json",
"--data-binary",
"@-",
])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("starting curl (required for eval HTTP streaming)")?;
child
.stdin
.take()
.unwrap()
.write_all(request.to_string().as_bytes())?;
let start = Instant::now();
let mut first = None;
let mut output = String::new();
let mut reasoning = String::new();
let mut prompt_tokens = 0;
let mut completion_tokens = 0;
for line in BufReader::new(child.stdout.take().unwrap()).lines() {
let line = line?;
let Some(data) = line.strip_prefix("data: ") else {
continue;
};
if data == "[DONE]" {
break;
}
let v: Value = serde_json::from_str(data).context("parsing streamed completion")?;
if let Some(usage) = v.get("usage") {
prompt_tokens = usage
.get("prompt_tokens")
.and_then(Value::as_u64)
.unwrap_or(prompt_tokens);
completion_tokens = usage
.get("completion_tokens")
.and_then(Value::as_u64)
.unwrap_or(completion_tokens);
}
let delta = &v["choices"][0]["delta"];
let content = delta.get("content").and_then(Value::as_str).unwrap_or("");
let thought = delta
.get("reasoning")
.or_else(|| delta.get("reasoning_content"))
.and_then(Value::as_str)
.unwrap_or("");
if first.is_none() && (!content.is_empty() || !thought.is_empty()) {
first = Some(start.elapsed());
}
output.push_str(content);
reasoning.push_str(thought);
}
let status = child.wait()?;
if !status.success() {
bail!("completion request failed with {status}");
}
let total = start.elapsed().as_secs_f64() * 1000.0;
Ok(Completion {
output,
reasoning,
prompt_tokens,
completion_tokens,
ttft_ms: first.map(|d| d.as_secs_f64() * 1000.0).unwrap_or(total),
total_ms: total,
})
}
fn evaluate(a: &Assertion, output: &str) -> bool {
let normalized = output.trim();
match a {
Assertion::Exact { value } => normalized.eq_ignore_ascii_case(value.trim()),
Assertion::Contains { value } => normalized.to_lowercase().contains(&value.to_lowercase()),
Assertion::ContainsAny { values } => values
.iter()
.any(|v| normalized.to_lowercase().contains(&v.to_lowercase())),
Assertion::NotContains { value } => {
!normalized.to_lowercase().contains(&value.to_lowercase())
}
Assertion::MaxWords { value } => normalized.split_whitespace().count() <= *value,
}
}
fn summarize(samples: &[Sample]) -> Summary {
let mut ttft: Vec<f64> = samples.iter().map(|s| s.ttft_ms).collect();
ttft.sort_by(f64::total_cmp);
let mean = |f: fn(&Sample) -> f64| {
if samples.is_empty() {
0.0
} else {
samples.iter().map(f).sum::<f64>() / samples.len() as f64
}
};
let percentile = |p: f64| {
if ttft.is_empty() {
0.0
} else {
ttft[((ttft.len() - 1) as f64 * p).round() as usize]
}
};
let passed = samples.iter().filter(|s| s.passed).count();
Summary {
passed,
total: samples.len(),
score: if samples.is_empty() {
0.0
} else {
passed as f64 / samples.len() as f64
},
mean_ttft_ms: mean(|s| s.ttft_ms),
p50_ttft_ms: percentile(0.50),
p95_ttft_ms: percentile(0.95),
mean_prefill_tps: mean(|s| s.prefill_tps),
mean_decode_tps: mean(|s| s.decode_tps),
}
}
fn load(root: &Path, name: &str) -> Result<Suite> {
load_all(root)?
.into_iter()
.find(|s| s.metadata.name == name)
.with_context(|| format!("no eval suite named '{name}' in {}/evals", root.display()))
}
fn load_all(root: &Path) -> Result<Vec<Suite>> {
let dir = root.join("evals");
let mut suites = Vec::new();
if !dir.exists() {
return Ok(suites);
}
for entry in fs::read_dir(&dir)? {
let path: PathBuf = entry?.path();
if path
.file_name()
.and_then(|n| n.to_str())
.map(|n| n.ends_with(".eval.yaml"))
.unwrap_or(false)
{
let suite: Suite = serde_yaml::from_str(&fs::read_to_string(&path)?)
.with_context(|| format!("parsing {}", path.display()))?;
suites.push(suite);
}
}
suites.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
Ok(suites)
}
fn now_ms() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
}
fn slug(s: &str) -> String {
s.chars()
.map(|c| {
if c.is_ascii_alphanumeric() {
c.to_ascii_lowercase()
} else {
'-'
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn assertions_are_case_insensitive_and_composable() {
let output = "Market risk remains, while company-specific risk falls.";
assert!(evaluate(
&Assertion::Contains {
value: "MARKET RISK".into()
},
output
));
assert!(evaluate(
&Assertion::ContainsAny {
values: vec!["idiosyncratic".into(), "company-specific".into()],
},
output
));
assert!(evaluate(
&Assertion::NotContains {
value: "markdown".into()
},
output
));
}
#[test]
fn exact_trims_but_does_not_accept_explanation() {
assert!(evaluate(&Assertion::Exact { value: "25".into() }, " 25\n"));
assert!(!evaluate(&Assertion::Exact { value: "25".into() }, "25 GB"));
}
#[test]
fn word_limit_counts_whitespace_tokens() {
assert!(evaluate(
&Assertion::MaxWords { value: 4 },
"one two three four"
));
assert!(!evaluate(
&Assertion::MaxWords { value: 3 },
"one two three four"
));
}
}
src/gateway.rs
+102
View File
@@ -0,0 +1,102 @@
//! Streaming-safe TCP gateway for OpenAI-compatible model servers.
//!
//! The gateway deliberately stays below HTTP: it forwards bytes unchanged, so
//! chunked responses and server-sent-event token streams retain their timing and
//! semantics while clients keep one stable Lumbridge Compute address.
use anyhow::{Context, Result};
use std::io;
use std::net::{Shutdown, TcpListener, TcpStream};
use std::thread;
pub fn run(listen: &str, upstream: &str) -> Result<()> {
let listener = TcpListener::bind(listen)
.with_context(|| format!("binding Lumbridge gateway at {listen}"))?;
println!("Lumbridge gateway {listen} -> {upstream}");
serve_listener(listener, upstream, None)
}
fn serve_listener(
listener: TcpListener,
upstream: &str,
max_connections: Option<usize>,
) -> Result<()> {
let mut accepted = 0usize;
for incoming in listener.incoming() {
let client = match incoming {
Ok(client) => client,
Err(error) => {
eprintln!("gateway accept failed: {error}");
continue;
}
};
let upstream = upstream.to_string();
thread::spawn(move || {
if let Err(error) = proxy(client, &upstream) {
eprintln!("gateway request failed: {error:#}");
}
});
accepted += 1;
if max_connections.is_some_and(|limit| accepted >= limit) {
break;
}
}
Ok(())
}
fn proxy(mut client: TcpStream, upstream_addr: &str) -> Result<()> {
client.set_nodelay(true).ok();
let mut upstream = TcpStream::connect(upstream_addr)
.with_context(|| format!("connecting gateway upstream {upstream_addr}"))?;
upstream.set_nodelay(true).ok();
let mut client_reader = client.try_clone()?;
let mut upstream_writer = upstream.try_clone()?;
let request = thread::spawn(move || -> io::Result<u64> {
let copied = io::copy(&mut client_reader, &mut upstream_writer)?;
upstream_writer.shutdown(Shutdown::Write).ok();
Ok(copied)
});
io::copy(&mut upstream, &mut client)?;
client.shutdown(Shutdown::Write).ok();
request
.join()
.map_err(|_| anyhow::anyhow!("gateway request-copy thread panicked"))??;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
#[test]
fn gateway_forwards_bidirectional_bytes_without_buffering_protocols() {
let upstream = TcpListener::bind("127.0.0.1:0").unwrap();
let upstream_addr = upstream.local_addr().unwrap();
let upstream_thread = thread::spawn(move || {
let (mut socket, _) = upstream.accept().unwrap();
let mut request = [0u8; 4];
socket.read_exact(&mut request).unwrap();
assert_eq!(&request, b"ping");
socket.write_all(b"pong").unwrap();
});
let gateway = TcpListener::bind("127.0.0.1:0").unwrap();
let gateway_addr = gateway.local_addr().unwrap();
let gateway_thread = thread::spawn(move || {
serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap();
});
let mut client = TcpStream::connect(gateway_addr).unwrap();
client.write_all(b"ping").unwrap();
client.shutdown(Shutdown::Write).unwrap();
let mut response = Vec::new();
client.read_to_end(&mut response).unwrap();
assert_eq!(response, b"pong");
upstream_thread.join().unwrap();
gateway_thread.join().unwrap();
}
}
src/governor.rs
+223
View File
@@ -0,0 +1,223 @@
//! The Governor — the kernel of Lumbridge Compute.
//!
//! On a unified-memory box, over-commit doesn't fail gracefully: the whole machine
//! thrashes and wedges (SSH/ping included) before the OOM killer acts. The Governor
//! makes that impossible via (1) admission control against a hard budget and
//! (2) a watchdog on `MemAvailable` that kills the newest model before thrash.
//!
//! This module currently provides the *sensing* + *admission* half. The watchdog and
//! process spawn/kill land with the `up`/`activate` commands.
use crate::config::{Model, Registry};
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::time::{Duration, Instant};
/// Usable memory for models; the rest is reserved for the OS/desktop.
pub const DEFAULT_BUDGET_GB: f64 = 100.0;
/// Extra headroom required before admitting a new model.
pub const SAFETY_MARGIN_GB: f64 = 8.0;
/// If `MemAvailable` dips below this, the watchdog kills the newest model.
pub const WATCHDOG_FLOOR_GB: f64 = 3.0;
/// A model is considered "running" if its serving port accepts a connection.
pub fn port_open(port: u16) -> bool {
let addr = SocketAddr::from(([127, 0, 0, 1], port));
TcpStream::connect_timeout(&addr, Duration::from_millis(150)).is_ok()
}
pub fn is_running(m: &Model) -> bool {
let Some(port) = m.serve.port else {
return false;
};
if !port_open(port) {
return false;
}
match &m.health_contains {
Some(marker) => health_response(m, port)
.map(|response| response.contains(marker))
.unwrap_or(false),
None => true,
}
}
/// Fetch the local HTTP health endpoint without adding an HTTP client runtime.
/// Registry health URLs are deliberately localhost-only.
fn health_response(m: &Model, port: u16) -> Option<String> {
let path = m
.health
.as_deref()
.and_then(|url| url.split_once("localhost"))
.map(|(_, tail)| tail.trim_start_matches(|c: char| c.is_ascii_digit() || c == ':'))
.filter(|path| path.starts_with('/'))
.unwrap_or("/");
let addr = SocketAddr::from(([127, 0, 0, 1], port));
let mut stream = TcpStream::connect_timeout(&addr, Duration::from_millis(300)).ok()?;
stream
.set_read_timeout(Some(Duration::from_millis(500)))
.ok()?;
stream
.write_all(format!("GET {path} HTTP/1.0\r\nHost: localhost\r\n\r\n").as_bytes())
.ok()?;
let mut response = String::new();
stream.read_to_string(&mut response).ok()?;
Some(response)
}
pub fn running_ids(reg: &Registry) -> Vec<String> {
reg.models
.iter()
.filter(|(_, m)| is_running(m))
.map(|(id, _)| id.clone())
.collect()
}
/// Sum of footprints of all currently-serving registered models.
pub fn committed_gb(reg: &Registry) -> f64 {
reg.models
.values()
.filter(|m| is_running(m))
.map(|m| m.footprint_gb)
.sum()
}
/// Admission control. Two independent ceilings, and both must hold.
///
/// 1. **Declared** — `committed + add + margin <= budget`.
/// 2. **Observed** — `add + margin <= available`.
///
/// The second one is what makes the promise real. `budget_gb`, and every
/// `footprint_gb` feeding `committed_gb`, are numbers a human typed into the
/// registry. If any of them is optimistic — a model that grows past its declared
/// footprint, a KV cache larger than expected, anything started outside Compute —
/// the declared check happily passes while the box is already out of memory, and
/// on unified memory that ends in a wedged machine rather than a failed malloc.
/// `available_gb` comes from `MemAvailable`, which counts reality.
///
/// `available_gb` is `None` when sensing failed. That falls back to the declared
/// ceiling alone: refusing every start because /proc/meminfo was unreadable would
/// turn a sensing failure into a total outage.
pub fn can_admit(
add_gb: f64,
committed_gb: f64,
budget_gb: f64,
available_gb: Option<f64>,
) -> bool {
if committed_gb + add_gb + SAFETY_MARGIN_GB > budget_gb {
return false;
}
match available_gb {
Some(available) => add_gb + SAFETY_MARGIN_GB <= available,
None => true,
}
}
/// What admission will actually enforce right now, for reporting. The lower of the
/// declared headroom and the observed headroom.
pub fn headroom_gb(committed_gb: f64, budget_gb: f64, available_gb: Option<f64>) -> f64 {
let declared = budget_gb - committed_gb - SAFETY_MARGIN_GB;
match available_gb {
Some(available) => declared.min(available - SAFETY_MARGIN_GB),
None => declared,
}
}
/// Block until the exact registered model is healthy, or `timeout` elapses.
/// Models without a port are considered instantly ready.
pub fn wait_healthy(m: &Model, timeout: Duration) -> bool {
let Some(_) = m.serve.port else {
return true;
};
let start = Instant::now();
while start.elapsed() < timeout {
if is_running(m) {
return true;
}
std::thread::sleep(Duration::from_millis(400));
}
false
}
#[cfg(test)]
mod tests {
use super::*;
// A 108 GB Scene budget on a box with plenty free — the ordinary case.
const BUDGET: f64 = 108.0;
#[test]
fn admits_when_both_declared_and_observed_ceilings_allow_it() {
assert!(can_admit(66.0, 9.0, BUDGET, Some(100.0)));
}
#[test]
fn refuses_when_the_declared_budget_is_exceeded() {
// 66 + 40 + 8 margin = 114 > 108, even though the box has memory free.
assert!(!can_admit(66.0, 40.0, BUDGET, Some(100.0)));
}
#[test]
fn refuses_when_the_box_is_out_of_memory_even_though_the_paperwork_agrees() {
// This is the case the declared check alone could never catch: nothing is
// registered as committed, so the budget says there is 100 GB of room, but
// MemAvailable says 20 GB. Something outside Compute is holding the pool.
assert!(can_admit(66.0, 0.0, BUDGET, None), "declared check passes");
assert!(!can_admit(66.0, 0.0, BUDGET, Some(20.0)));
}
#[test]
fn the_safety_margin_is_enforced_against_observed_memory_too() {
// 66 GB model with exactly 66 GB free is a refusal: the margin has to fit.
assert!(!can_admit(66.0, 0.0, BUDGET, Some(66.0)));
assert!(!can_admit(
66.0,
0.0,
BUDGET,
Some(66.0 + SAFETY_MARGIN_GB - 0.1)
));
assert!(can_admit(66.0, 0.0, BUDGET, Some(66.0 + SAFETY_MARGIN_GB)));
}
#[test]
fn failed_sensing_falls_back_to_the_declared_ceiling_rather_than_refusing_everything() {
assert!(can_admit(66.0, 9.0, BUDGET, None));
// ...but it must not become a way to bypass the declared budget.
assert!(!can_admit(66.0, 40.0, BUDGET, None));
}
#[test]
fn a_zero_reading_refuses_everything_rather_than_admitting_everything() {
// parse_meminfo yields 0.0 for an unparseable /proc. That has to read as
// "no memory", not as "no constraint".
assert!(!can_admit(1.0, 0.0, BUDGET, Some(0.0)));
}
#[test]
fn headroom_never_exceeds_what_the_machine_actually_has() {
// The shape of a real regression: a generous declared budget on a smaller box. Reporting
// budget-minus-committed here promises room the next admission will refuse, and both the
// MCP tool and the HTTP API serve this number to callers that cannot check it themselves.
let box_available = 55.8;
let declared_budget = 100.0;
let reported = headroom_gb(0.0, declared_budget, Some(box_available));
assert!(
reported <= box_available,
"reported {reported} GB of headroom on a box with {box_available} GB free",
);
assert!(can_admit(
reported,
0.0,
declared_budget,
Some(box_available)
));
}
#[test]
fn headroom_reports_the_binding_constraint_not_the_generous_one() {
// Declared says 33 GB spare; the box says 12 GB spare minus margin.
assert_eq!(headroom_gb(67.0, BUDGET, None), 33.0);
assert_eq!(headroom_gb(67.0, BUDGET, Some(12.0)), 4.0);
// And the declared ceiling still wins when it is the tighter of the two.
assert_eq!(headroom_gb(100.0, BUDGET, Some(90.0)), 0.0);
}
}
src/http.rs
+326
View File
@@ -0,0 +1,326 @@
//! A read-only HTTP control API.
//!
//! This exists so a web UI can read a node's state without shelling out to the CLI. It is a
//! second transport over the operations the MCP server already models — `collect_status`,
//! `collect_models`, `collect_scenes`, `collect_scene`, `collect_evals` — deliberately not a
//! second implementation of them, so the two surfaces cannot drift.
//!
//! Hand-rolled on `std::net`, matching `gateway.rs`. An HTTP framework would pull a dependency
//! tree an order of magnitude larger than the whole rest of this binary, to serve five routes
//! that return pre-serialised JSON.
//!
//! # What this deliberately does not do
//!
//! **It never mutates.** No activate, no stop, no eval run. A read-only surface that a browser
//! can reach is a much smaller thing to get right than one that can move a node's memory around,
//! and the read half is what a dashboard actually needs.
//!
//! # Why the defaults are what they are
//!
//! - **Loopback only.** The listen address defaults to `127.0.0.1`. There is no authentication
//! worth the name here, so a bind to `0.0.0.0` publishes your node's inventory to the network.
//! - **CORS off.** No origin is allowed unless named with `--allow-origin`. Allowing `*` would
//! let *any* page you visit read what models you run, which is a fingerprint of your machine.
//! - **The token is optional but checked in constant time.** Loopback plus an origin allowlist
//! already stops the browser attack; the token is for the case where someone puts this behind
//! a proxy anyway.
use anyhow::{Context, Result};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::thread;
use std::time::Duration;
use crate::mcp::{collect_evals, collect_models, collect_scene, collect_scenes, collect_status};
/// Requests are tiny and come from localhost; anything slower than this is not a browser.
const READ_TIMEOUT: Duration = Duration::from_secs(5);
/// A request line plus headers. Anything larger is not a request we serve.
const MAX_HEAD_BYTES: usize = 8 * 1024;
pub struct Config {
pub root: PathBuf,
/// Origins permitted to read this API from a browser. Empty means none.
pub allowed_origins: Vec<String>,
/// When set, every request must carry `Authorization: Bearer <token>`.
pub token: Option<String>,
}
pub fn run(listen: &str, config: Config) -> Result<()> {
let listener = TcpListener::bind(listen)
.with_context(|| format!("binding the Lumbridge Compute API at {listen}"))?;
println!(
"Lumbridge Compute API on {listen} (read-only) · origins: {} · token: {}",
if config.allowed_origins.is_empty() {
"none".to_string()
} else {
config.allowed_origins.join(", ")
},
if config.token.is_some() {
"required"
} else {
"none"
},
);
if !listen.starts_with("127.") && !listen.starts_with("localhost") {
eprintln!(
"warning: {listen} is not loopback. This API has no authentication by default and \
reveals which models this node runs."
);
}
serve(listener, config)
}
fn serve(listener: TcpListener, config: Config) -> Result<()> {
let config = std::sync::Arc::new(config);
for incoming in listener.incoming() {
let stream = match incoming {
Ok(stream) => stream,
Err(error) => {
eprintln!("api accept failed: {error}");
continue;
}
};
let config = config.clone();
thread::spawn(move || {
if let Err(error) = handle(stream, &config) {
eprintln!("api request failed: {error:#}");
}
});
}
Ok(())
}
struct Request {
method: String,
path: String,
origin: Option<String>,
authorization: Option<String>,
}
/// Read the request line and headers. The body is ignored: every route is a GET.
fn read_request(stream: &TcpStream) -> Result<Option<Request>> {
let mut reader = BufReader::new(stream);
let mut head = String::new();
let mut total = 0usize;
loop {
let mut line = String::new();
let n = reader.read_line(&mut line)?;
if n == 0 {
return Ok(None); // client hung up
}
total += n;
if total > MAX_HEAD_BYTES {
return Ok(None);
}
if line == "\r\n" || line == "\n" {
break;
}
head.push_str(&line);
}
let mut lines = head.lines();
let Some(request_line) = lines.next() else {
return Ok(None);
};
let mut parts = request_line.split_whitespace();
let (Some(method), Some(target)) = (parts.next(), parts.next()) else {
return Ok(None);
};
let mut origin = None;
let mut authorization = None;
for line in lines {
let Some((name, value)) = line.split_once(':') else {
continue;
};
let value = value.trim().to_string();
match name.trim().to_ascii_lowercase().as_str() {
"origin" => origin = Some(value),
"authorization" => authorization = Some(value),
_ => {}
}
}
Ok(Some(Request {
method: method.to_string(),
// Query strings are not used by any route; dropping one keeps routing exact.
path: target.split('?').next().unwrap_or("/").to_string(),
origin,
authorization,
}))
}
/// Constant-time comparison so a token cannot be recovered a byte at a time from response timing.
fn token_ok(expected: &str, supplied: Option<&String>) -> bool {
let Some(supplied) = supplied.and_then(|v| v.strip_prefix("Bearer ")) else {
return false;
};
let a = expected.as_bytes();
let b = supplied.as_bytes();
// Length is compared without branching on it beyond the final AND.
let mut diff = (a.len() ^ b.len()) as u8;
for i in 0..a.len().max(b.len()) {
diff |= a.get(i).copied().unwrap_or(0) ^ b.get(i).copied().unwrap_or(0);
}
diff == 0
}
fn handle(mut stream: TcpStream, config: &Config) -> Result<()> {
stream.set_read_timeout(Some(READ_TIMEOUT))?;
stream.set_write_timeout(Some(READ_TIMEOUT))?;
let Some(request) = read_request(&stream)? else {
return Ok(());
};
// Echo the origin only when it is on the allowlist. Never `*`: this API describes the
// machine it runs on, so any-origin access means any page can fingerprint the node.
let allow_origin = request
.origin
.as_ref()
.filter(|o| config.allowed_origins.iter().any(|a| a == *o))
.cloned();
if request.method == "OPTIONS" {
return write_response(&mut stream, 204, "", allow_origin.as_deref(), true);
}
if let Some(expected) = &config.token {
if !token_ok(expected, request.authorization.as_ref()) {
return write_json(
&mut stream,
401,
r#"{"error":"unauthorized"}"#,
allow_origin.as_deref(),
);
}
}
if request.method != "GET" {
return write_json(
&mut stream,
405,
r#"{"error":"this API is read-only"}"#,
allow_origin.as_deref(),
);
}
let (status, body) = route(&request.path, &config.root);
write_json(&mut stream, status, &body, allow_origin.as_deref())
}
fn route(path: &str, root: &Path) -> (u16, String) {
let rendered = match path {
"/v1/health" => Ok(r#"{"ok":true,"service":"lumbridge-compute","api":"v1"}"#.to_string()),
"/v1/status" => collect_status(root).and_then(|r| Ok(serde_json::to_string(&r)?)),
"/v1/models" => collect_models(root).and_then(|r| Ok(serde_json::to_string(&r)?)),
"/v1/scenes" => collect_scenes(root).and_then(|r| Ok(serde_json::to_string(&r)?)),
"/v1/evals" => collect_evals(root).and_then(|r| Ok(serde_json::to_string(&r)?)),
other => match other.strip_prefix("/v1/scenes/") {
// Exactly one segment: /v1/scenes/a/b is not a route.
Some(name) if !name.is_empty() && !name.contains('/') => {
collect_scene(root, name).and_then(|r| Ok(serde_json::to_string(&r)?))
}
_ => return (404, r#"{"error":"no such route"}"#.to_string()),
},
};
match rendered {
Ok(body) => (200, body),
// A collector fails when the thing does not exist (an unknown Scene) or when the node's
// own config is unreadable. The message is the operator's, and this is a loopback API,
// so passing it through is more useful than flattening it to "error".
Err(error) => (
404,
serde_json::json!({ "error": format!("{error:#}") }).to_string(),
),
}
}
fn write_json(stream: &mut TcpStream, status: u16, body: &str, origin: Option<&str>) -> Result<()> {
write_response(stream, status, body, origin, false)
}
fn write_response(
stream: &mut TcpStream,
status: u16,
body: &str,
origin: Option<&str>,
preflight: bool,
) -> Result<()> {
let reason = match status {
200 => "OK",
204 => "No Content",
401 => "Unauthorized",
404 => "Not Found",
405 => "Method Not Allowed",
_ => "Error",
};
let mut head = format!("HTTP/1.1 {status} {reason}\r\n");
head.push_str("Content-Type: application/json\r\n");
head.push_str(&format!("Content-Length: {}\r\n", body.len()));
// This is live node state; a cached answer is a wrong answer.
head.push_str("Cache-Control: no-store\r\n");
head.push_str("Connection: close\r\n");
if let Some(origin) = origin {
head.push_str(&format!("Access-Control-Allow-Origin: {origin}\r\n"));
// Tell caches the body varies by origin, so an allowed origin's response can never be
// replayed to a disallowed one.
head.push_str("Vary: Origin\r\n");
if preflight {
head.push_str("Access-Control-Allow-Methods: GET, OPTIONS\r\n");
head.push_str("Access-Control-Allow-Headers: Authorization\r\n");
head.push_str("Access-Control-Max-Age: 600\r\n");
}
}
head.push_str("\r\n");
stream.write_all(head.as_bytes())?;
stream.write_all(body.as_bytes())?;
stream.flush()?;
Ok(())
}
/// Drain and discard — kept for symmetry with future routes that accept a body.
#[allow(dead_code)]
fn discard_body(reader: &mut impl Read) {
let mut sink = Vec::new();
let _ = reader.read_to_end(&mut sink);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn token_comparison_rejects_wrong_and_missing_and_prefixless() {
let t = "s3cret-token";
assert!(token_ok(t, Some(&format!("Bearer {t}"))));
assert!(!token_ok(t, None));
assert!(!token_ok(t, Some(&t.to_string()))); // no "Bearer " prefix
assert!(!token_ok(t, Some(&"Bearer wrong".to_string())));
// A correct prefix must not pass — this is the bug constant-time comparison exists for.
assert!(!token_ok(t, Some(&"Bearer s3cret".to_string())));
assert!(!token_ok(t, Some(&"Bearer s3cret-token-plus".to_string())));
}
#[test]
fn unknown_routes_404_and_scene_paths_take_exactly_one_segment() {
let root = Path::new("/nonexistent-root-for-routing-test");
assert_eq!(route("/v1/nope", root).0, 404);
assert_eq!(route("/v1/scenes/a/b", root).0, 404);
assert_eq!(route("/v1/scenes/", root).0, 404);
// Health needs no filesystem, so it answers even on a bogus root.
assert_eq!(route("/v1/health", root).0, 200);
}
#[test]
fn health_body_is_valid_json() {
let (status, body) = route("/v1/health", Path::new("/tmp"));
assert_eq!(status, 200);
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(parsed["ok"], true);
}
}
src/lifecycle.rs
+618
View File
@@ -0,0 +1,618 @@
//! Transactional Scene lifecycle and persisted desired-state recovery.
use anyhow::{bail, Context, Result};
use std::collections::{BTreeMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::os::fd::AsRawFd;
use std::path::Path;
use std::time::Duration;
use crate::config::{find_scene, Model, Registry, Scene};
use crate::governor;
use crate::mem;
use crate::proc::{self, Proc, State};
/// What activating a Scene would do, decided before anything is mutated.
///
/// `activate` renders this for the CLI and the MCP server returns it verbatim,
/// so a plan an operator reads and a plan an agent reads can never drift apart.
#[derive(Debug, Clone)]
pub struct Plan {
pub scene: String,
pub budget_gb: f64,
/// Registered models serving outside the target Scene; stopped first.
pub stop: Vec<String>,
/// Scene models not yet serving, in the order they would be admitted.
pub start: Vec<String>,
}
pub struct TransitionLock {
file: File,
}
impl TransitionLock {
pub fn acquire(root: &Path) -> Result<Self> {
let dir = root.join(".kuda");
fs::create_dir_all(&dir)?;
let path = dir.join("transition.lock");
let file = OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(&path)
.with_context(|| format!("opening transition lock {}", path.display()))?;
let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if result != 0 {
return Err(std::io::Error::last_os_error())
.context("another Scene transition is already running; wait for it to finish");
}
Ok(Self { file })
}
}
impl Drop for TransitionLock {
fn drop(&mut self) {
unsafe {
libc::flock(self.file.as_raw_fd(), libc::LOCK_UN);
}
}
}
pub fn adopt(root: &Path, name: &str) -> Result<()> {
let _lock = TransitionLock::acquire(root)?;
let registry = Registry::load(root)?;
let scene = find_scene(root, name)?;
validate_scene(&registry, &scene)?;
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
let running = governor::running_ids(&registry);
let extras: Vec<String> = running
.iter()
.filter(|id| !target.contains(id.as_str()))
.cloned()
.collect();
if !extras.is_empty() {
bail!(
"cannot adopt '{name}': registered model(s) outside the Scene are serving: {}",
extras.join(", ")
);
}
let previous = State::load_checked(root)?;
let mut adopted = BTreeMap::new();
for id in &scene.models {
let model = &registry.models[id];
if !governor::is_running(model) {
bail!("cannot adopt '{name}': exact model '{id}' is not healthy");
}
let legacy = previous.procs.get(id).with_context(|| {
format!(
"cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute"
)
})?;
let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port)
.with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?;
adopted.insert(id.clone(), captured);
}
let mut state = previous;
state.procs = adopted;
state.desired_scene = Some(name.to_string());
state.active_scene = Some(name.to_string());
state.last_known_good_scene = Some(name.to_string());
state.transition_scene = None;
state.last_error = None;
state.save(root)?;
println!("adopted exact running Scene '{name}' and captured process identities");
Ok(())
}
pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> {
let _lock = TransitionLock::acquire(root)?;
let prior = State::load_checked(root)?;
let prior_active = prior.active_scene.clone();
if dry_run {
return activate_once(root, name, true);
}
match activate_once(root, name, false) {
Ok(()) => {
let mut state = State::load_checked(root)?;
if prior_active.as_deref() != Some(name) {
if let Some(previous) = prior_active {
state.last_known_good_scene = Some(previous);
}
}
state.desired_scene = Some(name.to_string());
state.active_scene = Some(name.to_string());
if state.last_known_good_scene.is_none() {
state.last_known_good_scene = Some(name.to_string());
}
state.transition_scene = None;
state.last_error = None;
state.save(root)?;
println!("Scene '{name}' is active and persisted as desired");
Ok(())
}
Err(target_error) => {
let target_message = format!("activating '{name}' failed: {target_error:#}");
eprintln!("{target_message}");
let transition_started =
State::load_checked(root)?.transition_scene.as_deref() == Some(name);
let cleanup_error = if transition_started {
stop_all_owned(root).err()
} else {
None
};
let rollback = if transition_started && cleanup_error.is_none() {
prior_active
.as_deref()
.filter(|previous| *previous != name)
.map(|previous| (previous.to_string(), activate_once(root, previous, false)))
} else {
None
};
let mut state = State::load_checked(root)?;
state.transition_scene = None;
state.last_error = Some(target_message.clone());
match rollback {
Some((previous, Ok(()))) => {
state.desired_scene = Some(previous.clone());
state.active_scene = Some(previous.clone());
if state.last_known_good_scene.is_none() {
state.last_known_good_scene = Some(previous.clone());
}
state.save(root)?;
eprintln!("rolled back to Scene '{previous}'");
bail!("{target_message}; rolled back to '{previous}'")
}
Some((previous, Err(rollback_error))) => {
state.active_scene = None;
state.save(root)?;
bail!(
"{target_message}; rollback to '{previous}' also failed: {rollback_error:#}"
)
}
None if !transition_started => {
state.save(root)?;
bail!("{target_message}")
}
None if cleanup_error.is_none() => {
state.active_scene = None;
state.save(root)?;
bail!("{target_message}")
}
None => {
state.active_scene = None;
state.save(root)?;
let cleanup_error = cleanup_error.expect("guarded by match condition");
bail!("{target_message}; cleanup also failed: {cleanup_error:#}")
}
}
}
}
}
fn stop_all_owned(root: &Path) -> Result<()> {
let mut state = State::load_checked(root)?;
let owned: Vec<(String, Proc)> = state
.procs
.iter()
.filter(|(_, process)| process.owned_alive())
.map(|(id, process)| (id.clone(), process.clone()))
.collect();
for (id, process) in owned {
proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?;
state.procs.remove(&id);
state.save(root)?;
}
Ok(())
}
pub fn resume(root: &Path) -> Result<()> {
let state = State::load_checked(root)?;
let desired = state
.desired_scene
.clone()
.or_else(|| state.last_known_good_scene.clone())
.context("no desired Scene is persisted; activate or adopt one first")?;
let fallback = state.last_known_good_scene.clone();
match activate(root, &desired, false) {
Ok(()) => Ok(()),
Err(desired_error) => {
let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else {
return Err(desired_error).context("resuming desired Scene");
};
eprintln!(
"desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'"
);
activate(root, &fallback, false).with_context(|| {
format!(
"desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed"
)
})
}
}
}
pub fn deactivate(root: &Path) -> Result<()> {
let _lock = TransitionLock::acquire(root)?;
let mut state = State::load_checked(root)?;
let ids: Vec<String> = state.procs.keys().cloned().collect();
for id in ids {
let process = state.procs[&id].clone();
if process.owned_alive() {
println!(" stopping {id} (pid {})", process.pid);
proc::stop_owned(&process)?;
}
state.procs.remove(&id);
state.save(root)?;
}
state.desired_scene = None;
state.active_scene = None;
state.transition_scene = None;
state.last_error = None;
state.save(root)?;
println!("all Compute-managed models stopped; no Scene is desired");
Ok(())
}
/// Diff the target Scene against what is actually serving. Pure: it decides
/// *what* would change, never *whether* the change is allowed.
fn plan_transition(registry: &Registry, scene: &Scene) -> Plan {
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
let running = governor::running_ids(registry);
let stop: Vec<String> = running
.iter()
.filter(|id| !target.contains(id.as_str()))
.cloned()
.collect();
let mut start: Vec<String> = scene
.models
.iter()
.filter(|id| !governor::is_running(&registry.models[*id]))
.cloned()
.collect();
let listed = scene
.activation
.as_ref()
.and_then(|activation| activation.order.as_deref())
.map(|order| order == "listed")
.unwrap_or(false);
if !listed {
start.sort_by(|a, b| {
registry.models[a]
.footprint_gb
.partial_cmp(&registry.models[b].footprint_gb)
.unwrap()
});
}
Plan {
scene: scene.metadata.name.clone(),
budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB),
stop,
start,
}
}
/// Everything that must hold before the first process is signalled. Runs to
/// completion with nothing mutated, so a rejection here leaves the currently
/// active Scene exactly as it was.
fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> {
for id in &scene.models {
if governor::is_running(&registry.models[id]) {
let process = state.procs.get(id).with_context(|| {
format!(
"'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first"
)
})?;
if !process.owned_alive() {
bail!(
"ownership record for serving model '{id}' is stale; adopt the active Scene first"
);
}
}
}
for id in &plan.stop {
let process = state.procs.get(id).with_context(|| {
format!(
"'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching"
)
})?;
if !process.owned_alive() {
bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid");
}
}
for id in &plan.start {
let model = &registry.models[id];
if let Some(port) = model.serve.port {
if governor::port_open(port) && !governor::is_running(model) {
bail!("port {port} is occupied by a different model; refusing to start '{id}'");
}
}
}
Ok(())
}
/// `scene activate --dry-run` as data rather than as printed lines: the same
/// validation, the same plan, nothing written. Callers that need the plan
/// programmatically use this instead of scraping stdout.
pub fn plan(root: &Path, name: &str) -> Result<Plan> {
let _lock = TransitionLock::acquire(root)?;
let registry = Registry::load(root)?;
let scene = find_scene(root, name)?;
validate_scene(&registry, &scene)?;
let plan = plan_transition(&registry, &scene);
let mut state = State::load_checked(root)?;
state.procs.retain(|_, process| process.owned_alive());
preflight(&registry, &scene, &plan, &state)?;
Ok(plan)
}
fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> {
let registry = Registry::load(root)?;
let scene = find_scene(root, name)?;
validate_scene(&registry, &scene)?;
let plan = plan_transition(&registry, &scene);
println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb);
println!(" stop : {}", format_ids(&plan.stop));
println!(" start: {}", format_ids(&plan.start));
let mut state = State::load_checked(root)?;
state.procs.retain(|_, process| process.owned_alive());
preflight(&registry, &scene, &plan, &state)?;
if dry_run {
println!(" dry run: validation passed; nothing changed");
return Ok(());
}
state.transition_scene = Some(name.to_string());
state.save(root)?;
for id in &plan.stop {
let process = state.procs[id].clone();
println!(" stopping {id} (pid {})", process.pid);
proc::stop_owned(&process)?;
state.procs.remove(id);
state.save(root)?;
}
let mut committed: f64 = scene
.models
.iter()
.filter(|id| governor::is_running(&registry.models[*id]))
.map(|id| registry.models[id].footprint_gb)
.sum();
let wait_healthy = scene
.activation
.as_ref()
.and_then(|activation| activation.wait_healthy)
.unwrap_or(true);
for id in &plan.start {
let model: &Model = &registry.models[id];
// Re-read the pool before every start rather than once per activation: each model
// that comes up consumes real memory, and its true appetite is only knowable after
// it has allocated. A footprint that was optimistic shows up here, on the next
// model, instead of taking the box down.
let available_gb = mem::read().ok().map(|m| m.available_gb);
if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) {
match available_gb {
Some(available) => bail!(
"'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \
{:.1} GB actually available, {:.0} GB margin required",
model.footprint_gb,
committed,
plan.budget_gb,
available,
governor::SAFETY_MARGIN_GB
),
None => bail!(
"'{id}' would exceed the {:.0} GB Scene budget with the safety margin",
plan.budget_gb
),
}
}
let pid = proc::spawn(root, id, model)?;
state.seq += 1;
let process = Proc::capture(pid, state.seq, model.serve.port)
.with_context(|| format!("capturing ownership for newly started '{id}'"))?;
state.procs.insert(id.clone(), process.clone());
state.save(root)?;
committed += model.footprint_gb;
println!(" started {id} (pid {pid}); waiting for exact health");
if wait_healthy && !governor::wait_healthy(model, health_timeout()) {
let _ = proc::stop_owned(&process);
state.procs.remove(id);
state.save(root)?;
bail!("'{id}' did not report its exact health marker before timeout");
}
}
let unhealthy: Vec<String> = scene
.models
.iter()
.filter(|id| !governor::is_running(&registry.models[*id]))
.cloned()
.collect();
if !unhealthy.is_empty() {
bail!(
"Scene '{name}' is incomplete; exact health failed for: {}",
unhealthy.join(", ")
);
}
Ok(())
}
fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> {
let missing: Vec<&String> = scene
.models
.iter()
.filter(|id| !registry.models.contains_key(*id))
.collect();
if !missing.is_empty() {
bail!(
"Scene '{}' references unknown model id(s): {}",
scene.metadata.name,
missing
.iter()
.map(|id| id.as_str())
.collect::<Vec<_>>()
.join(", ")
);
}
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
let footprint: f64 = scene
.models
.iter()
.map(|id| registry.models[id].footprint_gb)
.sum();
if footprint + governor::SAFETY_MARGIN_GB > budget {
bail!(
"Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget",
scene.metadata.name,
footprint + governor::SAFETY_MARGIN_GB,
budget
);
}
Ok(())
}
fn health_timeout() -> Duration {
Duration::from_secs(if cfg!(test) { 5 } else { 900 })
}
fn format_ids(ids: &[String]) -> String {
if ids.is_empty() {
"(none)".to_string()
} else {
ids.join(", ")
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::TcpListener;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
fn fixture_root() -> (PathBuf, u16) {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}"));
fs::create_dir_all(root.join("registry")).unwrap();
fs::create_dir_all(root.join("scenes")).unwrap();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
drop(listener);
let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap();
let bad_port = bad_listener.local_addr().unwrap().port();
drop(bad_listener);
let registry = format!(
"apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n"
);
fs::write(root.join("registry/models.yaml"), registry).unwrap();
fs::write(
root.join("scenes/test.scene.yaml"),
"apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n",
)
.unwrap();
(root, port)
}
#[test]
fn activation_persists_and_deactivation_stops_owned_process() {
let (root, _port) = fixture_root();
activate(&root, "test", false).unwrap();
let state = State::load_checked(&root).unwrap();
assert_eq!(state.desired_scene.as_deref(), Some("test"));
assert_eq!(state.active_scene.as_deref(), Some("test"));
assert!(state.procs["fake"].owned_alive());
deactivate(&root).unwrap();
assert!(State::load_checked(&root).unwrap().procs.is_empty());
fs::remove_dir_all(root).unwrap();
}
#[test]
fn already_serving_scene_requires_explicit_adoption() {
let (root, port) = fixture_root();
let registry = Registry::load(&root).unwrap();
let model = &registry.models["fake"];
let pid = proc::spawn(&root, "fake", model).unwrap();
assert!(governor::wait_healthy(model, Duration::from_secs(5)));
let error = activate(&root, "test", true).unwrap_err().to_string();
assert!(error.contains("adopt the active Scene"));
let owned = Proc::capture(pid, 1, Some(port)).unwrap();
proc::stop_owned(&owned).unwrap();
fs::remove_dir_all(root).unwrap();
}
#[test]
fn preflight_rejection_preserves_healthy_active_scene() {
let (root, _port) = fixture_root();
activate(&root, "test", false).unwrap();
let before = State::load_checked(&root).unwrap();
let pid = before.procs["fake"].pid;
fs::write(
root.join("scenes/invalid.scene.yaml"),
"apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n",
)
.unwrap();
assert!(activate(&root, "invalid", false).is_err());
let after = State::load_checked(&root).unwrap();
assert_eq!(after.active_scene.as_deref(), Some("test"));
assert_eq!(after.desired_scene.as_deref(), Some("test"));
assert_eq!(after.procs["fake"].pid, pid);
assert!(after.procs["fake"].owned_alive());
deactivate(&root).unwrap();
fs::remove_dir_all(root).unwrap();
}
#[test]
fn failed_activation_cleans_up_partial_scene() {
let (root, port) = fixture_root();
fs::write(
root.join("scenes/failing.scene.yaml"),
"apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n",
)
.unwrap();
assert!(activate(&root, "failing", false).is_err());
let state = State::load_checked(&root).unwrap();
assert!(state.procs.is_empty());
assert!(state.active_scene.is_none());
assert!(!governor::port_open(port));
fs::remove_dir_all(root).unwrap();
}
#[test]
fn checked_state_load_rejects_malformed_yaml() {
let (root, _port) = fixture_root();
fs::create_dir_all(root.join(".kuda")).unwrap();
fs::write(root.join(".kuda/state.yaml"), "desired_scene: [").unwrap();
assert!(State::load_checked(&root).is_err());
fs::remove_dir_all(root).unwrap();
}
#[test]
fn state_write_replaces_complete_yaml_atomically() {
let (root, _port) = fixture_root();
let state = State {
desired_scene: Some("test".to_string()),
..State::default()
};
state.save(&root).unwrap();
let loaded = State::load_checked(&root).unwrap();
assert_eq!(loaded.desired_scene.as_deref(), Some("test"));
fs::remove_dir_all(root).unwrap();
}
}
src/main.rs
+431
View File
@@ -0,0 +1,431 @@
//! Lumbridge Compute — safe AI workload orchestration for accelerator nodes.
mod agent;
mod config;
mod eval;
mod gateway;
mod governor;
mod http;
mod lifecycle;
mod mcp;
mod mem;
mod proc;
use anyhow::Result;
use clap::{Parser, Subcommand};
use std::path::{Path, PathBuf};
use std::time::Duration;
use config::{find_scene, load_scenes, Registry, Scene};
use proc::State;
#[derive(Parser)]
#[command(
name = "lumbridge-compute",
version,
about = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes."
)]
struct Cli {
/// Root dir containing registry/ and scenes/ (default: $LUMBRIDGE_COMPUTE_ROOT, $KUDA_ROOT, or current dir)
#[arg(long, global = true)]
root: Option<PathBuf>,
#[command(subcommand)]
cmd: Cmd,
}
#[derive(Subcommand)]
enum Cmd {
/// Governor status: memory, budget, running set, headroom
Status,
/// Manage the model registry
Model {
#[command(subcommand)]
cmd: ModelCmd,
},
/// Manage scenes (bundles of models)
Scene {
#[command(subcommand)]
cmd: SceneCmd,
},
/// Run reproducible capability and serving-performance evaluations
Eval {
#[command(subcommand)]
cmd: EvalCmd,
},
/// Run the memory watchdog in the foreground (kills the newest model before OOM-wedge)
Watchdog {
/// Kill the newest model if MemAvailable dips below this many GB
#[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)]
floor: f64,
},
/// Run the stable streaming gateway in the foreground
Gateway {
#[arg(long, default_value = "127.0.0.1:8011")]
listen: String,
#[arg(long, default_value = "127.0.0.1:8001")]
upstream: String,
},
/// Serve the Governor as a read-only JSON API over HTTP
///
/// A second transport over the same operations the MCP server exposes, for a web UI. It
/// never mutates: no activation, no stop, no eval run.
Api {
/// Loopback by default. This API has no authentication unless --token is set, and it
/// reveals which models this node runs, so widening the bind is an explicit act.
#[arg(long, default_value = "127.0.0.1:8012")]
listen: String,
/// Browser origin permitted to read this API. Repeatable. Empty means no browser may
/// read it; `*` is deliberately not supported, because any page you visit would then
/// be able to fingerprint this machine.
#[arg(long = "allow-origin")]
allow_origin: Vec<String>,
/// Require `Authorization: Bearer <token>` on every request.
#[arg(long)]
token: Option<String>,
},
/// Serve the Governor to AI agents as MCP tools over stdio
Mcp {
/// Let an agent perform a real Scene transition, not just plan one.
/// Off by default: the agent writes its own tool arguments, so the only
/// meaningful gate on a production switch is one the operator sets here.
#[arg(long)]
allow_activate: bool,
},
/// Run the resident supervisor: resume desired Scene, gateway, and memory floor
Agent {
#[arg(long, default_value = "127.0.0.1:8011")]
listen: String,
#[arg(long, default_value = "127.0.0.1:8001")]
upstream: String,
#[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)]
floor: f64,
},
}
#[derive(Subcommand)]
enum ModelCmd {
/// List registered models and their live state
Ls,
}
#[derive(Subcommand)]
enum SceneCmd {
/// List scenes with total footprint
Ls,
/// Show a scene: models, footprints, and the Governor's admission verdict
Show { name: String },
/// Activate a scene: stop what's not in it, admit + start what is
Activate {
name: String,
/// Print the plan without changing anything
#[arg(long)]
dry_run: bool,
},
/// Adopt an already-running exact Scene and bind legacy PIDs to process identities
Adopt { name: String },
/// Resume the persisted desired Scene, falling back to the previous known-good Scene
Resume,
/// Stop all Lumbridge Compute-managed models
Deactivate,
}
#[derive(Subcommand)]
enum EvalCmd {
/// List available evaluation suites
Ls,
/// Run a suite against an OpenAI-compatible endpoint
Run {
suite: String,
#[arg(long, default_value = "http://127.0.0.1:8001/v1")]
base_url: String,
#[arg(long, default_value = "brain")]
model: String,
/// Override the suite's repetitions per case
#[arg(long)]
repeat: Option<u32>,
},
}
fn root_dir(cli: &Cli) -> PathBuf {
cli.root
.clone()
.or_else(|| {
std::env::var("LUMBRIDGE_COMPUTE_ROOT")
.ok()
.map(PathBuf::from)
})
.or_else(|| std::env::var("KUDA_ROOT").ok().map(PathBuf::from))
.unwrap_or_else(|| PathBuf::from("."))
}
fn main() -> Result<()> {
let cli = Cli::parse();
let root = root_dir(&cli);
match &cli.cmd {
Cmd::Status => cmd_status(&root),
Cmd::Model { cmd } => match cmd {
ModelCmd::Ls => cmd_model_ls(&root),
},
Cmd::Scene { cmd } => match cmd {
SceneCmd::Ls => cmd_scene_ls(&root),
SceneCmd::Show { name } => cmd_scene_show(&root, name),
SceneCmd::Activate { name, dry_run } => lifecycle::activate(&root, name, *dry_run),
SceneCmd::Adopt { name } => lifecycle::adopt(&root, name),
SceneCmd::Resume => lifecycle::resume(&root),
SceneCmd::Deactivate => lifecycle::deactivate(&root),
},
Cmd::Eval { cmd } => match cmd {
EvalCmd::Ls => eval::list(&root),
EvalCmd::Run {
suite,
base_url,
model,
repeat,
} => eval::run(&root, suite, base_url, model, *repeat),
},
Cmd::Watchdog { floor } => cmd_watchdog(&root, *floor),
Cmd::Gateway { listen, upstream } => gateway::run(listen, upstream),
Cmd::Api {
listen,
allow_origin,
token,
} => http::run(
listen,
http::Config {
root: root.clone(),
allowed_origins: allow_origin.clone(),
token: token.clone(),
},
),
Cmd::Mcp { allow_activate } => mcp::run(&root, *allow_activate),
Cmd::Agent {
listen,
upstream,
floor,
} => agent::run(&root, listen, upstream, *floor),
}
}
fn cmd_status(root: &Path) -> Result<()> {
let reg = Registry::load(root)?;
let m = mem::read()?;
let committed = governor::committed_gb(&reg);
let running = governor::running_ids(&reg);
let budget = governor::DEFAULT_BUDGET_GB;
// The binding constraint, not the generous one: admission enforces the declared
// budget AND observed memory, so reporting only the declared headroom would promise
// room the next `scene activate` is going to refuse.
let headroom = governor::headroom_gb(committed, budget, Some(m.available_gb)).max(0.0);
let declared_headroom = (budget - committed - governor::SAFETY_MARGIN_GB).max(0.0);
let managed = State::load_checked(root)?;
println!("Lumbridge Compute · governor");
println!(
" memory {:.1} GB total · {:.1} GB available",
m.total_gb, m.available_gb
);
println!(
" budget {:.0} GB (safety margin {:.0} · watchdog floor {:.0})",
budget,
governor::SAFETY_MARGIN_GB,
governor::WATCHDOG_FLOOR_GB
);
println!(
" committed {committed:.1} GB across {} model(s)",
running.len()
);
if headroom < declared_headroom {
println!(
" headroom {headroom:.1} GB admittable (budget allows {declared_headroom:.1}; \
MemAvailable is the tighter limit)"
);
} else {
println!(" headroom {headroom:.1} GB admittable");
}
println!();
println!(
" scene active={} desired={} fallback={}",
managed.active_scene.as_deref().unwrap_or("-"),
managed.desired_scene.as_deref().unwrap_or("-"),
managed.last_known_good_scene.as_deref().unwrap_or("-")
);
if let Some(error) = &managed.last_error {
println!(" last error {error}");
}
if running.is_empty() {
println!(" (no registered models currently serving)");
} else {
println!(" running:");
for id in &running {
let mdl = &reg.models[id];
let tag = if managed
.procs
.get(id)
.is_some_and(|process| process.owned_alive())
{
"compute"
} else {
"ext "
};
println!(
" ● [{tag}] {:<12} {:>5.0} GB :{:<5} {}",
id,
mdl.footprint_gb,
port_str(mdl.serve.port),
mdl.name
);
}
}
Ok(())
}
fn cmd_model_ls(root: &Path) -> Result<()> {
let reg = Registry::load(root)?;
println!(
"{:<12} {:>6} {:<6} {:<6} MODEL",
"ID", "GB", "STATE", "PORT"
);
for (id, m) in &reg.models {
println!(
"{:<12} {:>6.0} {:<6} {:<6} {}",
id,
m.footprint_gb,
if governor::is_running(m) {
"up"
} else {
"down"
},
port_str(m.serve.port),
m.name
);
}
Ok(())
}
fn cmd_scene_ls(root: &Path) -> Result<()> {
let reg = Registry::load(root)?;
let scenes = load_scenes(root)?;
if scenes.is_empty() {
println!("no scenes in {}/scenes", root.display());
return Ok(());
}
println!("{:<12} {:>6} {:<30} MODELS", "SCENE", "GB", "DESCRIPTION");
for s in &scenes {
let total = scene_footprint(s, &reg);
println!(
"{:<12} {:>6.0} {:<30} {}",
s.metadata.name,
total,
truncate(&s.metadata.description, 30),
s.models.join(", ")
);
}
Ok(())
}
fn cmd_scene_show(root: &Path, name: &str) -> Result<()> {
let reg = Registry::load(root)?;
let scene = find_scene(root, name)?;
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
println!(
"scene {} (v{})",
scene.metadata.name, scene.metadata.version
);
if !scene.metadata.description.is_empty() {
println!(" {}", scene.metadata.description);
}
println!(" budget {budget:.0} GB\n models:");
let mut total = 0.0;
let mut missing = Vec::new();
for id in &scene.models {
match reg.models.get(id) {
Some(m) => {
total += m.footprint_gb;
let state = if governor::is_running(m) {
"up"
} else {
"down"
};
println!(
" {:<12} {:>5.0} GB [{:<4}] {}",
id, m.footprint_gb, state, m.name
);
}
None => {
missing.push(id.clone());
println!(" {id:<12} ? [MISSING from registry]");
}
}
}
let needed = total + governor::SAFETY_MARGIN_GB;
println!(
"\n total footprint {total:.1} GB (+{:.0} safety = {needed:.1} GB)",
governor::SAFETY_MARGIN_GB
);
let verdict = if !missing.is_empty() {
format!("✗ {} model(s) missing from registry", missing.len())
} else if needed <= budget {
format!("✓ fits — {:.1} GB to spare", budget - needed)
} else {
format!("✗ exceeds budget by {:.1} GB", needed - budget)
};
println!(" admission {verdict}");
Ok(())
}
fn cmd_watchdog(root: &Path, floor: f64) -> Result<()> {
println!(
"Lumbridge Compute watchdog — killing the newest model if MemAvailable < {floor:.1} GB. Ctrl-C to stop."
);
loop {
let m = mem::read()?;
if m.available_gb < floor {
let mut state = State::load_checked(root)?;
match state.newest_alive() {
Some((id, p)) => {
eprintln!(
"watchdog: MemAvailable {:.1} GB < floor {:.1} — killing newest '{id}' (pid {})",
m.available_gb, floor, p.pid
);
proc::stop_owned(&p)?;
state.procs.remove(&id);
state.save(root)?;
}
None => eprintln!(
"watchdog: MemAvailable {:.1} GB < floor {:.1} but no Lumbridge Compute-managed model to kill!",
m.available_gb, floor
),
}
}
std::thread::sleep(Duration::from_secs(1));
}
}
// ---- helpers ---------------------------------------------------------------
fn scene_footprint(s: &Scene, reg: &Registry) -> f64 {
s.models
.iter()
.filter_map(|id| reg.models.get(id))
.map(|m| m.footprint_gb)
.sum()
}
fn port_str(p: Option<u16>) -> String {
p.map(|p| p.to_string()).unwrap_or_else(|| "-".to_string())
}
fn truncate(s: &str, n: usize) -> String {
if s.chars().count() <= n {
s.to_string()
} else {
format!(
"{}…",
s.chars().take(n.saturating_sub(1)).collect::<String>()
)
}
}
src/mcp.rs
+617
View File
@@ -0,0 +1,617 @@
//! MCP server: the Governor's surface, exposed to agents as tools.
//!
//! Agents already drive Compute by shelling out to the CLI and scraping the
//! column-aligned output. That works until a column moves. Speaking MCP means
//! the agent gets the same numbers the Governor reasons about, as data, with a
//! schema attached — and it means the *shape* of what an agent may do becomes
//! something this file decides rather than something `bash` decides.
//!
//! Two deliberate constraints shape everything below.
//!
//! **Read-first.** Every tool here except `scene_activate` is a pure read.
//! Activating a Scene stops every registered model that is not in it, which on
//! this box means taking down whatever is currently serving — a live voice
//! pipeline included. There is no undo an agent can reach for: if the target
//! Scene then fails to come up, recovery depends on rollback that may itself
//! fail. That asymmetry is why `scene adopt`, `scene resume`, and
//! `scene deactivate` are absent entirely; they are operator verbs whose
//! correctness depends on knowing what the box was doing five minutes ago.
//! `scene_activate` is exposed because planning a switch is genuinely the
//! useful thing an agent wants, and it defaults to planning only.
//!
//! **The safety boundary is the operator's, not the agent's.** A `dry_run`
//! argument defaulting to `true` documents intent but guards nothing: the agent
//! writes the arguments, so it can write `false`. The only boundary an agent
//! cannot cross is one set before it connects, so a real transition also
//! requires `lumbridge-compute mcp --allow-activate`, chosen by the human who
//! launched the server. Without that flag `dry_run: false` is refused, and the
//! refusal says so rather than silently planning instead.
use anyhow::{Context, Result};
use rmcp::handler::server::router::tool::ToolRouter;
use rmcp::handler::server::wrapper::Parameters;
use rmcp::model::{Implementation, ServerCapabilities, ServerInfo};
use rmcp::{tool, tool_handler, tool_router, ErrorData, Json, ServerHandler, ServiceExt};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::os::fd::FromRawFd;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::config::{find_scene, load_scenes, Registry};
use crate::proc::State;
use crate::{eval, governor, lifecycle, mem};
pub fn run(root: &Path, allow_activate: bool) -> Result<()> {
let transport_stdout = hand_over_stdout()?;
// The banner has to go to stderr for the same reason everything else does;
// it doubles as confirmation to the operator that the mutating tool is off.
eprintln!(
"Lumbridge Compute MCP server on stdio · root {} · scene activation {}",
root.display(),
if allow_activate {
"ENABLED (--allow-activate)"
} else {
"disabled; plan only"
}
);
let server = ComputeMcp::new(root.to_path_buf(), allow_activate);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("building the MCP runtime")?;
runtime.block_on(async move {
let transport = (
tokio::io::stdin(),
tokio::fs::File::from_std(transport_stdout),
);
let service = server
.serve(transport)
.await
.context("negotiating the MCP stdio session")?;
service.waiting().await.context("serving MCP over stdio")?;
Ok(())
})
}
/// Hand the real stdout to the transport and point fd 1 at stderr.
///
/// On stdio transport fd 1 *is* the JSON-RPC framing, and this crate reports
/// progress with `println!` throughout — `lifecycle::activate` narrates every
/// stop and start. One such line interleaved into the framing desynchronises
/// the client mid-transition, which is the worst possible moment to lose it.
/// Rather than audit every print (and every future one), move the file
/// descriptor: library output lands on stderr, where operators already read
/// this server's logs, and the protocol gets a channel nothing else can write.
fn hand_over_stdout() -> Result<std::fs::File> {
let saved = unsafe { libc::dup(libc::STDOUT_FILENO) };
if saved < 0 {
return Err(std::io::Error::last_os_error())
.context("duplicating stdout for the MCP transport");
}
if unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) } < 0 {
let error = std::io::Error::last_os_error();
unsafe { libc::close(saved) };
return Err(error).context("redirecting stdout to stderr");
}
// SAFETY: `saved` is a fresh descriptor from `dup` that nothing else owns.
Ok(unsafe { std::fs::File::from_raw_fd(saved) })
}
#[derive(Clone)]
struct ComputeMcp {
root: Arc<PathBuf>,
allow_activate: bool,
tool_router: ToolRouter<ComputeMcp>,
}
#[tool_router]
impl ComputeMcp {
fn new(root: PathBuf, allow_activate: bool) -> Self {
Self {
root: Arc::new(root),
allow_activate,
tool_router: Self::tool_router(),
}
}
/// Governor status for this node: unified-memory totals, the hard budget
/// and its margins, how much is committed by models that are actually
/// serving, how much is still admittable, the persisted Scene state, and
/// the last transition or watchdog error. Read this before reasoning about
/// whether anything else will fit.
#[tool(
name = "governor_status",
annotations(
title = "Governor status",
read_only_hint = true,
open_world_hint = false
)
)]
async fn governor_status(&self) -> Result<Json<StatusReport>, ErrorData> {
let root = self.root.clone();
offload(move || collect_status(&root)).await.map(Json)
}
/// List every model in the node's registry with its declared worst-case
/// footprint, serving port, and whether it is currently serving. Footprints
/// are what admission control is decided against, not observed usage.
#[tool(
name = "model_list",
annotations(
title = "List registered models",
read_only_hint = true,
open_world_hint = false
)
)]
async fn model_list(&self) -> Result<Json<ModelList>, ErrorData> {
let root = self.root.clone();
offload(move || collect_models(&root)).await.map(Json)
}
/// List the Scenes this node can activate, with each one's total footprint
/// and member models. A Scene is a named set of models brought up as a
/// single unit.
#[tool(
name = "scene_list",
annotations(title = "List scenes", read_only_hint = true, open_world_hint = false)
)]
async fn scene_list(&self) -> Result<Json<SceneList>, ErrorData> {
let root = self.root.clone();
offload(move || collect_scenes(&root)).await.map(Json)
}
/// Show one Scene: its models, their footprints and live state, and the
/// Governor's admission verdict — whether the Scene fits its budget once
/// the safety margin is added, and by how much it fits or misses.
#[tool(
name = "scene_show",
annotations(
title = "Show a scene and its admission verdict",
read_only_hint = true,
open_world_hint = false
)
)]
async fn scene_show(
&self,
Parameters(params): Parameters<SceneNameParams>,
) -> Result<Json<SceneDetail>, ErrorData> {
let root = self.root.clone();
offload(move || collect_scene(&root, &params.name))
.await
.map(Json)
}
/// List the evaluation suites available on this node, with their versions
/// and case counts. Running a suite is deliberately not exposed: it drives
/// real load against a serving model for an unbounded time.
#[tool(
name = "eval_list",
annotations(
title = "List eval suites",
read_only_hint = true,
open_world_hint = false
)
)]
async fn eval_list(&self) -> Result<Json<EvalList>, ErrorData> {
let root = self.root.clone();
offload(move || collect_evals(&root)).await.map(Json)
}
/// Plan — or, if the operator allowed it, perform — a Scene transition.
///
/// With `dry_run` true (the default) this runs every check that guards a
/// real transition and returns the models it would stop and start, changing
/// nothing. With `dry_run` false it actually switches the node: models
/// outside the Scene are stopped, which will interrupt anything they serve,
/// and the call blocks until every Scene model reports healthy or the
/// transition rolls back. A real transition additionally requires the
/// server to have been started with `--allow-activate`; without it,
/// `dry_run: false` is refused. Ask a human before setting it.
#[tool(
name = "scene_activate",
annotations(
title = "Activate a scene (plans by default)",
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = false
)
)]
async fn scene_activate(
&self,
Parameters(params): Parameters<SceneActivateParams>,
) -> Result<Json<ActivationReport>, ErrorData> {
if !params.dry_run && !self.allow_activate {
return Err(ErrorData::invalid_params(
"this MCP server is running plan-only: a real Scene transition needs an \
operator to restart it as `lumbridge-compute mcp --allow-activate`. \
Re-run with dry_run: true to see the plan.",
None,
));
}
let root = self.root.clone();
offload(move || activate_scene(&root, &params.name, params.dry_run))
.await
.map(Json)
}
}
#[tool_handler(router = self.tool_router)]
impl ServerHandler for ComputeMcp {
fn get_info(&self) -> ServerInfo {
ServerInfo::new(ServerCapabilities::builder().enable_tools().build())
.with_server_info(Implementation::new(
"lumbridge-compute",
env!("CARGO_PKG_VERSION"),
))
.with_instructions(
"Lumbridge Compute runs many AI models on one unified-memory box. There is a \
single memory pool shared by CPU and GPU, so over-committing does not fail \
gracefully — the machine thrashes and wedges before the OOM killer acts. The \
Governor prevents that by admitting a model only if its declared footprint \
plus a safety margin still fits a hard budget.\n\n\
Start from `governor_status` for what is committed and what is admittable, \
and `scene_show` for whether a named Scene would be admitted. Everything is \
read-only except `scene_activate`, which plans by default. Activating a Scene \
stops every registered model outside it, so treat it as a production change \
and get a human's agreement before asking for a non-dry run.",
)
}
}
// ---- tool parameters -------------------------------------------------------
#[derive(Debug, Deserialize, JsonSchema)]
struct SceneNameParams {
/// Scene name, as reported by `scene_list`.
name: String,
}
#[derive(Debug, Deserialize, JsonSchema)]
struct SceneActivateParams {
/// Scene name, as reported by `scene_list`.
name: String,
/// Validate and report the plan without changing anything. Defaults to
/// true; set it to false only with a human's explicit agreement.
#[serde(default = "yes")]
dry_run: bool,
}
fn yes() -> bool {
true
}
// ---- tool results ----------------------------------------------------------
//
// Lists are wrapped in objects rather than returned bare: MCP structured
// content must be a JSON object, and a named field leaves room to report
// alongside the list later without breaking a client's schema.
#[derive(Debug, Serialize, JsonSchema)]
pub(crate) struct StatusReport {
memory_total_gb: f64,
memory_available_gb: f64,
budget_gb: f64,
safety_margin_gb: f64,
watchdog_floor_gb: f64,
/// Sum of the footprints of every registered model currently serving.
committed_gb: f64,
/// What a new model could still claim without breaching the safety margin.
headroom_gb: f64,
/// Scene whose exact model health was last verified.
active_scene: Option<String>,
/// Scene the node should return to after a restart.
desired_scene: Option<String>,
/// Previous proven Scene, used if the desired one cannot resume.
last_known_good_scene: Option<String>,
last_error: Option<String>,
running: Vec<RunningModel>,
}
#[derive(Debug, Serialize, JsonSchema)]
struct RunningModel {
id: String,
name: String,
footprint_gb: f64,
port: Option<u16>,
/// True when Compute owns this process identity and may signal it. False
/// means the model is serving but was started outside Compute, so a Scene
/// transition will refuse to touch it until it is adopted.
compute_owned: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
pub(crate) struct ModelList {
models: Vec<ModelEntry>,
}
#[derive(Debug, Serialize, JsonSchema)]
struct ModelEntry {
id: String,
name: String,
/// Declared worst-case unified memory once serving, not observed usage.
footprint_gb: f64,
port: Option<u16>,
running: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
pub(crate) struct SceneList {
scenes: Vec<SceneEntry>,
}
#[derive(Debug, Serialize, JsonSchema)]
struct SceneEntry {
name: String,
version: u32,
description: String,
/// Total footprint of the Scene's models that exist in the registry.
footprint_gb: f64,
models: Vec<String>,
}
#[derive(Debug, Serialize, JsonSchema)]
pub(crate) struct SceneDetail {
name: String,
version: u32,
description: String,
budget_gb: f64,
models: Vec<SceneModel>,
/// Model ids the Scene references that the registry does not define.
missing_models: Vec<String>,
footprint_gb: f64,
/// Footprint plus the Governor's safety margin — the figure compared
/// against the budget.
required_gb: f64,
/// The Governor's verdict: would this Scene be admitted?
admits: bool,
verdict: String,
}
#[derive(Debug, Serialize, JsonSchema)]
struct SceneModel {
id: String,
/// Absent when the id is missing from the registry.
name: Option<String>,
footprint_gb: Option<f64>,
running: bool,
}
#[derive(Debug, Serialize, JsonSchema)]
pub(crate) struct EvalList {
suites: Vec<EvalSuite>,
}
#[derive(Debug, Serialize, JsonSchema)]
struct EvalSuite {
name: String,
version: u32,
cases: usize,
description: String,
}
#[derive(Debug, Serialize, JsonSchema)]
struct ActivationReport {
scene: String,
budget_gb: f64,
/// Serving models outside the Scene, which the transition stops first.
stop: Vec<String>,
/// Scene models not yet serving, in the order they would be admitted.
start: Vec<String>,
/// False when this was a plan and the node was left untouched.
applied: bool,
}
// ---- collectors ------------------------------------------------------------
//
// These compose the same primitives the CLI prints from — `governor`, `mem`,
// `config`, `proc::State` — rather than parsing the CLI's output. The numbers
// an agent sees are therefore the numbers admission control uses, by
// construction.
pub(crate) fn collect_status(root: &Path) -> Result<StatusReport> {
let registry = Registry::load(root)?;
let memory = mem::read()?;
let committed = governor::committed_gb(&registry);
let managed = State::load_checked(root)?;
let budget = governor::DEFAULT_BUDGET_GB;
let running = governor::running_ids(&registry)
.into_iter()
.map(|id| {
let model = &registry.models[&id];
RunningModel {
name: model.name.clone(),
footprint_gb: model.footprint_gb,
port: model.serve.port,
compute_owned: managed
.procs
.get(&id)
.is_some_and(|process| process.owned_alive()),
id,
}
})
.collect();
Ok(StatusReport {
memory_total_gb: memory.total_gb,
memory_available_gb: memory.available_gb,
budget_gb: budget,
safety_margin_gb: governor::SAFETY_MARGIN_GB,
watchdog_floor_gb: governor::WATCHDOG_FLOOR_GB,
// `+ 0.0` normalises the negative zero an empty sum can produce, which serialises as
// "-0.0" and reads as a bug to anyone consuming this JSON.
committed_gb: committed + 0.0,
// The BINDING constraint, not the declared one. This used to be
// `budget - committed - margin`, which ignores how much memory the box actually has —
// so on a 62 GB machine with a 100 GB budget it reported 92 GB of headroom that the
// next admission would refuse. cmd_status was fixed when admission started consulting
// MemAvailable; this path was missed, which meant every agent driving the node over
// MCP got the optimistic number.
headroom_gb: governor::headroom_gb(committed, budget, Some(memory.available_gb)).max(0.0),
active_scene: managed.active_scene,
desired_scene: managed.desired_scene,
last_known_good_scene: managed.last_known_good_scene,
last_error: managed.last_error,
running,
})
}
pub(crate) fn collect_models(root: &Path) -> Result<ModelList> {
let registry = Registry::load(root)?;
Ok(ModelList {
models: registry
.models
.iter()
.map(|(id, model)| ModelEntry {
id: id.clone(),
name: model.name.clone(),
footprint_gb: model.footprint_gb,
port: model.serve.port,
running: governor::is_running(model),
})
.collect(),
})
}
pub(crate) fn collect_scenes(root: &Path) -> Result<SceneList> {
let registry = Registry::load(root)?;
Ok(SceneList {
scenes: load_scenes(root)?
.into_iter()
.map(|scene| SceneEntry {
name: scene.metadata.name,
version: scene.metadata.version,
description: scene.metadata.description,
footprint_gb: scene
.models
.iter()
.filter_map(|id| registry.models.get(id))
.map(|model| model.footprint_gb)
.sum(),
models: scene.models,
})
.collect(),
})
}
pub(crate) fn collect_scene(root: &Path, name: &str) -> Result<SceneDetail> {
let registry = Registry::load(root)?;
let scene = find_scene(root, name)?;
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
let mut footprint = 0.0;
let mut missing = Vec::new();
let mut models = Vec::new();
for id in &scene.models {
match registry.models.get(id) {
Some(model) => {
footprint += model.footprint_gb;
models.push(SceneModel {
id: id.clone(),
name: Some(model.name.clone()),
footprint_gb: Some(model.footprint_gb),
running: governor::is_running(model),
});
}
None => {
missing.push(id.clone());
models.push(SceneModel {
id: id.clone(),
name: None,
footprint_gb: None,
running: false,
});
}
}
}
// Same three-way verdict the CLI prints: a missing id is fatal regardless
// of arithmetic, because the Scene cannot be resolved at all.
let required = footprint + governor::SAFETY_MARGIN_GB;
let (admits, verdict) = if !missing.is_empty() {
(
false,
format!("{} model(s) missing from the registry", missing.len()),
)
} else if required <= budget {
(
true,
format!("fits with {:.1} GB to spare", budget - required),
)
} else {
(
false,
format!("exceeds the budget by {:.1} GB", required - budget),
)
};
Ok(SceneDetail {
name: scene.metadata.name,
version: scene.metadata.version,
description: scene.metadata.description,
budget_gb: budget,
models,
missing_models: missing,
footprint_gb: footprint,
required_gb: required,
admits,
verdict,
})
}
pub(crate) fn collect_evals(root: &Path) -> Result<EvalList> {
Ok(EvalList {
suites: eval::catalogue(root)?
.into_iter()
.map(|suite| EvalSuite {
name: suite.name,
version: suite.version,
cases: suite.cases,
description: suite.description,
})
.collect(),
})
}
fn activate_scene(root: &Path, name: &str, dry_run: bool) -> Result<ActivationReport> {
// Plan first either way. On a real transition this is the plan we report;
// `activate` re-derives and re-validates its own under the transition lock,
// so the report describes intent and the lock still owns the truth.
let plan = lifecycle::plan(root, name)?;
if !dry_run {
lifecycle::activate(root, name, false)?;
}
Ok(ActivationReport {
scene: plan.scene,
budget_gb: plan.budget_gb,
stop: plan.stop,
start: plan.start,
applied: !dry_run,
})
}
/// Run a synchronous Governor call off the transport's thread.
///
/// Every read here touches `/proc` and probes serving ports with blocking
/// socket timeouts, and a real transition can sit for minutes waiting on model
/// health. Doing that inline would stall the JSON-RPC reader for the duration,
/// so the synchronous core stays on the blocking pool where it belongs.
async fn offload<T, F>(work: F) -> Result<T, ErrorData>
where
F: FnOnce() -> Result<T> + Send + 'static,
T: Send + 'static,
{
match tokio::task::spawn_blocking(work).await {
Ok(Ok(value)) => Ok(value),
// `{:#}` keeps anyhow's context chain, which is where the actionable
// half of a Compute error lives ("adopt the active Scene first").
Ok(Err(error)) => Err(ErrorData::internal_error(format!("{error:#}"), None)),
Err(join) => Err(ErrorData::internal_error(
format!("Compute worker task failed: {join}"),
None,
)),
}
}
src/mem.rs
+89
View File
@@ -0,0 +1,89 @@
//! Unified-memory sensing. On a shared-memory box there's one pool, so `MemAvailable`
//! from /proc/meminfo is the single source of truth the Governor guards.
use anyhow::{Context, Result};
use std::fs;
pub struct MemInfo {
pub total_gb: f64,
pub available_gb: f64,
}
pub fn read() -> Result<MemInfo> {
let s = fs::read_to_string("/proc/meminfo").context("reading /proc/meminfo")?;
Ok(parse_meminfo(&s))
}
/// Split out from `read` so the parser is testable without a real /proc.
pub fn parse_meminfo(s: &str) -> MemInfo {
let mut total = 0.0;
let mut available = 0.0;
for line in s.lines() {
if let Some(v) = line.strip_prefix("MemTotal:") {
total = kb_to_gb(v);
} else if let Some(v) = line.strip_prefix("MemAvailable:") {
available = kb_to_gb(v);
}
}
MemInfo {
total_gb: total,
available_gb: available,
}
}
/// Parse a " 123456 kB" meminfo value into GB.
fn kb_to_gb(v: &str) -> f64 {
v.split_whitespace()
.next()
.and_then(|n| n.parse::<f64>().ok())
.unwrap_or(0.0)
/ 1024.0
/ 1024.0
}
#[cfg(test)]
mod tests {
use super::*;
const SPARK: &str = "\
MemTotal: 127512345 kB
MemFree: 2048000 kB
MemAvailable: 104857600 kB
Buffers: 123456 kB
";
#[test]
fn reads_total_and_available_and_ignores_other_fields() {
let m = parse_meminfo(SPARK);
assert!((m.total_gb - 121.6).abs() < 0.1, "total was {}", m.total_gb);
assert!(
(m.available_gb - 100.0).abs() < 0.1,
"available was {}",
m.available_gb
);
}
#[test]
fn missing_fields_read_as_zero_rather_than_panicking() {
// A zero here is what makes admission refuse, so an unparseable /proc must not
// look like an empty box with room to spare.
let m = parse_meminfo("SomethingElse: 1 kB\n");
assert_eq!(m.total_gb, 0.0);
assert_eq!(m.available_gb, 0.0);
}
#[test]
fn malformed_values_do_not_panic() {
let m = parse_meminfo("MemTotal: not-a-number kB\nMemAvailable:\n");
assert_eq!(m.total_gb, 0.0);
assert_eq!(m.available_gb, 0.0);
}
#[test]
fn memavailable_is_not_confused_with_memfree() {
// MemFree is much smaller than MemAvailable on a box with page cache; picking the
// wrong one would make the watchdog fire constantly.
let m = parse_meminfo(SPARK);
assert!(m.available_gb > 50.0);
}
}
src/proc.rs
+470
View File
@@ -0,0 +1,470 @@
//! Process lifecycle: spawn a model in its own process group, track it in a state
//! file, and stop it (SIGTERM → SIGKILL to the whole group). The Governor decides
//! *whether* to start; this module *how*.
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::io::Write;
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
use std::process::{self, Command, Stdio};
use std::time::Duration;
use crate::config::Model;
/// One Lumbridge Compute-managed process. `seq` is a monotonic launch counter so the watchdog
/// can always find the *newest* model to sacrifice first.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Proc {
pub pid: i32,
pub seq: u64,
pub port: Option<u16>,
/// Linux boot id plus process start ticks bind ownership to one process
/// incarnation, preventing a reused PID from ever being signalled.
#[serde(default)]
pub boot_id: Option<String>,
#[serde(default)]
pub start_time_ticks: Option<u64>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct State {
#[serde(default)]
pub seq: u64,
#[serde(default)]
pub procs: BTreeMap<String, Proc>,
#[serde(default)]
pub desired_scene: Option<String>,
#[serde(default)]
pub active_scene: Option<String>,
/// Previous proven scene used if the desired scene cannot resume.
#[serde(default)]
pub last_known_good_scene: Option<String>,
#[serde(default)]
pub transition_scene: Option<String>,
#[serde(default)]
pub last_error: Option<String>,
}
fn state_dir(root: &Path) -> PathBuf {
root.join(".kuda")
}
impl State {
pub fn load_checked(root: &Path) -> Result<State> {
let p = state_dir(root).join("state.yaml");
match fs::read_to_string(&p) {
Ok(s) => serde_yaml::from_str(&s)
.with_context(|| format!("parsing persisted state {}", p.display())),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(State::default()),
Err(error) => Err(error).with_context(|| format!("reading {}", p.display())),
}
}
pub fn save(&self, root: &Path) -> Result<()> {
let dir = state_dir(root);
fs::create_dir_all(&dir)?;
let p = dir.join("state.yaml");
let tmp = dir.join(format!(".state.yaml.{}.tmp", process::id()));
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&tmp)
.with_context(|| format!("creating {}", tmp.display()))?;
file.write_all(serde_yaml::to_string(self)?.as_bytes())?;
file.sync_all()?;
fs::rename(&tmp, &p).with_context(|| format!("atomically replacing {}", p.display()))?;
File::open(&dir)
.with_context(|| format!("opening state directory {}", dir.display()))?
.sync_all()
.with_context(|| format!("syncing state directory {}", dir.display()))?;
Ok(())
}
/// Newest still-alive managed model id, if any.
pub fn newest_alive(&self) -> Option<(String, Proc)> {
self.procs
.iter()
.filter(|(_, p)| p.owned_alive())
.max_by_key(|(_, p)| p.seq)
.map(|(id, p)| (id.clone(), p.clone()))
}
}
#[derive(Debug)]
struct ProcessIdentity {
boot_id: String,
start_time_ticks: u64,
process_group_id: i32,
state: char,
}
impl Proc {
/// Capture the identity of a process that Lumbridge Compute just spawned or
/// that an operator explicitly adopted from legacy state.
pub fn capture(pid: i32, seq: u64, port: Option<u16>) -> Result<Proc> {
let identity = process_identity(pid)?;
if identity.process_group_id != pid {
bail!(
"refusing process {pid}: process group {} does not equal pid",
identity.process_group_id
);
}
Ok(Proc {
pid,
seq,
port,
boot_id: Some(identity.boot_id),
start_time_ticks: Some(identity.start_time_ticks),
})
}
pub fn owned_alive(&self) -> bool {
self.validate_owned().is_ok()
}
fn validate_owned(&self) -> Result<()> {
let expected_boot = self
.boot_id
.as_deref()
.context("legacy process record has no boot identity; adopt the active scene first")?;
let expected_start = self
.start_time_ticks
.context("legacy process record has no start identity; adopt the active scene first")?;
let current = process_identity(self.pid)?;
if current.boot_id != expected_boot || current.start_time_ticks != expected_start {
bail!(
"pid {} no longer identifies the process Compute started",
self.pid
);
}
if current.state == 'Z' {
bail!("pid {} is a zombie awaiting reap", self.pid);
}
if current.process_group_id != self.pid {
bail!(
"pid {} now belongs to process group {}; refusing group signal",
self.pid,
current.process_group_id
);
}
Ok(())
}
}
fn process_identity(pid: i32) -> Result<ProcessIdentity> {
let boot_id = fs::read_to_string("/proc/sys/kernel/random/boot_id")
.context("reading Linux boot id")?
.trim()
.to_string();
let stat_path = format!("/proc/{pid}/stat");
let stat = fs::read_to_string(&stat_path)
.with_context(|| format!("reading process identity {stat_path}"))?;
let (state, process_group_id, start_time_ticks) = parse_stat_identity(&stat)?;
Ok(ProcessIdentity {
boot_id,
start_time_ticks,
process_group_id,
state,
})
}
fn parse_stat_identity(stat: &str) -> Result<(char, i32, u64)> {
// `comm` is parenthesized and may contain spaces, so locate its final `)`
// before indexing the fields that follow it. The remainder starts at field 3.
let close = stat
.rfind(')')
.context("malformed /proc stat: missing process name terminator")?;
let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect();
let state = fields
.first()
.and_then(|value| value.chars().next())
.context("malformed /proc stat: missing process state")?;
let process_group_id = fields
.get(2)
.context("malformed /proc stat: missing process group")?
.parse::<i32>()
.context("parsing process group id")?;
let start_time_ticks = fields
.get(19)
.context("malformed /proc stat: missing start time")?
.parse::<u64>()
.context("parsing process start time")?;
Ok((state, process_group_id, start_time_ticks))
}
/// Expand a leading `~/` to `$HOME`.
fn expand(s: &str) -> String {
if let Some(rest) = s.strip_prefix("~/") {
if let Ok(home) = std::env::var("HOME") {
return format!("{home}/{rest}");
}
}
s.to_string()
}
/// The fraction of the pool a vLLM model may reserve, derived from its declared
/// footprint rather than typed separately.
///
/// `footprint_gb` is what the Governor budgets against; `--gpu-memory-utilization` is
/// what actually reserves the memory. Keeping them as two hand-copied numbers means
/// admission control can be arithmetically correct and still wrong about the machine —
/// the registry says a model takes 66 GB while the flag lets it reserve 0.55 of a
/// 121.6 GB pool, which is 67. Deriving one from the other makes the declared footprint
/// the single source of truth.
///
/// An explicit `gpu-memory-utilization` in `serve.args` always wins; this only fills in
/// the gap. Returns `None` when the pool size is unknown, in which case vLLM's own
/// default applies exactly as before.
fn derived_gpu_memory_utilization(m: &Model, mem_total_gb: Option<f64>) -> Option<f64> {
if m.serve.args.contains_key("gpu-memory-utilization") {
return None;
}
let total = mem_total_gb?;
if total <= 0.0 || m.footprint_gb <= 0.0 {
return None;
}
let fraction = m.footprint_gb / total;
// Never hand vLLM a fraction that would reserve the whole box.
(fraction > 0.0 && fraction < 0.95).then_some((fraction * 1000.0).round() / 1000.0)
}
/// `kind`-based builder. argv[0] is the program.
pub fn build_argv(m: &Model, mem_total_gb: Option<f64>) -> Result<Vec<String>> {
if !m.serve.command.is_empty() {
return Ok(m.serve.command.iter().map(|s| expand(s)).collect());
}
match m.serve.kind.as_str() {
"vllm" => {
let executable = m
.serve
.executable
.as_deref()
.map(expand)
.unwrap_or_else(|| "vllm".to_string());
let mut a = vec![executable, "serve".to_string()];
if let Some(w) = &m.serve.weights {
a.push(expand(w));
}
if !m.serve.served_name.is_empty() {
a.push("--served-model-name".into());
a.extend(m.serve.served_name.iter().cloned());
}
a.push("--host".into());
a.push("0.0.0.0".into());
if let Some(p) = m.serve.port {
a.push("--port".into());
a.push(p.to_string());
}
if let Some(fraction) = derived_gpu_memory_utilization(m, mem_total_gb) {
a.push("--gpu-memory-utilization".into());
a.push(format!("{fraction}"));
}
for (k, v) in &m.serve.args {
render_arg(&mut a, k, v);
}
Ok(a)
}
other => bail!(
"model '{}' has no explicit `command` and kind '{}' has no builder yet — \
add a `command: [...]` to the registry entry",
m.name,
other
),
}
}
fn render_arg(out: &mut Vec<String>, k: &str, v: &serde_yaml::Value) {
use serde_yaml::Value;
let flag = format!("--{k}");
match v {
Value::Bool(true) => out.push(flag),
Value::Bool(false) => {}
Value::String(s) => {
out.push(flag);
out.push(s.clone());
}
Value::Number(n) => {
out.push(flag);
out.push(n.to_string());
}
_ => {
out.push(flag);
out.push(
serde_yaml::to_string(v)
.unwrap_or_default()
.trim()
.to_string(),
);
}
}
}
/// Spawn `id`'s model detached in its own process group, logs → compatibility path `.kuda/logs/<id>.log`.
/// Returns the child pid (also its process-group id).
pub fn spawn(root: &Path, id: &str, m: &Model) -> Result<i32> {
let argv = build_argv(m, crate::mem::read().ok().map(|info| info.total_gb))?;
let logdir = state_dir(root).join("logs");
fs::create_dir_all(&logdir)?;
let log = File::create(logdir.join(format!("{id}.log")))?;
let errlog = log.try_clone()?;
let mut cmd = Command::new(&argv[0]);
cmd.args(&argv[1..])
.stdin(Stdio::null())
.stdout(Stdio::from(log))
.stderr(Stdio::from(errlog))
.process_group(0); // own group → clean group-kill, immune to CLI's signals
for (k, v) in &m.serve.env {
cmd.env(k, expand(v));
}
let mut child = cmd
.spawn()
.with_context(|| format!("spawning '{id}': {}", argv.join(" ")))?;
let pid = child.id() as i32;
// A resident agent may outlive many model processes. Reap each child when
// it exits so failed runtimes cannot accumulate as zombies under the agent.
std::thread::spawn(move || {
let _ = child.wait();
});
Ok(pid)
}
/// Stop only the exact process group captured in this ownership record.
pub fn stop_owned(proc: &Proc) -> Result<()> {
if !proc.owned_alive() {
return Ok(());
}
proc.validate_owned()?;
let term = unsafe { libc::kill(-proc.pid, libc::SIGTERM) };
if term != 0 {
return Err(std::io::Error::last_os_error())
.with_context(|| format!("sending SIGTERM to owned process group {}", proc.pid));
}
for _ in 0..60 {
if !proc.owned_alive() {
return Ok(());
}
std::thread::sleep(Duration::from_millis(100));
}
proc.validate_owned()?;
let kill = unsafe { libc::kill(-proc.pid, libc::SIGKILL) };
if kill != 0 {
return Err(std::io::Error::last_os_error())
.with_context(|| format!("sending SIGKILL to owned process group {}", proc.pid));
}
for _ in 0..20 {
if !proc.owned_alive() {
return Ok(());
}
std::thread::sleep(Duration::from_millis(100));
}
bail!("owned process group {} survived SIGKILL", proc.pid)
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a Model the way the registry does — through serde — so these tests exercise
/// the real deserialization path rather than a hand-assembled struct.
fn vllm_model(footprint_gb: f64, extra_args: &str) -> Model {
let yaml = format!(
"name: brain\nfootprint_gb: {footprint_gb}\nserve:\n kind: vllm\n port: 8001\n args:\n max-model-len: 8192\n{extra_args}"
);
serde_yaml::from_str(&yaml).expect("test model yaml")
}
#[test]
fn gpu_memory_utilization_is_derived_from_the_declared_footprint() {
// 66 GB of a 121.6 GB pool is 0.543 — not the 0.55 that used to be typed in
// separately, which is the whole point: one number, not two that can drift.
let m = vllm_model(66.0, "");
assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), Some(0.543));
let argv = build_argv(&m, Some(121.6)).unwrap();
let i = argv
.iter()
.position(|a| a == "--gpu-memory-utilization")
.unwrap();
assert_eq!(argv[i + 1], "0.543");
}
#[test]
fn an_explicit_flag_in_the_registry_always_wins() {
let m = vllm_model(66.0, " gpu-memory-utilization: 0.8\n");
assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None);
// ...and it is rendered exactly once, from serve.args.
let argv = build_argv(&m, Some(121.6)).unwrap();
assert_eq!(
argv.iter()
.filter(|a| *a == "--gpu-memory-utilization")
.count(),
1
);
let i = argv
.iter()
.position(|a| a == "--gpu-memory-utilization")
.unwrap();
assert_eq!(argv[i + 1], "0.8");
}
#[test]
fn unknown_pool_size_leaves_vllms_own_default_alone() {
let m = vllm_model(66.0, "");
assert_eq!(derived_gpu_memory_utilization(&m, None), None);
assert!(!build_argv(&m, None)
.unwrap()
.iter()
.any(|a| a == "--gpu-memory-utilization"));
}
#[test]
fn a_footprint_that_would_claim_the_whole_box_is_not_derived() {
// Better to let vLLM apply its own default than to hand it 0.99 and wedge the box.
let m = vllm_model(120.0, "");
assert_eq!(derived_gpu_memory_utilization(&m, Some(121.6)), None);
}
#[test]
fn nonsense_inputs_do_not_produce_a_flag() {
assert_eq!(
derived_gpu_memory_utilization(&vllm_model(0.0, ""), Some(121.6)),
None
);
assert_eq!(
derived_gpu_memory_utilization(&vllm_model(66.0, ""), Some(0.0)),
None
);
}
#[test]
fn an_explicit_command_bypasses_the_builder_entirely() {
let mut m = vllm_model(66.0, "");
m.serve.command = vec!["python".into(), "-m".into(), "server".into()];
assert_eq!(build_argv(&m, Some(121.6)).unwrap(), m.serve.command);
}
#[test]
fn proc_stat_parser_handles_spaces_in_process_name() {
let stat = "123 (worker process) S 1 123 123 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 4567";
let (state, pgrp, start) = parse_stat_identity(stat).unwrap();
assert_eq!(state, 'S');
assert_eq!(pgrp, 123);
assert_eq!(start, 4567);
}
#[test]
fn legacy_records_are_never_treated_as_owned() {
let proc = Proc {
pid: std::process::id() as i32,
seq: 1,
port: None,
boot_id: None,
start_time_ticks: None,
};
assert!(!proc.owned_alive());
}
}
systemd/lumbridge-compute-agent.service
+20
View File
@@ -0,0 +1,20 @@
[Unit]
Description=Lumbridge Compute resident Scene supervisor and model gateway
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=lumbridge
Environment=HOME=/var/lib/lumbridge
Environment=LUMBRIDGE_COMPUTE_ROOT=$LUMBRIDGE_COMPUTE_ROOT
ExecStart=/usr/local/bin/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3
Restart=always
RestartSec=2
# Model process groups are lifecycle-owned by Compute and must survive an agent
# binary restart; the agent will revalidate their boot/start identities on resume.
KillMode=process
TimeoutStopSec=15
[Install]
WantedBy=multi-user.target