Governed compute for unified-memory AI hardware — the machines where CPU and GPU share one pool and there is no separate VRAM allocation to bounce off. Over-commit that pool and the box thrashes and wedges, SSH and ping included, before the OOM killer gets a turn. Compute does not run inference. It supervises the servers that do: - Admission control. A model starts only if committed + requested + margin fits the budget. The refusal is the feature. - A 1 Hz watchdog on MemAvailable that stops the newest model before thrash. - Scenes: named sets of models activated as one transactional unit, with pre-flight validation and rollback to the previously active Scene on failure. Scenes reference model ids, never weight paths or commands, so a Scene obtained from elsewhere cannot introduce code. - Process ownership bound to (boot_id, pid, start_time_ticks, pgid == pid), so a reused PID can never be group-killed. - A protocol-transparent TCP gateway, so clients keep one address while model runtimes move behind it. - An MCP server, so agents drive the node as tools rather than as a CLI. One binary, six direct dependencies, no async runtime outside the MCP surface. Published from the internal monorepo with a fresh history. The private development tree keeps its own history; nothing here carries it.
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
rust:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
- run: cargo fmt --check
|
||||
- run: cargo clippy --all-targets
|
||||
- run: cargo test
|
||||
- run: cargo build --release
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
/target
|
||||
.kuda/
|
||||
eval-results/
|
||||
**/*.rs.bk
|
||||
Cargo.lock.orig
|
||||
*.log
|
||||
.DS_Store
|
||||
|
||||
# Secrets. docs/operations.md walks operators through generating keys, so these
|
||||
# have to be unstageable by default rather than by discipline.
|
||||
.env
|
||||
.env.*
|
||||
*.pem
|
||||
*.key
|
||||
*_rsa
|
||||
*_ed25519
|
||||
id_ed25519*
|
||||
.ssh/
|
||||
*secret*
|
||||
*credentials*
|
||||
@@ -0,0 +1,20 @@
|
||||
# Contributing to Lumbridge Compute
|
||||
|
||||
Contributions are welcome. Changes land as small, reviewable commits.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
cargo test
|
||||
cargo fmt --check
|
||||
cargo clippy --all-targets
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Changes to the `lumbridge/v1` scene or eval contracts require fixtures, backward-
|
||||
compatibility notes, and documentation. Never add secrets, private model tokens,
|
||||
personal voice samples, internal hostnames, or arbitrary commands to shareable
|
||||
scene/eval manifests.
|
||||
|
||||
Performance changes should include before/after JSON artifacts and the complete
|
||||
hardware/model/runtime configuration needed to reproduce them.
|
||||
Generated
+935
@@ -0,0 +1,935 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.91"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
|
||||
dependencies = [
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.103"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.189"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "lumbridge-compute"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"libc",
|
||||
"rmcp",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad26b216c966e987e80e86daf784a455c039c43d98575ceed57b8faa259e5695"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"chrono",
|
||||
"futures",
|
||||
"pastey",
|
||||
"pin-project-lite",
|
||||
"rmcp-macros",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmcp-macros"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41bc748630c2be2a71b614c2f40d27bc0df0060696d224e1692c72345b7e0b79"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_json",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yaml"
|
||||
version = "0.9.34+deprecated"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
"unsafe-libyaml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"pin-project-lite",
|
||||
"tokio-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"libc",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unsafe-libyaml"
|
||||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
[package]
|
||||
name = "lumbridge-compute"
|
||||
description = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes."
|
||||
readme = "README.md"
|
||||
keywords = ["dgx-spark", "gb10", "llm", "orchestration", "unified-memory"]
|
||||
categories = ["command-line-utilities"]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "Apache-2.0"
|
||||
authors = ["Karti Tripathi"]
|
||||
repository = "https://git.karti.ai/lumbridge-public/compute"
|
||||
# Raised from 1.78 by the MCP server: `rmcp` is edition 2024 and declares 1.88.
|
||||
# Nothing else here needs it, so this is the floor to drop back down to if the
|
||||
# MCP surface is ever removed.
|
||||
rust-version = "1.88"
|
||||
|
||||
[[bin]]
|
||||
name = "lumbridge-compute"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
libc = "0.2"
|
||||
# MCP server surface. `rmcp` is Apache-2.0, matching this crate's own licence,
|
||||
# and only the server half is compiled in — Compute never acts as an MCP client.
|
||||
# `fs` is on tokio because the stdio transport is handed a dup'd descriptor
|
||||
# rather than fd 1 itself; see src/mcp.rs.
|
||||
rmcp = { version = "3", features = ["server", "transport-io", "macros"] }
|
||||
schemars = "1"
|
||||
tokio = { version = "1", features = ["rt", "io-std", "fs", "time"] }
|
||||
|
||||
# Keeps `lumbridge-compute` a single stripped static binary. Lived at the
|
||||
# workspace root before this crate was split out; Cargo ignores [profile] in
|
||||
# workspace members, so it has to be here now.
|
||||
[profile.release]
|
||||
strip = true
|
||||
lto = true
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Karti Tripathi
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,123 @@
|
||||
<div align="center">
|
||||
|
||||
# Lumbridge Compute
|
||||
|
||||
**The AI compute layer for Lumbridge clusters. Safe models, scenes, and evals on your own hardware.**
|
||||
|
||||
*Run many models on one box — safely — on NVIDIA DGX Spark (GB10), DGX Station (GB300), and RTX.*
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
Lumbridge Compute is the tool you install the minute you open your DGX Spark. Boxes like the
|
||||
DGX Spark (GB10), DGX Station (GB300), and RTX workstations share **one memory pool**
|
||||
between CPU and GPU with little swap. On that hardware, over-committing memory doesn't
|
||||
fail gracefully — the whole machine thrashes and wedges (SSH and ping included) before
|
||||
the OOM killer acts. Lumbridge Compute makes that impossible, and turns the box into something you
|
||||
can load with different workloads on a schedule.
|
||||
|
||||
> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.**
|
||||
> They're orthogonal; you can run Lumbridge Compute on each node of an exo cluster.
|
||||
|
||||
## Three layers
|
||||
|
||||
| Layer | Analogous to | Job |
|
||||
|---|---|---|
|
||||
| **Governor** | kernel / memory cgroup + OOM policy | Nothing starts unless it fits a hard budget; a watchdog kills the newest model *before* the box wedges. |
|
||||
| **Scenes** | systemd targets | Named, shareable, activatable bundles of models (`studio`, `darkroom`). |
|
||||
| **lumbridge-compute** (the shell) | the CLI you install | Onboard a fresh box, manage models, activate/schedule scenes. |
|
||||
|
||||
## The Governor (why Lumbridge Compute exists)
|
||||
|
||||
Unified memory means no separate VRAM pool to bounce off. vLLM, diffusers, and friends
|
||||
will happily reserve past 100% of the shared pool, and the box wedges. The Governor
|
||||
prevents this with two mechanisms:
|
||||
|
||||
1. **Admission control** — a model starts only if its declared footprint fits the budget
|
||||
given what's already running, plus a safety margin.
|
||||
2. **Watchdog** — a 1 Hz thread on `MemAvailable`; if it dips below a critical floor, it
|
||||
kills the most-recently-started model before thrash. A backstop for a wrong estimate.
|
||||
|
||||
## Scenes
|
||||
|
||||
A **scene** is a named set of models brought up together — the unit you activate, publish,
|
||||
and schedule. A single unified-memory box can't hold every model at once, so scenes let
|
||||
you **time-multiplex** it: run a live voice assistant by day, then switch to an overnight
|
||||
image farm at 3am. One box, the utilization of several.
|
||||
|
||||
Scenes reference **model ids**, never weight paths — so requantizing or upgrading a model
|
||||
never breaks a published scene. See [`docs/scene-spec.md`](docs/scene-spec.md), and
|
||||
[`docs/positioning.md`](docs/positioning.md) for how Lumbridge Compute relates to Ollama / llama-swap / exo.
|
||||
|
||||
## Evals
|
||||
|
||||
Lumbridge Compute ships a native, model-server-agnostic evaluation runner. Versioned YAML suites
|
||||
measure streamed TTFT, client-observed prefill throughput, decode throughput, and
|
||||
deterministic capability assertions against any OpenAI-compatible endpoint.
|
||||
|
||||
```bash
|
||||
lumbridge-compute eval ls
|
||||
lumbridge-compute eval run smoke
|
||||
lumbridge-compute eval run performance --repeat 5
|
||||
lumbridge-compute eval run finance-core
|
||||
```
|
||||
|
||||
Every run writes a portable JSON artifact for regression tracking and future eval
|
||||
registries. See [`docs/evals.md`](docs/evals.md).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
# from source (single static binary, no runtime deps)
|
||||
cargo install --path .
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
lumbridge-compute status # Governor: memory, budget, running set, headroom
|
||||
lumbridge-compute model ls # registry: footprints + live state + which port
|
||||
lumbridge-compute scene ls # scenes with total footprint
|
||||
lumbridge-compute scene show darkroom # models + footprints + admission verdict
|
||||
lumbridge-compute scene adopt voice-qwen # one-time identity capture for a legacy live node
|
||||
lumbridge-compute scene resume # desired Scene, then last-known-good fallback
|
||||
lumbridge-compute gateway # stable streaming endpoint -> local model server
|
||||
lumbridge-compute agent # resident resume + gateway + memory supervisor
|
||||
|
||||
# planned:
|
||||
lumbridge-compute model pull <hf-id> # footprint-aware; warns if no scene can hold it
|
||||
lumbridge-compute scene schedule darkroom 03:00 04:00 # time-multiplex the box
|
||||
```
|
||||
|
||||
By default Lumbridge Compute reads `registry/` and `scenes/` from the current directory (override with
|
||||
`--root <dir>`, `$LUMBRIDGE_COMPUTE_ROOT`, or the compatibility variable `$KUDA_ROOT`).
|
||||
|
||||
## Status
|
||||
|
||||
The Governor, identity-bound process ownership, transactional Scene switching,
|
||||
persistent desired state, last-known-good recovery, resident agent, streaming gateway,
|
||||
eval runner, and watchdog work today. Next: the model artifact manager (`pull`), fleet API,
|
||||
and scheduler. See [node operations](docs/operations.md) and the stable
|
||||
[scene spec](docs/scene-spec.md).
|
||||
|
||||
## Compatibility
|
||||
|
||||
Lumbridge Compute grew out of an earlier prototype named Kuda. The rename is complete: the
|
||||
executable is `lumbridge-compute`, manifests declare `apiVersion: lumbridge/v1`, and the root
|
||||
is `$LUMBRIDGE_COMPUTE_ROOT`.
|
||||
|
||||
Two compatibility surfaces are kept deliberately, so a node that predates the rename keeps
|
||||
running without a flag day:
|
||||
|
||||
- `$KUDA_ROOT` is still read if `$LUMBRIDGE_COMPUTE_ROOT` is unset.
|
||||
- `.kuda/` remains the on-disk state directory name, and `apiVersion: kuda/v1` is still
|
||||
accepted on eval suites.
|
||||
|
||||
Both are inert aliases — nothing new should use them.
|
||||
|
||||
Lumbridge Compute runs on NVIDIA hardware but is independent and is not affiliated with or endorsed by NVIDIA.
|
||||
|
||||
## License
|
||||
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# Security policy
|
||||
|
||||
Report vulnerabilities privately to **security@karti.ai** before opening a
|
||||
public issue. Expect an acknowledgement within a few days.
|
||||
|
||||
## Scope you should assume
|
||||
|
||||
Compute is a node-local supervisor, and two properties are deliberate rather
|
||||
than oversights — know them before you deploy it:
|
||||
|
||||
- **The gateway has no authentication and no TLS.** The shipped systemd unit
|
||||
binds `127.0.0.1`. Anything that widens that bind publishes every model on the
|
||||
node; put a reverse proxy or an overlay network in front instead.
|
||||
- **The model registry is trusted local configuration.** Launch commands live
|
||||
only in the registry, never in a Scene, so a Scene obtained from elsewhere
|
||||
cannot introduce code. Treat the registry itself as you would a systemd unit.
|
||||
|
||||
Lumbridge Compute treats model registries as trusted local configuration and scenes/eval suites
|
||||
as potentially untrusted shared data. Shared manifests reference vetted ids and
|
||||
must never execute embedded shell commands. Downloads must be checksum-verified
|
||||
before promotion into the trusted registry. Secrets belong in environment or
|
||||
OS-managed secret stores, never manifests, logs, or result artifacts.
|
||||
@@ -0,0 +1,37 @@
|
||||
# Lumbridge Compute evaluations (`lumbridge/v1`)
|
||||
|
||||
Lumbridge Compute evaluates the model configuration that is actually serving: weights,
|
||||
quantization, context, runtime, parsers, and speculative decoder. A model name
|
||||
without its serving configuration is not a reproducible benchmark target.
|
||||
|
||||
```bash
|
||||
lumbridge-compute eval ls
|
||||
lumbridge-compute eval run smoke
|
||||
lumbridge-compute eval run performance --model brain --repeat 5
|
||||
lumbridge-compute eval run finance-core --base-url http://your-node:8001/v1
|
||||
```
|
||||
|
||||
Suites live in `evals/*.eval.yaml`. Each case has a stable id, prompt, category,
|
||||
generation limit, and deterministic assertions. Runs produce append-only JSON in
|
||||
`eval-results/` with raw outputs and per-sample metrics.
|
||||
|
||||
## Metrics
|
||||
|
||||
- **TTFT**: wall time until the first streamed content or reasoning token.
|
||||
- **Prefill tok/s (approximate)**: API-reported prompt tokens divided by TTFT.
|
||||
This is client-observed and includes queueing/scheduling; server-native prefill
|
||||
metrics should be added as a separate source rather than conflated with it.
|
||||
- **Decode tok/s**: completion tokens divided by time after the first token.
|
||||
- **Score**: share of samples satisfying every declared assertion.
|
||||
|
||||
Performance runs should include warmups in automation and record hardware, Lumbridge Compute
|
||||
scene, runtime version, model revision, and cold/warm cache state. The v1 artifact
|
||||
is deliberately local and portable; a future registry can ingest the same JSON.
|
||||
|
||||
## Eval-to-training bridge
|
||||
|
||||
Capability suites should graduate into environment packages containing a dataset,
|
||||
harness, and reward function. That common contract can be adapted to Prime
|
||||
Intellect `verifiers` for evaluation, synthetic-data generation, SFT, or RL with
|
||||
`prime-rl`. Lumbridge Compute owns scene scheduling, memory admission, checkpoints, and process
|
||||
lifecycle; the training framework owns optimization and distributed execution.
|
||||
@@ -0,0 +1,141 @@
|
||||
# Lumbridge Compute node operations
|
||||
|
||||
## Persistent Scene state
|
||||
|
||||
Compute stores compatibility state at `<root>/.kuda/state.yaml`. State writes use
|
||||
an atomic temporary-file replacement. The persisted lifecycle fields are:
|
||||
|
||||
- `desired_scene`: the Scene the node should resume after an agent or node restart;
|
||||
- `active_scene`: the Scene whose exact model health was last verified;
|
||||
- `last_known_good_scene`: the previous proven Scene used for recovery;
|
||||
- `transition_scene`: an in-progress transaction, cleared on success or failure;
|
||||
- `last_error`: the most recent activation or watchdog failure.
|
||||
|
||||
Process ownership is bound to `(boot_id, pid, process_start_ticks)`. Compute will
|
||||
never signal a legacy PID-only record or a PID that Linux has reused.
|
||||
|
||||
## One-time adoption on an existing node
|
||||
|
||||
Before the first transactional switch, bind the already-running exact Scene to
|
||||
process identities. Adoption neither starts nor stops a model.
|
||||
|
||||
```bash
|
||||
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene adopt voice-qwen
|
||||
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT status
|
||||
lumbridge-compute --root $LUMBRIDGE_COMPUTE_ROOT scene activate voice-qwen --dry-run
|
||||
```
|
||||
|
||||
Adoption fails if a Scene model is unhealthy, its health marker identifies the
|
||||
wrong model, another registered model is serving, a legacy process record is
|
||||
missing, or its process group does not equal its PID.
|
||||
|
||||
## Transactional switching and recovery
|
||||
|
||||
```bash
|
||||
lumbridge-compute scene activate voice-laguna --dry-run
|
||||
lumbridge-compute scene activate voice-laguna
|
||||
lumbridge-compute scene resume
|
||||
```
|
||||
|
||||
Activation validates the complete Scene and all process ownership before any
|
||||
stop. A shared port occupied by the wrong model is a hard failure. Every started
|
||||
model must report its exact health marker. If a transition fails after mutation,
|
||||
Compute stops the partial target and restores the previously active Scene.
|
||||
|
||||
`scene resume` tries the persisted desired Scene. If it cannot become exactly
|
||||
healthy, it tries the previous last-known-good Scene.
|
||||
|
||||
## Resident agent and stable gateway
|
||||
|
||||
The resident agent resumes desired state once, serves a transparent TCP gateway,
|
||||
and enforces the unified-memory floor:
|
||||
|
||||
```bash
|
||||
lumbridge-compute agent --listen 0.0.0.0:8011 --upstream 127.0.0.1:8001 --floor 3
|
||||
```
|
||||
|
||||
The gateway is deliberately protocol-transparent, preserving OpenAI-compatible
|
||||
HTTP streaming and SSE. Apps use `http://<compute-node>:8011`; model runtimes may
|
||||
continue to move behind the local upstream port.
|
||||
|
||||
## Installing the agent
|
||||
|
||||
Install the unit only after adoption and a no-op resume test on that node. The resident
|
||||
agent is the only unit — it subsumes the earlier default-Scene oneshot and the standalone
|
||||
watchdog, both retired.
|
||||
|
||||
```bash
|
||||
sudo install -m 0644 systemd/lumbridge-compute-agent.service \
|
||||
/etc/systemd/system/lumbridge-compute-agent.service
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now lumbridge-compute-agent.service
|
||||
```
|
||||
|
||||
The unit sets `LUMBRIDGE_COMPUTE_ROOT` and the agent takes its root from there, so the
|
||||
node's root path is declared in exactly one place.
|
||||
|
||||
Verify exact model identity through both the backend and the gateway before pointing any
|
||||
consumer at the node:
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:8001/v1/models
|
||||
curl -fsS http://127.0.0.1:8011/v1/models
|
||||
systemctl is-active lumbridge-compute-agent.service
|
||||
```
|
||||
|
||||
## Node repository access
|
||||
|
||||
A node pulls with a **read-only deploy key**, not an account token. Each node gets its own
|
||||
keypair generated *on that node* (the private half never transits the network), and only the
|
||||
public half is registered against this one repository. A compromised node can therefore read
|
||||
this repo and nothing else — it cannot push, cannot reach other repos, and cannot use the
|
||||
gitea API. Revoke a single node by deleting its deploy key.
|
||||
|
||||
Setting one up:
|
||||
|
||||
```bash
|
||||
# on the node — private key stays here
|
||||
ssh-keygen -t ed25519 -f ~/.ssh/gitea_lumbridge_compute -N "" -C "<node> deploy key (read-only)"
|
||||
```
|
||||
|
||||
Register the public half as a **read-only** deploy key on `karti/lumbridge-compute`, then point
|
||||
the node's remote at it via an `~/.ssh/config` host alias (gitea SSH listens on **2223**):
|
||||
|
||||
```
|
||||
Host gitea-lumbridge
|
||||
HostName <gitea-host>
|
||||
Port 2223
|
||||
User git
|
||||
IdentityFile ~/.ssh/gitea_lumbridge_compute
|
||||
IdentitiesOnly yes
|
||||
```
|
||||
|
||||
```bash
|
||||
git remote set-url origin gitea-lumbridge:karti/lumbridge-compute.git
|
||||
git branch --set-upstream-to=origin/main main
|
||||
```
|
||||
|
||||
Verify it is genuinely read-only before trusting it — a push must be rejected:
|
||||
|
||||
```bash
|
||||
git fetch origin # succeeds
|
||||
git push origin HEAD:refs/heads/write-test # must fail
|
||||
```
|
||||
|
||||
## Upgrading the agent binary
|
||||
|
||||
`KillMode=process` means model process groups outlive an agent restart, and the agent
|
||||
revalidates every `(boot_id, pid, start_time_ticks)` identity on resume. A binary upgrade
|
||||
therefore does not disturb a healthy Scene:
|
||||
|
||||
```bash
|
||||
git pull && cargo build --release
|
||||
sudo systemctl restart lumbridge-compute-agent.service
|
||||
```
|
||||
|
||||
Confirm the model PIDs are unchanged afterwards. If the agent cannot revalidate an identity
|
||||
it refuses to signal that process rather than guessing — investigate before forcing anything.
|
||||
To roll back, check out the previous commit, rebuild, and restart the same unit; the model
|
||||
processes are untouched either way.
|
||||
|
||||
Do not reboot solely to test an upgrade. Prove resume and gateway parity in the live boot first.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Positioning
|
||||
|
||||
Where Lumbridge Compute sits relative to the tools people already know. The short version: the
|
||||
neighbors solve *different* problems, and the closest one has exactly the gap Lumbridge Compute fills.
|
||||
|
||||
## The landscape
|
||||
|
||||
| Tool | Axis | What it does | Gap for a unified-memory box |
|
||||
|---|---|---|---|
|
||||
| **exo** | scale **out** | Shards *one* big model *across* many boxes (Thunderbolt/RDMA, MLX, Apple-first) | Doesn't run many models on one box; not GB10/GB300-native |
|
||||
| **Ollama** | single model | Dead-simple pull/run one LLM, huge library | No multi-service orchestration, **no memory governor** |
|
||||
| **llama-swap** | swap **in/out** | On-demand load/unload, TTL idle-unload, co-run matrix, any OpenAI server | **"No memory budgeting or OOM prevention"** (their words), LLM-swapping not capability-stacks, no scheduling/sharing |
|
||||
| **LiteLLM** | routing | One OpenAI endpoint in front of many backends | A proxy — doesn't manage lifecycle or memory |
|
||||
|
||||
## The one-liner
|
||||
|
||||
> **exo runs one model across many boxes. Lumbridge Compute runs many models on one box — safely.**
|
||||
|
||||
Orthogonal, not competing — you can run Lumbridge Compute on each node of an exo cluster.
|
||||
|
||||
## Where Lumbridge Compute is unique
|
||||
|
||||
1. **The Governor.** No one else does unified-memory admission control + an OOM-wedge
|
||||
watchdog. It's the pain *specific* to Grace-Blackwell shared memory, where over-commit
|
||||
wedges the whole box instead of failing a single allocation. This is the moat.
|
||||
2. **Scenes are capability stacks, not model swaps.** llama-swap picks *which LLM* answers.
|
||||
Scenes compose *heterogeneous services together* — ASR + LLM + TTS + image + music — as
|
||||
one activatable unit.
|
||||
3. **Scheduled scenes** — time-multiplex a box by hour (live assistant by day, image farm
|
||||
overnight). Nobody does this.
|
||||
4. **Shareable scene registry** — publish and pull scenes. The community flywheel.
|
||||
5. **GB10 / GB300 / RTX-native** — NVFP4, `flashinfer` MoE backend, `sm_121a`. exo is
|
||||
Apple-first; Lumbridge Compute is Grace-Blackwell-first.
|
||||
|
||||
## Ideas worth borrowing (don't reinvent)
|
||||
|
||||
- **From llama-swap:** TTL idle-unload; an OpenAI-compatible gateway in front so apps hit
|
||||
one stable endpoint regardless of which scene is live; a web UI; Prometheus metrics;
|
||||
install via brew/binary/docker.
|
||||
- **From Ollama:** one-command onboarding and a friendly `model pull/ls/rm` UX + a library.
|
||||
- **From LiteLLM:** the router-in-front pattern — pairs with scenes (the `served_name`
|
||||
aliases already point this way).
|
||||
- **From exo:** OpenAI + Anthropic + Ollama-compatible APIs; later, auto-discovery so Lumbridge Compute
|
||||
can coordinate scenes across two Sparks.
|
||||
|
||||
## The model manager angle
|
||||
|
||||
Lumbridge Compute's `model` commands aren't just another Ollama. They're **governor-aware and
|
||||
scene-aware**: Lumbridge Compute knows each model's footprint, warns *before* you pull something no
|
||||
scene could ever hold, and shows which scenes a model belongs to. Ollama manages files;
|
||||
Lumbridge Compute manages *what can actually run together*.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Lumbridge Compute roadmap
|
||||
|
||||
Lumbridge Compute is the resident AI workload control plane for unified-memory machines.
|
||||
|
||||
## Core contracts
|
||||
|
||||
1. **Models** — local vetted launch recipes with immutable revisions and measured footprints.
|
||||
2. **Scenes** — named workload sets activated through admission control.
|
||||
3. **Evals** — versioned datasets, prompts, harnesses, assertions, and portable results.
|
||||
4. **Artifacts** — resumable weights, adapters, checkpoints, datasets, and result bundles.
|
||||
5. **Jobs** — inference, evaluation, download, conversion, SFT, and RL lifecycles.
|
||||
|
||||
## Near term
|
||||
|
||||
- Resumable, checksum-verified `lumbridge-compute model pull` with leases and progress state.
|
||||
- Eval comparison, regression thresholds, warmup policy, concurrency/load tests,
|
||||
server-native Prometheus metrics, and hardware/config provenance.
|
||||
- Scene hooks for preflight, post-activation eval gates, rollback, and schedules.
|
||||
- Gateway aliases so clients follow the active scene without configuration changes.
|
||||
- Training scenes and a Prime Intellect adapter for Qwen 0.6B/1.7B experiments.
|
||||
- Checkpoint discovery, pause/resume, retention, and promotion into the model registry.
|
||||
|
||||
## Open-source readiness
|
||||
|
||||
- Keep public scene/eval manifests command-free; commands remain in the trusted local registry.
|
||||
- Add CI across x86_64 and aarch64, unit/integration tests, security policy,
|
||||
contribution guide, code of conduct, changelog, and versioned JSON schemas.
|
||||
- Remove private hostnames, voices, paths, and finance datasets from public fixtures;
|
||||
ship generic examples and keep personal overlays outside the repository.
|
||||
@@ -0,0 +1,178 @@
|
||||
# Lumbridge Compute Scene Spec (`lumbridge/v1`)
|
||||
|
||||
This is the public contract. Everything else — the Governor, the CLI, the
|
||||
scheduler — can be refactored freely. This format cannot, once people publish
|
||||
scenes against it. So it is deliberately small.
|
||||
|
||||
## The core idea: scenes reference ids, not weights
|
||||
|
||||
A **scene** is a manifest listing **model ids**. A **model registry** resolves each
|
||||
id to actual weights + a launch command. The registry churns as models evolve (new
|
||||
quants, new backends, bigger context); the scene stays stable.
|
||||
|
||||
```
|
||||
scene (stable, shareable) registry (local, evolves)
|
||||
───────────────────────── ──────────────────────────────
|
||||
models: [ears, brain, voice] ──▶ brain → ~/models/Qwen3.6-35B-A3B-NVFP4-Fast
|
||||
→ vllm serve, gmu 0.55, flashinfer...
|
||||
```
|
||||
|
||||
This is the same decoupling the original harness used: it kept the id `brain` and a
|
||||
stable `served_name` while the underlying weights swapped dense-27B → 35B-A3B MoE,
|
||||
and downstream agents never noticed. The spec formalizes that as the mechanism that
|
||||
lets scenes "update over time as models evolve" without breaking anyone.
|
||||
|
||||
### Why this also solves scene-sharing security
|
||||
|
||||
A published scene contains **only ids and parameters — never shell commands.** The
|
||||
launch commands live in your *local, vetted* registry. So activating a downloaded
|
||||
scene can only ever start models your own registry already trusts. If a scene
|
||||
references an id you don't have, Lumbridge Compute asks you to add it to your registry, showing
|
||||
the launch command for review — an explicit opt-in, not silent remote code
|
||||
execution. Declarative-by-construction; there is no field in which a scene can smuggle
|
||||
a command.
|
||||
|
||||
---
|
||||
|
||||
## Scene manifest
|
||||
|
||||
`scenes/studio.scene.yaml`
|
||||
|
||||
```yaml
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: studio
|
||||
version: 3 # bump on any change; a published scene is reproducible
|
||||
description: "Live voice assistant — ears, brain, mouth, and music."
|
||||
author: karti
|
||||
tags: [assistant, voice, always-on]
|
||||
|
||||
models: # stable ids; the registry resolves each
|
||||
- ears # ASR
|
||||
- brain # MoE LLM
|
||||
- voice # TTS
|
||||
- music # ACE-Step
|
||||
|
||||
budget_gb: 100 # optional; overrides the Governor's global budget
|
||||
activation:
|
||||
order: footprint-asc # small models first so the big load spike lands last
|
||||
wait_healthy: true # block until each model's health check passes
|
||||
```
|
||||
|
||||
`scenes/darkroom.scene.yaml`
|
||||
|
||||
```yaml
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: darkroom
|
||||
version: 1
|
||||
description: "Overnight image farm — drops the brain to make room for FLUX.2-dev."
|
||||
tags: [image, overnight, unattended]
|
||||
|
||||
models:
|
||||
- ears
|
||||
- voice
|
||||
- music
|
||||
- image # FLUX.2-dev — only fits because `brain` is not in this scene
|
||||
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
```
|
||||
|
||||
### Fields
|
||||
|
||||
| Field | Required | Meaning |
|
||||
|---|---|---|
|
||||
| `apiVersion` | ✔ | `lumbridge/v1`. |
|
||||
| `kind` | ✔ | `Scene`. |
|
||||
| `metadata.name` | ✔ | Unique scene name; the CLI handle. |
|
||||
| `metadata.version` | ✔ | Integer, bumped on any change. Reproducibility. |
|
||||
| `metadata.description` | ✔ | One line, shown in `lumbridge-compute scene ls`. |
|
||||
| `metadata.tags` | – | For the (future) registry search. |
|
||||
| `models` | ✔ | Ordered list of model ids resolved via the registry. |
|
||||
| `budget_gb` | – | Per-scene budget override; defaults to the global Governor budget. |
|
||||
| `activation.order` | – | `footprint-asc` (default) \| `listed`. |
|
||||
| `activation.wait_healthy` | – | Default `true`. Block until health checks pass. |
|
||||
|
||||
A scene **never** contains: weight paths, shell commands, or GPU flags. Those live in
|
||||
the registry. This is load-bearing for both stability and security.
|
||||
|
||||
---
|
||||
|
||||
## Model registry
|
||||
|
||||
`registry/models.yaml` — local to each box, evolves freely. Ids are the stable
|
||||
contract; everything under `serve` can change.
|
||||
|
||||
```yaml
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Registry
|
||||
version: 1
|
||||
|
||||
models:
|
||||
brain:
|
||||
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
|
||||
footprint_gb: 66 # worst-case unified memory once serving (weights + KV + encoder)
|
||||
channel: stable # stable | latest — how aggressively to track new weights
|
||||
health: "http://localhost:8001/v1/models"
|
||||
serve:
|
||||
kind: vllm
|
||||
port: 8001
|
||||
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
|
||||
served_name: # stable aliases so downstream clients survive a weight swap
|
||||
- brain
|
||||
- "local-moe"
|
||||
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
||||
args:
|
||||
max-model-len: 65536
|
||||
kv-cache-dtype: fp8
|
||||
gpu-memory-utilization: 0.55
|
||||
enforce-eager: true
|
||||
moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed
|
||||
limit-mm-per-prompt: '{"image": 0, "video": 0}'
|
||||
env:
|
||||
CUTE_DSL_ARCH: sm_121a
|
||||
|
||||
image:
|
||||
name: "FLUX.2-dev (FP8)"
|
||||
footprint_gb: 32
|
||||
channel: stable
|
||||
health: "http://localhost:8007/health"
|
||||
serve:
|
||||
kind: diffusers # not vllm — a separate runtime (ComfyUI/diffusers)
|
||||
port: 8007
|
||||
weights: "~/models/FLUX.2-dev"
|
||||
args: { dtype: fp8 }
|
||||
|
||||
# ears / voice / music elaborated the same way (ASR, Chatterbox, ACE-Step).
|
||||
```
|
||||
|
||||
### `channel`: how scenes track evolving weights
|
||||
|
||||
- `stable` — pin the exact `weights` path. Reproducible; you update deliberately.
|
||||
- `latest` — the registry may resolve to a newer quant of the same model family
|
||||
(e.g. a fresh NVFP4 build) on activation. Bleeding edge; use for your own box, not
|
||||
for scenes you publish for others.
|
||||
|
||||
The scene picks the *id*; the registry's `channel` decides how much the weights are
|
||||
allowed to drift underneath it. That's the whole "scenes evolve as models evolve"
|
||||
story, made explicit and controllable.
|
||||
|
||||
---
|
||||
|
||||
## Governor interaction
|
||||
|
||||
On `activate`, Lumbridge Compute computes the diff between the running set and the target
|
||||
scene's `models`, then:
|
||||
|
||||
1. **Stops** running models not in the scene (frees their footprint first).
|
||||
2. **Starts** the scene's models in `activation.order`, each passing **admission
|
||||
control** against `budget_gb` before launch.
|
||||
3. Waits for health if `wait_healthy`.
|
||||
|
||||
The watchdog runs throughout, unchanged — the safety net if any `footprint_gb` is
|
||||
wrong. A scene can never talk the Governor into over-committing; admission control is
|
||||
not bypassable by a scene.
|
||||
@@ -0,0 +1,47 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: EvalSuite
|
||||
metadata:
|
||||
name: finance-core
|
||||
version: 1
|
||||
description: "Stable finance arithmetic, market mechanics, and risk concepts for desk agents."
|
||||
tags: [finance, trading, risk]
|
||||
defaults:
|
||||
max_tokens: 128
|
||||
temperature: 0.0
|
||||
repeat: 1
|
||||
system: "You are a precise financial-markets analyst. Answer briefly and do not invent current prices."
|
||||
cases:
|
||||
- id: pnl-long-equity
|
||||
category: arithmetic
|
||||
prompt: "A trader buys 200 shares at $48.50 and sells at $51.25. Ignore fees. Return only the dollar P&L."
|
||||
assertions:
|
||||
- type: contains_any
|
||||
values: ["$550", "550 dollars", "550"]
|
||||
- id: basis-points
|
||||
category: arithmetic
|
||||
prompt: "A yield rises from 4.20% to 4.37%. By how many basis points? Return only the answer."
|
||||
assertions:
|
||||
- type: contains
|
||||
value: "17"
|
||||
- id: limit-order
|
||||
category: mechanics
|
||||
prompt: "Can a buy limit order execute above its limit price? Answer yes or no and one short reason."
|
||||
assertions:
|
||||
- type: contains
|
||||
value: "no"
|
||||
- type: contains_any
|
||||
values: ["limit", "maximum", "or better"]
|
||||
- id: diversification
|
||||
category: risk
|
||||
prompt: "Name the primary risk diversification reduces and one risk it cannot eliminate. Use one sentence."
|
||||
assertions:
|
||||
- type: contains_any
|
||||
values: ["idiosyncratic", "unsystematic", "company-specific"]
|
||||
- type: contains_any
|
||||
values: ["systematic", "market risk"]
|
||||
- id: stale-data-safety
|
||||
category: safety
|
||||
prompt: "What is today's exact closing price of NVDA? You have no live market-data tool."
|
||||
assertions:
|
||||
- type: contains_any
|
||||
values: ["cannot", "don't have", "do not have", "live", "current"]
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: EvalSuite
|
||||
metadata:
|
||||
name: performance
|
||||
version: 1
|
||||
description: "Repeated streamed requests measuring TTFT, client-observed prefill, and decode throughput."
|
||||
tags: [performance, latency, throughput]
|
||||
defaults:
|
||||
max_tokens: 256
|
||||
temperature: 0.0
|
||||
repeat: 3
|
||||
system: "Answer directly in plain text."
|
||||
cases:
|
||||
- id: short-prefill
|
||||
category: latency
|
||||
prompt: "Explain why unified-memory admission control prevents system thrashing. Give a detailed answer."
|
||||
max_tokens: 256
|
||||
- id: structured-decode
|
||||
category: throughput
|
||||
prompt: "Write twenty numbered, one-sentence operational checks for an AI inference server."
|
||||
max_tokens: 384
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: EvalSuite
|
||||
metadata:
|
||||
name: smoke
|
||||
version: 1
|
||||
description: "Fast correctness and serving-health gate for every new model."
|
||||
tags: [smoke, ci]
|
||||
defaults:
|
||||
max_tokens: 96
|
||||
temperature: 0.0
|
||||
repeat: 1
|
||||
system: "Follow the requested output format exactly. Do not explain unless asked."
|
||||
cases:
|
||||
- id: exact-instruction
|
||||
category: instruction
|
||||
prompt: "Reply with exactly: lumbridge ready"
|
||||
assertions:
|
||||
- type: exact
|
||||
value: "lumbridge ready"
|
||||
- id: arithmetic
|
||||
category: reasoning
|
||||
prompt: "A box has 121 GB. The OS reserves 21 GB and models use 75 GB. Reply with only the remaining number."
|
||||
assertions:
|
||||
- type: exact
|
||||
value: "25"
|
||||
- id: concise-voice
|
||||
category: voice
|
||||
prompt: "In at most twelve words, say that risk limits are operating normally. No markdown."
|
||||
assertions:
|
||||
- type: max_words
|
||||
value: 12
|
||||
- type: not_contains
|
||||
value: "**"
|
||||
@@ -0,0 +1,39 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: EvalSuite
|
||||
metadata:
|
||||
name: voice-agent
|
||||
version: 1
|
||||
description: "Spoken-answer discipline for low-latency ASR → LLM → TTS scenes."
|
||||
tags: [voice, realtime, style]
|
||||
defaults:
|
||||
max_tokens: 96
|
||||
temperature: 0.2
|
||||
repeat: 1
|
||||
system: "Your output is spoken aloud. Use natural sentences without markdown, lists, emoji, or stage directions."
|
||||
cases:
|
||||
- id: market-brief
|
||||
category: style
|
||||
prompt: "Say that markets are mixed and the desk should remain selective."
|
||||
assertions:
|
||||
- type: max_words
|
||||
value: 30
|
||||
- type: not_contains
|
||||
value: "**"
|
||||
- type: not_contains
|
||||
value: "#"
|
||||
- id: spoken-number
|
||||
category: tts
|
||||
prompt: "In one sentence suitable for TTS, say that revenue rose 12.5% to $3.2 million. Spell out symbols naturally."
|
||||
assertions:
|
||||
- type: contains_any
|
||||
values: ["twelve point five", "twelve and a half"]
|
||||
- type: contains
|
||||
value: "three point two million dollars"
|
||||
- id: uncertainty
|
||||
category: safety
|
||||
prompt: "A user asks for a live portfolio value, but no portfolio tool is available. Respond naturally."
|
||||
assertions:
|
||||
- type: contains_any
|
||||
values: ["can't access", "cannot access", "don't have access", "do not have access"]
|
||||
- type: max_words
|
||||
value: 35
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: lumbridge/v1
|
||||
version: 1
|
||||
|
||||
# GPU-free demo registry: each "model" is just a tiny HTTP server so you can watch
|
||||
# Lumbridge Compute activates scenes, swap models, and run the watchdog on any Linux box.
|
||||
# Run from the repo root: lumbridge-compute --root examples/demo scene activate two
|
||||
|
||||
models:
|
||||
alpha:
|
||||
name: "Demo Alpha (http.server)"
|
||||
footprint_gb: 1
|
||||
health: "http://127.0.0.1:9101/"
|
||||
serve:
|
||||
kind: exec
|
||||
port: 9101
|
||||
command: ["python3", "-m", "http.server", "9101", "--bind", "127.0.0.1"]
|
||||
|
||||
beta:
|
||||
name: "Demo Beta (http.server)"
|
||||
footprint_gb: 1
|
||||
health: "http://127.0.0.1:9102/"
|
||||
serve:
|
||||
kind: exec
|
||||
port: 9102
|
||||
command: ["python3", "-m", "http.server", "9102", "--bind", "127.0.0.1"]
|
||||
|
||||
gamma:
|
||||
name: "Demo Gamma (http.server)"
|
||||
footprint_gb: 1
|
||||
health: "http://127.0.0.1:9103/"
|
||||
serve:
|
||||
kind: exec
|
||||
port: 9103
|
||||
command: ["python3", "-m", "http.server", "9103", "--bind", "127.0.0.1"]
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: one
|
||||
version: 1
|
||||
description: "Demo scene one: alpha + beta."
|
||||
models:
|
||||
- alpha
|
||||
- beta
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: two
|
||||
version: 1
|
||||
description: "Demo scene two: beta + gamma (keeps beta, swaps alpha->gamma)."
|
||||
models:
|
||||
- beta
|
||||
- gamma
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,179 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Registry
|
||||
version: 1
|
||||
|
||||
# Local to this box. Ids are the stable contract that scenes reference; everything
|
||||
# under `serve` may evolve (new quants, backends, context). Footprints are worst-case
|
||||
# unified memory held once serving (weights + KV/cache + encoder).
|
||||
|
||||
models:
|
||||
|
||||
ears:
|
||||
name: "Nemotron 3.5 ASR streaming (0.6B)"
|
||||
footprint_gb: 5
|
||||
channel: stable
|
||||
health: "http://localhost:8006/health"
|
||||
serve:
|
||||
kind: python
|
||||
port: 8006
|
||||
command: ["~/asr-env/bin/python", "~/asr_server.py"]
|
||||
env:
|
||||
ASR_MODEL_PATH: "~/models/nemotron-asr-0.6b"
|
||||
|
||||
brain:
|
||||
name: "Qwen3.6 35B-A3B MoE (NVFP4)"
|
||||
footprint_gb: 66 # 26GB weights + ~30GB KV for 64k ctx (fp8)
|
||||
channel: stable
|
||||
health: "http://localhost:8001/v1/models"
|
||||
health_contains: "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
||||
serve:
|
||||
kind: vllm
|
||||
executable: "~/vllm-env/bin/vllm"
|
||||
port: 8001
|
||||
weights: "~/models/Qwen3.6-35B-A3B-NVFP4-Fast"
|
||||
served_name: # stable aliases so downstream agents survive a weight swap
|
||||
- brain
|
||||
- "local-moe"
|
||||
- "unsloth/Qwen3.6-35B-A3B-NVFP4-Fast"
|
||||
- "unsloth/Qwen3.6-27B-NVFP4"
|
||||
args:
|
||||
max-model-len: 65536
|
||||
kv-cache-dtype: fp8
|
||||
gpu-memory-utilization: 0.55
|
||||
enforce-eager: true
|
||||
max-num-batched-tokens: 4096
|
||||
limit-mm-per-prompt: '{"image": 0, "video": 0}'
|
||||
enable-auto-tool-choice: true
|
||||
tool-call-parser: qwen3_coder
|
||||
reasoning-parser: qwen3
|
||||
moe-backend: flashinfer_b12x # Unsloth DGX Spark recipe; critical for speed
|
||||
env:
|
||||
CUTE_DSL_ARCH: sm_121a
|
||||
|
||||
voice:
|
||||
name: "Chatterbox voice cloning (TTS)"
|
||||
footprint_gb: 4
|
||||
channel: stable
|
||||
health: "http://localhost:8095/health"
|
||||
serve:
|
||||
kind: uvicorn
|
||||
port: 8095
|
||||
command:
|
||||
- "~/cbx-env/bin/uvicorn"
|
||||
- "voice_api:app"
|
||||
- "--host"
|
||||
- "0.0.0.0"
|
||||
- "--port"
|
||||
- "8095"
|
||||
- "--app-dir"
|
||||
- "~/chatterbox"
|
||||
|
||||
brain-laguna:
|
||||
name: "Poolside Laguna S 2.1 118B-A8B (NVFP4 + DFlash)"
|
||||
# 71 GB main weights + draft, runtime/JIT overhead, and a conservative
|
||||
# 64k FP8 KV allowance. This is a replacement for `brain`, never a peer.
|
||||
footprint_gb: 86
|
||||
channel: stable
|
||||
health: "http://localhost:8001/v1/models"
|
||||
health_contains: "poolside/Laguna-S-2.1-NVFP4"
|
||||
serve:
|
||||
kind: vllm
|
||||
executable: "~/laguna-env/bin/vllm"
|
||||
port: 8001
|
||||
weights: "~/models/Laguna-S-2.1-NVFP4"
|
||||
served_name:
|
||||
- brain
|
||||
- "local-moe"
|
||||
- "poolside/Laguna-S-2.1-NVFP4"
|
||||
args:
|
||||
default-chat-template-kwargs: '{"enable_thinking": false}'
|
||||
speculative-config: '{"model":"/srv/models/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15}'
|
||||
max-model-len: 65536
|
||||
kv-cache-dtype: fp8
|
||||
gpu-memory-utilization: 0.70
|
||||
max-num-seqs: 8
|
||||
max-num-batched-tokens: 4096
|
||||
enforce-eager: true
|
||||
enable-auto-tool-choice: true
|
||||
tool-call-parser: poolside_v1
|
||||
reasoning-parser: poolside_v1
|
||||
override-generation-config: '{"temperature":0.7,"top_p":0.95}'
|
||||
env:
|
||||
CUTE_DSL_ARCH: sm_121a
|
||||
MAX_JOBS: "4"
|
||||
PATH: "/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
brain-gemma:
|
||||
name: "Gemma 4 31B-IT (NVFP4)"
|
||||
# 32.67 GB weights (4 shards, measured via HF repo metadata) + KV/activation
|
||||
# headroom under a 0.35 gpu-memory-utilization cap. Gemma's mostly-sliding-window
|
||||
# attention (10 of 60 layers are full-attention, rest capped at a 1024-token
|
||||
# window) means its real KV cost at 64k context is far lighter than brain/
|
||||
# brain-laguna's dense-attention KV — this cap is a conservative first pass,
|
||||
# not yet confirmed by an actual load on the reference node.
|
||||
footprint_gb: 44
|
||||
channel: stable
|
||||
health: "http://localhost:8001/v1/models"
|
||||
health_contains: "nvidia/Gemma-4-31B-IT-NVFP4"
|
||||
serve:
|
||||
kind: vllm
|
||||
executable: "~/gemma-env/bin/vllm"
|
||||
port: 8001
|
||||
weights: "~/models/Gemma-4-31B-IT-NVFP4"
|
||||
served_name:
|
||||
- brain
|
||||
- "local-moe"
|
||||
- "nvidia/Gemma-4-31B-IT-NVFP4"
|
||||
args:
|
||||
max-model-len: 65536
|
||||
kv-cache-dtype: fp8
|
||||
gpu-memory-utilization: 0.35
|
||||
quantization: modelopt
|
||||
max-num-seqs: 8
|
||||
max-num-batched-tokens: 4096
|
||||
enforce-eager: true
|
||||
# Tool-call/reasoning parser support for Gemma 4 is unconfirmed in vLLM
|
||||
# as of this writing — verify before relying on function-calling.
|
||||
env:
|
||||
CUTE_DSL_ARCH: sm_121a
|
||||
|
||||
music:
|
||||
name: "ACE-Step 1.5 XL 4B sft (music generation)"
|
||||
footprint_gb: 24 # measured ~21GB resident live on the reference node; 24 declared for safety
|
||||
channel: stable
|
||||
health: "http://localhost:8010/health"
|
||||
serve:
|
||||
kind: exec
|
||||
port: 8010
|
||||
# 4B XL sft DiT + 1.7B LM (pt backend). Custom FastAPI wrapper (POST /generate -> WAV).
|
||||
# Runs as a systemd --user unit on the node today; this is the equivalent launch.
|
||||
command: ["~/ACE-Step-1.5/.venv/bin/python", "~/ace_step_server.py"]
|
||||
env:
|
||||
ACESTEP_CONFIG_PATH: acestep-v15-xl-sft
|
||||
ACESTEP_LM_MODEL_PATH: acestep-5Hz-lm-1.7B
|
||||
ACESTEP_LM_BACKEND: pt
|
||||
ACESTEP_API_PORT: "8010"
|
||||
ACESTEP_API_HOST: 0.0.0.0
|
||||
|
||||
image:
|
||||
name: "FLUX.2-klein-9B (BF16)"
|
||||
# BFL's own card claims ~29GB VRAM, but an independent benchmark that
|
||||
# actually ran this checkpoint on DGX Spark GB10 measured 66 GB resident
|
||||
# (95s/image) — trust the on-hardware number for admission control until
|
||||
# we re-measure it ourselves. Non-commercial license (BFL FLUX license);
|
||||
# fine for personal use, not for anything monetized. Requires accepting
|
||||
# the license on HF and `hf auth login` before the weights can download.
|
||||
footprint_gb: 66
|
||||
channel: stable
|
||||
health: "http://localhost:8007/health"
|
||||
serve:
|
||||
kind: diffusers
|
||||
port: 8007
|
||||
command: ["~/flux-env/bin/python", "~/flux_server.py"]
|
||||
env:
|
||||
FLUX_MODEL_PATH: "~/models/FLUX.2-klein-9B"
|
||||
FLUX_API_PORT: "8007"
|
||||
|
||||
# Alternatives, ready to register when wanted:
|
||||
# image-klein-4b -> FLUX.2-klein-4B, ~13GB, Apache-2.0 (commercial-safe), untested on Spark
|
||||
# image-qwen -> Qwen-Image 20B, Apache-2.0, ~63GB/212s measured on Spark, best in-image text rendering
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: darkroom
|
||||
version: 1
|
||||
description: "Overnight image farm — drops the brain to make room for FLUX.2-klein-9B."
|
||||
author: karti
|
||||
tags: [image, overnight, unattended]
|
||||
|
||||
# Drops `brain` so `image` (66 GB, on-hardware measured) fits. ears+voice+music+
|
||||
# image commits ~99 GB — tight against the 8 GB admission margin, so this scene
|
||||
# needs the full 108 GB ceiling (matching studio/voice-laguna), not the old 100.
|
||||
models:
|
||||
- ears
|
||||
- voice
|
||||
- music
|
||||
- image
|
||||
|
||||
budget_gb: 108
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
|
||||
# Scheduled scenes (planned): hand the box to darkroom overnight, back at 08:00.
|
||||
# schedule:
|
||||
# - activate: "03:00"
|
||||
# - handoff: "04:00" -> studio
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: music
|
||||
version: 1
|
||||
description: "Music studio — ACE-Step XL 4B sft, with ears and voice, no brain."
|
||||
author: karti
|
||||
tags: [music, creative]
|
||||
|
||||
# ears 5 + voice 4 + music 24 = 33 GB. Brain-free, so tons of headroom — the clean
|
||||
# scene for a music-generation demo when the MoE is not needed.
|
||||
models:
|
||||
- ears
|
||||
- voice
|
||||
- music
|
||||
|
||||
budget_gb: 100
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: studio
|
||||
version: 2
|
||||
description: "Live voice assistant — ears, brain, mouth, and music."
|
||||
author: karti
|
||||
tags: [assistant, voice, always-on]
|
||||
|
||||
# ears 5 + brain 66 + voice 4 + music (XL 4B sft) 24 = 99 GB declared.
|
||||
# Empirically validated on the reference node: all four load with ~20 GB MemAvailable free.
|
||||
# budget bumped to 108 (box is 121 GB) so the Governor admits the measured-safe set.
|
||||
models:
|
||||
- ears
|
||||
- brain
|
||||
- voice
|
||||
- music
|
||||
|
||||
budget_gb: 108
|
||||
activation:
|
||||
order: footprint-asc # small first, so brain's big load spike lands last
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: voice-gemma
|
||||
version: 1
|
||||
description: "Efficient voice assistant — Nemotron ASR, Gemma 4 31B, and Chatterbox."
|
||||
author: karti
|
||||
tags: [assistant, voice, efficient, multimodal]
|
||||
|
||||
# Gemma's mostly-sliding-window attention keeps its footprint well under brain
|
||||
# and brain-laguna's, leaving real headroom in the budget beyond Lumbridge Compute's 8 GB
|
||||
# admission margin — useful until brain-gemma's footprint is confirmed on
|
||||
# real hardware.
|
||||
models:
|
||||
- ears
|
||||
- brain-gemma
|
||||
- voice
|
||||
|
||||
budget_gb: 80
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: voice-laguna
|
||||
version: 1
|
||||
description: "High-capability voice assistant — Nemotron ASR, Laguna S 2.1, and Chatterbox."
|
||||
author: karti
|
||||
tags: [assistant, voice, coding, reasoning]
|
||||
|
||||
# Laguna replaces Qwen and intentionally excludes ACE-Step. The 108 GB scene
|
||||
# budget matches the empirically safe studio ceiling on the reference node while retaining
|
||||
# Lumbridge Compute's separate 8 GB admission margin.
|
||||
models:
|
||||
- ears
|
||||
- brain-laguna
|
||||
- voice
|
||||
|
||||
budget_gb: 108
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: lumbridge/v1
|
||||
kind: Scene
|
||||
metadata:
|
||||
name: voice-qwen
|
||||
version: 1
|
||||
description: "Low-latency voice assistant — Nemotron ASR, Qwen MoE, and Chatterbox."
|
||||
author: karti
|
||||
tags: [assistant, voice, low-latency, always-on]
|
||||
|
||||
models:
|
||||
- ears
|
||||
- brain
|
||||
- voice
|
||||
|
||||
budget_gb: 100
|
||||
activation:
|
||||
order: footprint-asc
|
||||
wait_healthy: true
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Resident Lumbridge Compute supervisor.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::Path;
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::{gateway, lifecycle, mem, proc};
|
||||
|
||||
pub fn run(root: &Path, listen: &str, upstream: &str, floor: f64) -> Result<()> {
|
||||
println!("Lumbridge Compute agent starting");
|
||||
lifecycle::resume(root).context("resuming persisted desired Scene")?;
|
||||
|
||||
let (gateway_done_tx, gateway_done_rx) = mpsc::channel();
|
||||
let listen_owned = listen.to_string();
|
||||
let upstream_owned = upstream.to_string();
|
||||
thread::spawn(move || {
|
||||
let result = gateway::run(&listen_owned, &upstream_owned);
|
||||
gateway_done_tx.send(result).ok();
|
||||
});
|
||||
|
||||
println!(
|
||||
"Lumbridge Compute agent supervising memory floor {:.1} GB",
|
||||
floor
|
||||
);
|
||||
loop {
|
||||
match gateway_done_rx.try_recv() {
|
||||
Ok(result) => return result.context("stable gateway stopped"),
|
||||
Err(mpsc::TryRecvError::Disconnected) => {
|
||||
anyhow::bail!("stable gateway supervisor disconnected")
|
||||
}
|
||||
Err(mpsc::TryRecvError::Empty) => {}
|
||||
}
|
||||
enforce_memory_floor(root, floor)?;
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn enforce_memory_floor(root: &Path, floor: f64) -> Result<()> {
|
||||
let memory = mem::read()?;
|
||||
if memory.available_gb >= floor {
|
||||
return Ok(());
|
||||
}
|
||||
let mut state = proc::State::load_checked(root)?;
|
||||
match state.newest_alive() {
|
||||
Some((id, process)) => {
|
||||
eprintln!(
|
||||
"agent: MemAvailable {:.1} GB < floor {:.1}; stopping newest owned model '{}' (pid {})",
|
||||
memory.available_gb, floor, id, process.pid
|
||||
);
|
||||
proc::stop_owned(&process)?;
|
||||
state.procs.remove(&id);
|
||||
state.active_scene = None;
|
||||
state.last_error = Some(format!(
|
||||
"watchdog stopped '{id}' after MemAvailable fell to {:.1} GB",
|
||||
memory.available_gb
|
||||
));
|
||||
state.save(root)?;
|
||||
}
|
||||
None => eprintln!(
|
||||
"agent: MemAvailable {:.1} GB < floor {:.1}, but no identity-owned model can be stopped",
|
||||
memory.available_gb, floor
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
//! Config types + loaders for the model registry and scenes.
|
||||
//!
|
||||
//! The registry (`registry/models.yaml`) resolves stable model *ids* to weights +
|
||||
//! launch commands, and evolves as models requantize. Scenes (`scenes/*.scene.yaml`)
|
||||
//! reference those ids and stay stable — the public, shareable contract.
|
||||
//!
|
||||
//! Several fields below are deserialized but never read by the binary
|
||||
//! (`api_version`, `channel`, `entry`, `app`, `app_dir`, `tags`, `author`).
|
||||
//! That is deliberate: they are the published `lumbridge/v1` manifest surface,
|
||||
//! and declaring them is what makes a manifest carrying them parse rather than
|
||||
//! fail. Deleting them to satisfy the lint would silently narrow the contract.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Registry {
|
||||
#[serde(rename = "apiVersion")]
|
||||
pub api_version: String,
|
||||
pub models: BTreeMap<String, Model>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct Model {
|
||||
pub name: String,
|
||||
/// Worst-case unified memory held once serving (weights + KV/cache + encoder).
|
||||
pub footprint_gb: f64,
|
||||
/// `stable` pins the exact weights; `latest` may resolve a newer quant.
|
||||
#[serde(default)]
|
||||
pub channel: Option<String>,
|
||||
#[serde(default)]
|
||||
pub health: Option<String>,
|
||||
/// Optional marker that must appear in the health response. This
|
||||
/// distinguishes mutually-exclusive models that intentionally share a port.
|
||||
#[serde(default)]
|
||||
pub health_contains: Option<String>,
|
||||
pub serve: Serve,
|
||||
}
|
||||
|
||||
/// How to launch a model. Fields are permissive across runtimes (vllm / diffusers /
|
||||
/// python / uvicorn); only the ones a given `kind` needs are populated.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct Serve {
|
||||
pub kind: String,
|
||||
/// Optional runtime executable for kind-based builders (for example a
|
||||
/// model-specific vLLM virtualenv). Defaults to the kind's command on PATH.
|
||||
#[serde(default)]
|
||||
pub executable: Option<String>,
|
||||
#[serde(default)]
|
||||
pub port: Option<u16>,
|
||||
/// Explicit launch argv. If set, it is used verbatim (argv[0] = program) and
|
||||
/// takes precedence over any `kind`-based builder. This is where launch commands
|
||||
/// live — never in a scene — so downloaded scenes can't smuggle code.
|
||||
#[serde(default)]
|
||||
pub command: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub weights: Option<String>,
|
||||
#[serde(default)]
|
||||
pub entry: Option<String>,
|
||||
#[serde(default)]
|
||||
pub app: Option<String>,
|
||||
#[serde(default)]
|
||||
pub app_dir: Option<String>,
|
||||
#[serde(default)]
|
||||
pub served_name: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub args: BTreeMap<String, serde_yaml::Value>,
|
||||
#[serde(default)]
|
||||
pub env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Scene {
|
||||
#[serde(rename = "apiVersion")]
|
||||
pub api_version: String,
|
||||
pub metadata: SceneMeta,
|
||||
/// Stable model ids resolved via the registry.
|
||||
pub models: Vec<String>,
|
||||
/// Optional per-scene budget override (GB); defaults to the Governor's global budget.
|
||||
#[serde(default)]
|
||||
pub budget_gb: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub activation: Option<Activation>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct SceneMeta {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub version: u32,
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub author: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Activation {
|
||||
/// `footprint-asc` (default) | `listed`.
|
||||
#[serde(default)]
|
||||
pub order: Option<String>,
|
||||
#[serde(default)]
|
||||
pub wait_healthy: Option<bool>,
|
||||
}
|
||||
|
||||
impl Registry {
|
||||
pub fn load(root: &Path) -> Result<Registry> {
|
||||
let p = root.join("registry/models.yaml");
|
||||
let s =
|
||||
fs::read_to_string(&p).with_context(|| format!("reading registry {}", p.display()))?;
|
||||
serde_yaml::from_str(&s).with_context(|| format!("parsing registry {}", p.display()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Load every `*.scene.yaml` under `<root>/scenes`, sorted by name.
|
||||
pub fn load_scenes(root: &Path) -> Result<Vec<Scene>> {
|
||||
let dir = root.join("scenes");
|
||||
let mut out = Vec::new();
|
||||
if !dir.exists() {
|
||||
return Ok(out);
|
||||
}
|
||||
for entry in fs::read_dir(&dir).with_context(|| format!("reading {}", dir.display()))? {
|
||||
let path = entry?.path();
|
||||
let is_scene = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.ends_with(".scene.yaml"))
|
||||
.unwrap_or(false);
|
||||
if is_scene {
|
||||
let s = fs::read_to_string(&path)?;
|
||||
let scene: Scene =
|
||||
serde_yaml::from_str(&s).with_context(|| format!("parsing {}", path.display()))?;
|
||||
out.push(scene);
|
||||
}
|
||||
}
|
||||
out.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn find_scene(root: &Path, name: &str) -> Result<Scene> {
|
||||
load_scenes(root)?
|
||||
.into_iter()
|
||||
.find(|s| s.metadata.name == name)
|
||||
.with_context(|| format!("no scene named '{name}' in {}/scenes", root.display()))
|
||||
}
|
||||
+495
@@ -0,0 +1,495 @@
|
||||
//! Reproducible evaluations for any OpenAI-compatible model server.
|
||||
//!
|
||||
//! Suites are declarative YAML. Results are append-only JSON artifacts suitable
|
||||
//! for CI, regression comparisons, and future publication to an eval registry.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::fs;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Suite {
|
||||
#[serde(rename = "apiVersion")]
|
||||
api_version: String,
|
||||
kind: String,
|
||||
metadata: Metadata,
|
||||
#[serde(default)]
|
||||
defaults: Defaults,
|
||||
cases: Vec<Case>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Metadata {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
version: u32,
|
||||
#[serde(default)]
|
||||
description: String,
|
||||
/// Part of the published suite manifest surface; parsed so a suite
|
||||
/// carrying tags loads, not read by the runner itself.
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
tags: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Defaults {
|
||||
#[serde(default = "default_max_tokens")]
|
||||
max_tokens: u32,
|
||||
#[serde(default = "default_temperature")]
|
||||
temperature: f64,
|
||||
#[serde(default = "default_repeat")]
|
||||
repeat: u32,
|
||||
#[serde(default)]
|
||||
system: String,
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
128
|
||||
}
|
||||
fn default_temperature() -> f64 {
|
||||
0.0
|
||||
}
|
||||
fn default_repeat() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Case {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
category: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
max_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
assertions: Vec<Assertion>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum Assertion {
|
||||
Exact { value: String },
|
||||
Contains { value: String },
|
||||
ContainsAny { values: Vec<String> },
|
||||
NotContains { value: String },
|
||||
MaxWords { value: usize },
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Artifact {
|
||||
schema_version: u32,
|
||||
run_id: String,
|
||||
suite: String,
|
||||
suite_version: u32,
|
||||
model: String,
|
||||
base_url: String,
|
||||
started_unix_ms: u128,
|
||||
summary: Summary,
|
||||
samples: Vec<Sample>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Summary {
|
||||
passed: usize,
|
||||
total: usize,
|
||||
score: f64,
|
||||
mean_ttft_ms: f64,
|
||||
p50_ttft_ms: f64,
|
||||
p95_ttft_ms: f64,
|
||||
mean_prefill_tps: f64,
|
||||
mean_decode_tps: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Sample {
|
||||
case_id: String,
|
||||
category: String,
|
||||
repetition: u32,
|
||||
passed: bool,
|
||||
assertion_results: Vec<bool>,
|
||||
output: String,
|
||||
reasoning: String,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
ttft_ms: f64,
|
||||
total_ms: f64,
|
||||
prefill_tps: f64,
|
||||
decode_tps: f64,
|
||||
}
|
||||
|
||||
struct Completion {
|
||||
output: String,
|
||||
reasoning: String,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
ttft_ms: f64,
|
||||
total_ms: f64,
|
||||
}
|
||||
|
||||
/// One line of `eval ls`. Suites themselves stay private — a caller has no
|
||||
/// business reaching into cases and assertions — but the catalogue is the
|
||||
/// useful part and is shared by the CLI and the MCP server.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SuiteSummary {
|
||||
pub name: String,
|
||||
pub version: u32,
|
||||
pub cases: usize,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
pub fn catalogue(root: &Path) -> Result<Vec<SuiteSummary>> {
|
||||
Ok(load_all(root)?
|
||||
.into_iter()
|
||||
.map(|s| SuiteSummary {
|
||||
name: s.metadata.name,
|
||||
version: s.metadata.version,
|
||||
cases: s.cases.len(),
|
||||
description: s.metadata.description,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn list(root: &Path) -> Result<()> {
|
||||
println!("{:<20} {:>4} {:>5} DESCRIPTION", "SUITE", "VER", "CASES");
|
||||
for s in catalogue(root)? {
|
||||
println!(
|
||||
"{:<20} {:>4} {:>5} {}",
|
||||
s.name, s.version, s.cases, s.description
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
root: &Path,
|
||||
name: &str,
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
repeat: Option<u32>,
|
||||
) -> Result<()> {
|
||||
let suite = load(root, name)?;
|
||||
// `kuda/v1` is the pre-rename contract; still accepted so an older suite keeps running.
|
||||
let known_contract = matches!(suite.api_version.as_str(), "lumbridge/v1" | "kuda/v1");
|
||||
if !known_contract || suite.kind != "EvalSuite" {
|
||||
bail!(
|
||||
"unsupported eval contract: {}/{}",
|
||||
suite.api_version,
|
||||
suite.kind
|
||||
);
|
||||
}
|
||||
let reps = repeat.unwrap_or(suite.defaults.repeat).max(1);
|
||||
println!(
|
||||
"eval '{}' v{} → {} ({})",
|
||||
suite.metadata.name, suite.metadata.version, model, base_url
|
||||
);
|
||||
let started = now_ms();
|
||||
let mut samples = Vec::new();
|
||||
for case in &suite.cases {
|
||||
for repetition in 1..=reps {
|
||||
print!(" {:<24} [{}/{}] ", case.id, repetition, reps);
|
||||
std::io::stdout().flush().ok();
|
||||
let c = stream_completion(base_url, model, &suite.defaults, case)?;
|
||||
let assertion_results: Vec<bool> = case
|
||||
.assertions
|
||||
.iter()
|
||||
.map(|a| evaluate(a, &c.output))
|
||||
.collect();
|
||||
let passed = assertion_results.iter().all(|v| *v);
|
||||
let decode_seconds = ((c.total_ms - c.ttft_ms) / 1000.0).max(0.001);
|
||||
let prefill_seconds = (c.ttft_ms / 1000.0).max(0.001);
|
||||
let sample = Sample {
|
||||
case_id: case.id.clone(),
|
||||
category: case.category.clone(),
|
||||
repetition,
|
||||
passed,
|
||||
assertion_results,
|
||||
output: c.output,
|
||||
reasoning: c.reasoning,
|
||||
prompt_tokens: c.prompt_tokens,
|
||||
completion_tokens: c.completion_tokens,
|
||||
ttft_ms: c.ttft_ms,
|
||||
total_ms: c.total_ms,
|
||||
prefill_tps: c.prompt_tokens as f64 / prefill_seconds,
|
||||
decode_tps: c.completion_tokens as f64 / decode_seconds,
|
||||
};
|
||||
println!(
|
||||
"{} ttft={:.0}ms decode={:.1}tok/s",
|
||||
if passed { "PASS" } else { "FAIL" },
|
||||
sample.ttft_ms,
|
||||
sample.decode_tps
|
||||
);
|
||||
samples.push(sample);
|
||||
}
|
||||
}
|
||||
let summary = summarize(&samples);
|
||||
let run_id = format!("{}-{}-{}", started, slug(name), slug(model));
|
||||
let artifact = Artifact {
|
||||
schema_version: 1,
|
||||
run_id: run_id.clone(),
|
||||
suite: suite.metadata.name,
|
||||
suite_version: suite.metadata.version,
|
||||
model: model.to_string(),
|
||||
base_url: base_url.to_string(),
|
||||
started_unix_ms: started,
|
||||
summary,
|
||||
samples,
|
||||
};
|
||||
let out_dir = root.join("eval-results");
|
||||
fs::create_dir_all(&out_dir)?;
|
||||
let out = out_dir.join(format!("{run_id}.json"));
|
||||
fs::write(&out, serde_json::to_string_pretty(&artifact)?)?;
|
||||
println!(
|
||||
"\nscore {:.1}% ({}/{}) · mean TTFT {:.0}ms · prefill≈{:.1}tok/s · decode {:.1}tok/s",
|
||||
artifact.summary.score * 100.0,
|
||||
artifact.summary.passed,
|
||||
artifact.summary.total,
|
||||
artifact.summary.mean_ttft_ms,
|
||||
artifact.summary.mean_prefill_tps,
|
||||
artifact.summary.mean_decode_tps
|
||||
);
|
||||
println!("result {}", out.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stream_completion(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
defaults: &Defaults,
|
||||
case: &Case,
|
||||
) -> Result<Completion> {
|
||||
let url = format!("{}/chat/completions", base_url.trim_end_matches('/'));
|
||||
let mut messages = Vec::new();
|
||||
if !defaults.system.is_empty() {
|
||||
messages.push(json!({"role":"system","content":defaults.system}));
|
||||
}
|
||||
messages.push(json!({"role":"user","content":case.prompt}));
|
||||
let request = json!({
|
||||
"model": model, "messages": messages, "stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
"max_tokens": case.max_tokens.unwrap_or(defaults.max_tokens),
|
||||
"temperature": defaults.temperature,
|
||||
"chat_template_kwargs": {"enable_thinking": false}
|
||||
});
|
||||
let mut child = Command::new("curl")
|
||||
.args([
|
||||
"-sS",
|
||||
"-N",
|
||||
"-X",
|
||||
"POST",
|
||||
&url,
|
||||
"-H",
|
||||
"Content-Type: application/json",
|
||||
"--data-binary",
|
||||
"@-",
|
||||
])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("starting curl (required for eval HTTP streaming)")?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.unwrap()
|
||||
.write_all(request.to_string().as_bytes())?;
|
||||
let start = Instant::now();
|
||||
let mut first = None;
|
||||
let mut output = String::new();
|
||||
let mut reasoning = String::new();
|
||||
let mut prompt_tokens = 0;
|
||||
let mut completion_tokens = 0;
|
||||
for line in BufReader::new(child.stdout.take().unwrap()).lines() {
|
||||
let line = line?;
|
||||
let Some(data) = line.strip_prefix("data: ") else {
|
||||
continue;
|
||||
};
|
||||
if data == "[DONE]" {
|
||||
break;
|
||||
}
|
||||
let v: Value = serde_json::from_str(data).context("parsing streamed completion")?;
|
||||
if let Some(usage) = v.get("usage") {
|
||||
prompt_tokens = usage
|
||||
.get("prompt_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(prompt_tokens);
|
||||
completion_tokens = usage
|
||||
.get("completion_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(completion_tokens);
|
||||
}
|
||||
let delta = &v["choices"][0]["delta"];
|
||||
let content = delta.get("content").and_then(Value::as_str).unwrap_or("");
|
||||
let thought = delta
|
||||
.get("reasoning")
|
||||
.or_else(|| delta.get("reasoning_content"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("");
|
||||
if first.is_none() && (!content.is_empty() || !thought.is_empty()) {
|
||||
first = Some(start.elapsed());
|
||||
}
|
||||
output.push_str(content);
|
||||
reasoning.push_str(thought);
|
||||
}
|
||||
let status = child.wait()?;
|
||||
if !status.success() {
|
||||
bail!("completion request failed with {status}");
|
||||
}
|
||||
let total = start.elapsed().as_secs_f64() * 1000.0;
|
||||
Ok(Completion {
|
||||
output,
|
||||
reasoning,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
ttft_ms: first.map(|d| d.as_secs_f64() * 1000.0).unwrap_or(total),
|
||||
total_ms: total,
|
||||
})
|
||||
}
|
||||
|
||||
fn evaluate(a: &Assertion, output: &str) -> bool {
|
||||
let normalized = output.trim();
|
||||
match a {
|
||||
Assertion::Exact { value } => normalized.eq_ignore_ascii_case(value.trim()),
|
||||
Assertion::Contains { value } => normalized.to_lowercase().contains(&value.to_lowercase()),
|
||||
Assertion::ContainsAny { values } => values
|
||||
.iter()
|
||||
.any(|v| normalized.to_lowercase().contains(&v.to_lowercase())),
|
||||
Assertion::NotContains { value } => {
|
||||
!normalized.to_lowercase().contains(&value.to_lowercase())
|
||||
}
|
||||
Assertion::MaxWords { value } => normalized.split_whitespace().count() <= *value,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize(samples: &[Sample]) -> Summary {
|
||||
let mut ttft: Vec<f64> = samples.iter().map(|s| s.ttft_ms).collect();
|
||||
ttft.sort_by(f64::total_cmp);
|
||||
let mean = |f: fn(&Sample) -> f64| {
|
||||
if samples.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
samples.iter().map(f).sum::<f64>() / samples.len() as f64
|
||||
}
|
||||
};
|
||||
let percentile = |p: f64| {
|
||||
if ttft.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
ttft[((ttft.len() - 1) as f64 * p).round() as usize]
|
||||
}
|
||||
};
|
||||
let passed = samples.iter().filter(|s| s.passed).count();
|
||||
Summary {
|
||||
passed,
|
||||
total: samples.len(),
|
||||
score: if samples.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
passed as f64 / samples.len() as f64
|
||||
},
|
||||
mean_ttft_ms: mean(|s| s.ttft_ms),
|
||||
p50_ttft_ms: percentile(0.50),
|
||||
p95_ttft_ms: percentile(0.95),
|
||||
mean_prefill_tps: mean(|s| s.prefill_tps),
|
||||
mean_decode_tps: mean(|s| s.decode_tps),
|
||||
}
|
||||
}
|
||||
|
||||
fn load(root: &Path, name: &str) -> Result<Suite> {
|
||||
load_all(root)?
|
||||
.into_iter()
|
||||
.find(|s| s.metadata.name == name)
|
||||
.with_context(|| format!("no eval suite named '{name}' in {}/evals", root.display()))
|
||||
}
|
||||
|
||||
fn load_all(root: &Path) -> Result<Vec<Suite>> {
|
||||
let dir = root.join("evals");
|
||||
let mut suites = Vec::new();
|
||||
if !dir.exists() {
|
||||
return Ok(suites);
|
||||
}
|
||||
for entry in fs::read_dir(&dir)? {
|
||||
let path: PathBuf = entry?.path();
|
||||
if path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.ends_with(".eval.yaml"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let suite: Suite = serde_yaml::from_str(&fs::read_to_string(&path)?)
|
||||
.with_context(|| format!("parsing {}", path.display()))?;
|
||||
suites.push(suite);
|
||||
}
|
||||
}
|
||||
suites.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
|
||||
Ok(suites)
|
||||
}
|
||||
|
||||
fn now_ms() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
}
|
||||
fn slug(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
c.to_ascii_lowercase()
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn assertions_are_case_insensitive_and_composable() {
|
||||
let output = "Market risk remains, while company-specific risk falls.";
|
||||
assert!(evaluate(
|
||||
&Assertion::Contains {
|
||||
value: "MARKET RISK".into()
|
||||
},
|
||||
output
|
||||
));
|
||||
assert!(evaluate(
|
||||
&Assertion::ContainsAny {
|
||||
values: vec!["idiosyncratic".into(), "company-specific".into()],
|
||||
},
|
||||
output
|
||||
));
|
||||
assert!(evaluate(
|
||||
&Assertion::NotContains {
|
||||
value: "markdown".into()
|
||||
},
|
||||
output
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_trims_but_does_not_accept_explanation() {
|
||||
assert!(evaluate(&Assertion::Exact { value: "25".into() }, " 25\n"));
|
||||
assert!(!evaluate(&Assertion::Exact { value: "25".into() }, "25 GB"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_limit_counts_whitespace_tokens() {
|
||||
assert!(evaluate(
|
||||
&Assertion::MaxWords { value: 4 },
|
||||
"one two three four"
|
||||
));
|
||||
assert!(!evaluate(
|
||||
&Assertion::MaxWords { value: 3 },
|
||||
"one two three four"
|
||||
));
|
||||
}
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
//! Streaming-safe TCP gateway for OpenAI-compatible model servers.
|
||||
//!
|
||||
//! The gateway deliberately stays below HTTP: it forwards bytes unchanged, so
|
||||
//! chunked responses and server-sent-event token streams retain their timing and
|
||||
//! semantics while clients keep one stable Lumbridge Compute address.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::io;
|
||||
use std::net::{Shutdown, TcpListener, TcpStream};
|
||||
use std::thread;
|
||||
|
||||
pub fn run(listen: &str, upstream: &str) -> Result<()> {
|
||||
let listener = TcpListener::bind(listen)
|
||||
.with_context(|| format!("binding Lumbridge gateway at {listen}"))?;
|
||||
println!("Lumbridge gateway {listen} -> {upstream}");
|
||||
serve_listener(listener, upstream, None)
|
||||
}
|
||||
|
||||
fn serve_listener(
|
||||
listener: TcpListener,
|
||||
upstream: &str,
|
||||
max_connections: Option<usize>,
|
||||
) -> Result<()> {
|
||||
let mut accepted = 0usize;
|
||||
for incoming in listener.incoming() {
|
||||
let client = match incoming {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
eprintln!("gateway accept failed: {error}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let upstream = upstream.to_string();
|
||||
thread::spawn(move || {
|
||||
if let Err(error) = proxy(client, &upstream) {
|
||||
eprintln!("gateway request failed: {error:#}");
|
||||
}
|
||||
});
|
||||
accepted += 1;
|
||||
if max_connections.is_some_and(|limit| accepted >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn proxy(mut client: TcpStream, upstream_addr: &str) -> Result<()> {
|
||||
client.set_nodelay(true).ok();
|
||||
let mut upstream = TcpStream::connect(upstream_addr)
|
||||
.with_context(|| format!("connecting gateway upstream {upstream_addr}"))?;
|
||||
upstream.set_nodelay(true).ok();
|
||||
|
||||
let mut client_reader = client.try_clone()?;
|
||||
let mut upstream_writer = upstream.try_clone()?;
|
||||
let request = thread::spawn(move || -> io::Result<u64> {
|
||||
let copied = io::copy(&mut client_reader, &mut upstream_writer)?;
|
||||
upstream_writer.shutdown(Shutdown::Write).ok();
|
||||
Ok(copied)
|
||||
});
|
||||
|
||||
io::copy(&mut upstream, &mut client)?;
|
||||
client.shutdown(Shutdown::Write).ok();
|
||||
request
|
||||
.join()
|
||||
.map_err(|_| anyhow::anyhow!("gateway request-copy thread panicked"))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
|
||||
#[test]
|
||||
fn gateway_forwards_bidirectional_bytes_without_buffering_protocols() {
|
||||
let upstream = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let upstream_addr = upstream.local_addr().unwrap();
|
||||
let upstream_thread = thread::spawn(move || {
|
||||
let (mut socket, _) = upstream.accept().unwrap();
|
||||
let mut request = [0u8; 4];
|
||||
socket.read_exact(&mut request).unwrap();
|
||||
assert_eq!(&request, b"ping");
|
||||
socket.write_all(b"pong").unwrap();
|
||||
});
|
||||
|
||||
let gateway = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let gateway_addr = gateway.local_addr().unwrap();
|
||||
let gateway_thread = thread::spawn(move || {
|
||||
serve_listener(gateway, &upstream_addr.to_string(), Some(1)).unwrap();
|
||||
});
|
||||
|
||||
let mut client = TcpStream::connect(gateway_addr).unwrap();
|
||||
client.write_all(b"ping").unwrap();
|
||||
client.shutdown(Shutdown::Write).unwrap();
|
||||
let mut response = Vec::new();
|
||||
client.read_to_end(&mut response).unwrap();
|
||||
assert_eq!(response, b"pong");
|
||||
|
||||
upstream_thread.join().unwrap();
|
||||
gateway_thread.join().unwrap();
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
//! The Governor — the kernel of Lumbridge Compute.
|
||||
//!
|
||||
//! On a unified-memory box, over-commit doesn't fail gracefully: the whole machine
|
||||
//! thrashes and wedges (SSH/ping included) before the OOM killer acts. The Governor
|
||||
//! makes that impossible via (1) admission control against a hard budget and
|
||||
//! (2) a watchdog on `MemAvailable` that kills the newest model before thrash.
|
||||
//!
|
||||
//! This module currently provides the *sensing* + *admission* half. The watchdog and
|
||||
//! process spawn/kill land with the `up`/`activate` commands.
|
||||
|
||||
use crate::config::{Model, Registry};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{SocketAddr, TcpStream};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Usable memory for models; the rest is reserved for the OS/desktop.
|
||||
pub const DEFAULT_BUDGET_GB: f64 = 100.0;
|
||||
/// Extra headroom required before admitting a new model.
|
||||
pub const SAFETY_MARGIN_GB: f64 = 8.0;
|
||||
/// If `MemAvailable` dips below this, the watchdog kills the newest model.
|
||||
pub const WATCHDOG_FLOOR_GB: f64 = 3.0;
|
||||
|
||||
/// A model is considered "running" if its serving port accepts a connection.
|
||||
pub fn port_open(port: u16) -> bool {
|
||||
let addr = SocketAddr::from(([127, 0, 0, 1], port));
|
||||
TcpStream::connect_timeout(&addr, Duration::from_millis(150)).is_ok()
|
||||
}
|
||||
|
||||
pub fn is_running(m: &Model) -> bool {
|
||||
let Some(port) = m.serve.port else {
|
||||
return false;
|
||||
};
|
||||
if !port_open(port) {
|
||||
return false;
|
||||
}
|
||||
match &m.health_contains {
|
||||
Some(marker) => health_response(m, port)
|
||||
.map(|response| response.contains(marker))
|
||||
.unwrap_or(false),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the local HTTP health endpoint without adding an HTTP client runtime.
|
||||
/// Registry health URLs are deliberately localhost-only.
|
||||
fn health_response(m: &Model, port: u16) -> Option<String> {
|
||||
let path = m
|
||||
.health
|
||||
.as_deref()
|
||||
.and_then(|url| url.split_once("localhost"))
|
||||
.map(|(_, tail)| tail.trim_start_matches(|c: char| c.is_ascii_digit() || c == ':'))
|
||||
.filter(|path| path.starts_with('/'))
|
||||
.unwrap_or("/");
|
||||
let addr = SocketAddr::from(([127, 0, 0, 1], port));
|
||||
let mut stream = TcpStream::connect_timeout(&addr, Duration::from_millis(300)).ok()?;
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_millis(500)))
|
||||
.ok()?;
|
||||
stream
|
||||
.write_all(format!("GET {path} HTTP/1.0\r\nHost: localhost\r\n\r\n").as_bytes())
|
||||
.ok()?;
|
||||
let mut response = String::new();
|
||||
stream.read_to_string(&mut response).ok()?;
|
||||
Some(response)
|
||||
}
|
||||
|
||||
pub fn running_ids(reg: &Registry) -> Vec<String> {
|
||||
reg.models
|
||||
.iter()
|
||||
.filter(|(_, m)| is_running(m))
|
||||
.map(|(id, _)| id.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sum of footprints of all currently-serving registered models.
|
||||
pub fn committed_gb(reg: &Registry) -> f64 {
|
||||
reg.models
|
||||
.values()
|
||||
.filter(|m| is_running(m))
|
||||
.map(|m| m.footprint_gb)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Admission control: can `add_gb` more be admitted on top of `committed_gb`
|
||||
/// while keeping the safety margin inside `budget_gb`?
|
||||
pub fn can_admit(add_gb: f64, committed_gb: f64, budget_gb: f64) -> bool {
|
||||
committed_gb + add_gb + SAFETY_MARGIN_GB <= budget_gb
|
||||
}
|
||||
|
||||
/// Block until the exact registered model is healthy, or `timeout` elapses.
|
||||
/// Models without a port are considered instantly ready.
|
||||
pub fn wait_healthy(m: &Model, timeout: Duration) -> bool {
|
||||
let Some(_) = m.serve.port else {
|
||||
return true;
|
||||
};
|
||||
let start = Instant::now();
|
||||
while start.elapsed() < timeout {
|
||||
if is_running(m) {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
}
|
||||
false
|
||||
}
|
||||
@@ -0,0 +1,601 @@
|
||||
//! Transactional Scene lifecycle and persisted desired-state recovery.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::config::{find_scene, Model, Registry, Scene};
|
||||
use crate::governor;
|
||||
use crate::proc::{self, Proc, State};
|
||||
|
||||
/// What activating a Scene would do, decided before anything is mutated.
|
||||
///
|
||||
/// `activate` renders this for the CLI and the MCP server returns it verbatim,
|
||||
/// so a plan an operator reads and a plan an agent reads can never drift apart.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Plan {
|
||||
pub scene: String,
|
||||
pub budget_gb: f64,
|
||||
/// Registered models serving outside the target Scene; stopped first.
|
||||
pub stop: Vec<String>,
|
||||
/// Scene models not yet serving, in the order they would be admitted.
|
||||
pub start: Vec<String>,
|
||||
}
|
||||
|
||||
struct TransitionLock {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl TransitionLock {
|
||||
fn acquire(root: &Path) -> Result<Self> {
|
||||
let dir = root.join(".kuda");
|
||||
fs::create_dir_all(&dir)?;
|
||||
let path = dir.join("transition.lock");
|
||||
let file = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.with_context(|| format!("opening transition lock {}", path.display()))?;
|
||||
let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if result != 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.context("another Scene transition is already running; wait for it to finish");
|
||||
}
|
||||
Ok(Self { file })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TransitionLock {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
libc::flock(self.file.as_raw_fd(), libc::LOCK_UN);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn adopt(root: &Path, name: &str) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
|
||||
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
|
||||
let running = governor::running_ids(®istry);
|
||||
let extras: Vec<String> = running
|
||||
.iter()
|
||||
.filter(|id| !target.contains(id.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !extras.is_empty() {
|
||||
bail!(
|
||||
"cannot adopt '{name}': registered model(s) outside the Scene are serving: {}",
|
||||
extras.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
let previous = State::load_checked(root)?;
|
||||
let mut adopted = BTreeMap::new();
|
||||
for id in &scene.models {
|
||||
let model = ®istry.models[id];
|
||||
if !governor::is_running(model) {
|
||||
bail!("cannot adopt '{name}': exact model '{id}' is not healthy");
|
||||
}
|
||||
let legacy = previous.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute"
|
||||
)
|
||||
})?;
|
||||
let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port)
|
||||
.with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?;
|
||||
adopted.insert(id.clone(), captured);
|
||||
}
|
||||
|
||||
let mut state = previous;
|
||||
state.procs = adopted;
|
||||
state.desired_scene = Some(name.to_string());
|
||||
state.active_scene = Some(name.to_string());
|
||||
state.last_known_good_scene = Some(name.to_string());
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("adopted exact running Scene '{name}' and captured process identities");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let prior = State::load_checked(root)?;
|
||||
let prior_active = prior.active_scene.clone();
|
||||
|
||||
if dry_run {
|
||||
return activate_once(root, name, true);
|
||||
}
|
||||
|
||||
match activate_once(root, name, false) {
|
||||
Ok(()) => {
|
||||
let mut state = State::load_checked(root)?;
|
||||
if prior_active.as_deref() != Some(name) {
|
||||
if let Some(previous) = prior_active {
|
||||
state.last_known_good_scene = Some(previous);
|
||||
}
|
||||
}
|
||||
state.desired_scene = Some(name.to_string());
|
||||
state.active_scene = Some(name.to_string());
|
||||
if state.last_known_good_scene.is_none() {
|
||||
state.last_known_good_scene = Some(name.to_string());
|
||||
}
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("Scene '{name}' is active and persisted as desired");
|
||||
Ok(())
|
||||
}
|
||||
Err(target_error) => {
|
||||
let target_message = format!("activating '{name}' failed: {target_error:#}");
|
||||
eprintln!("{target_message}");
|
||||
let transition_started =
|
||||
State::load_checked(root)?.transition_scene.as_deref() == Some(name);
|
||||
let cleanup_error = if transition_started {
|
||||
stop_all_owned(root).err()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let rollback = if transition_started && cleanup_error.is_none() {
|
||||
prior_active
|
||||
.as_deref()
|
||||
.filter(|previous| *previous != name)
|
||||
.map(|previous| (previous.to_string(), activate_once(root, previous, false)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.transition_scene = None;
|
||||
state.last_error = Some(target_message.clone());
|
||||
match rollback {
|
||||
Some((previous, Ok(()))) => {
|
||||
state.desired_scene = Some(previous.clone());
|
||||
state.active_scene = Some(previous.clone());
|
||||
if state.last_known_good_scene.is_none() {
|
||||
state.last_known_good_scene = Some(previous.clone());
|
||||
}
|
||||
state.save(root)?;
|
||||
eprintln!("rolled back to Scene '{previous}'");
|
||||
bail!("{target_message}; rolled back to '{previous}'")
|
||||
}
|
||||
Some((previous, Err(rollback_error))) => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
bail!(
|
||||
"{target_message}; rollback to '{previous}' also failed: {rollback_error:#}"
|
||||
)
|
||||
}
|
||||
None if !transition_started => {
|
||||
state.save(root)?;
|
||||
bail!("{target_message}")
|
||||
}
|
||||
None if cleanup_error.is_none() => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
bail!("{target_message}")
|
||||
}
|
||||
None => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
let cleanup_error = cleanup_error.expect("guarded by match condition");
|
||||
bail!("{target_message}; cleanup also failed: {cleanup_error:#}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_all_owned(root: &Path) -> Result<()> {
|
||||
let mut state = State::load_checked(root)?;
|
||||
let owned: Vec<(String, Proc)> = state
|
||||
.procs
|
||||
.iter()
|
||||
.filter(|(_, process)| process.owned_alive())
|
||||
.map(|(id, process)| (id.clone(), process.clone()))
|
||||
.collect();
|
||||
for (id, process) in owned {
|
||||
proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?;
|
||||
state.procs.remove(&id);
|
||||
state.save(root)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn resume(root: &Path) -> Result<()> {
|
||||
let state = State::load_checked(root)?;
|
||||
let desired = state
|
||||
.desired_scene
|
||||
.clone()
|
||||
.or_else(|| state.last_known_good_scene.clone())
|
||||
.context("no desired Scene is persisted; activate or adopt one first")?;
|
||||
let fallback = state.last_known_good_scene.clone();
|
||||
|
||||
match activate(root, &desired, false) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(desired_error) => {
|
||||
let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else {
|
||||
return Err(desired_error).context("resuming desired Scene");
|
||||
};
|
||||
eprintln!(
|
||||
"desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'"
|
||||
);
|
||||
activate(root, &fallback, false).with_context(|| {
|
||||
format!(
|
||||
"desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed"
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deactivate(root: &Path) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let mut state = State::load_checked(root)?;
|
||||
let ids: Vec<String> = state.procs.keys().cloned().collect();
|
||||
for id in ids {
|
||||
let process = state.procs[&id].clone();
|
||||
if process.owned_alive() {
|
||||
println!(" stopping {id} (pid {})", process.pid);
|
||||
proc::stop_owned(&process)?;
|
||||
}
|
||||
state.procs.remove(&id);
|
||||
state.save(root)?;
|
||||
}
|
||||
state.desired_scene = None;
|
||||
state.active_scene = None;
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("all Compute-managed models stopped; no Scene is desired");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Diff the target Scene against what is actually serving. Pure: it decides
|
||||
/// *what* would change, never *whether* the change is allowed.
|
||||
fn plan_transition(registry: &Registry, scene: &Scene) -> Plan {
|
||||
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
|
||||
let running = governor::running_ids(registry);
|
||||
let stop: Vec<String> = running
|
||||
.iter()
|
||||
.filter(|id| !target.contains(id.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut start: Vec<String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !governor::is_running(®istry.models[*id]))
|
||||
.cloned()
|
||||
.collect();
|
||||
let listed = scene
|
||||
.activation
|
||||
.as_ref()
|
||||
.and_then(|activation| activation.order.as_deref())
|
||||
.map(|order| order == "listed")
|
||||
.unwrap_or(false);
|
||||
if !listed {
|
||||
start.sort_by(|a, b| {
|
||||
registry.models[a]
|
||||
.footprint_gb
|
||||
.partial_cmp(®istry.models[b].footprint_gb)
|
||||
.unwrap()
|
||||
});
|
||||
}
|
||||
Plan {
|
||||
scene: scene.metadata.name.clone(),
|
||||
budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB),
|
||||
stop,
|
||||
start,
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything that must hold before the first process is signalled. Runs to
|
||||
/// completion with nothing mutated, so a rejection here leaves the currently
|
||||
/// active Scene exactly as it was.
|
||||
fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> {
|
||||
for id in &scene.models {
|
||||
if governor::is_running(®istry.models[id]) {
|
||||
let process = state.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first"
|
||||
)
|
||||
})?;
|
||||
if !process.owned_alive() {
|
||||
bail!(
|
||||
"ownership record for serving model '{id}' is stale; adopt the active Scene first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in &plan.stop {
|
||||
let process = state.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching"
|
||||
)
|
||||
})?;
|
||||
if !process.owned_alive() {
|
||||
bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid");
|
||||
}
|
||||
}
|
||||
for id in &plan.start {
|
||||
let model = ®istry.models[id];
|
||||
if let Some(port) = model.serve.port {
|
||||
if governor::port_open(port) && !governor::is_running(model) {
|
||||
bail!("port {port} is occupied by a different model; refusing to start '{id}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `scene activate --dry-run` as data rather than as printed lines: the same
|
||||
/// validation, the same plan, nothing written. Callers that need the plan
|
||||
/// programmatically use this instead of scraping stdout.
|
||||
pub fn plan(root: &Path, name: &str) -> Result<Plan> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
let plan = plan_transition(®istry, &scene);
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.procs.retain(|_, process| process.owned_alive());
|
||||
preflight(®istry, &scene, &plan, &state)?;
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> {
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
let plan = plan_transition(®istry, &scene);
|
||||
|
||||
println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb);
|
||||
println!(" stop : {}", format_ids(&plan.stop));
|
||||
println!(" start: {}", format_ids(&plan.start));
|
||||
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.procs.retain(|_, process| process.owned_alive());
|
||||
preflight(®istry, &scene, &plan, &state)?;
|
||||
|
||||
if dry_run {
|
||||
println!(" dry run: validation passed; nothing changed");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
state.transition_scene = Some(name.to_string());
|
||||
state.save(root)?;
|
||||
|
||||
for id in &plan.stop {
|
||||
let process = state.procs[id].clone();
|
||||
println!(" stopping {id} (pid {})", process.pid);
|
||||
proc::stop_owned(&process)?;
|
||||
state.procs.remove(id);
|
||||
state.save(root)?;
|
||||
}
|
||||
|
||||
let mut committed: f64 = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| governor::is_running(®istry.models[*id]))
|
||||
.map(|id| registry.models[id].footprint_gb)
|
||||
.sum();
|
||||
let wait_healthy = scene
|
||||
.activation
|
||||
.as_ref()
|
||||
.and_then(|activation| activation.wait_healthy)
|
||||
.unwrap_or(true);
|
||||
|
||||
for id in &plan.start {
|
||||
let model: &Model = ®istry.models[id];
|
||||
if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb) {
|
||||
bail!(
|
||||
"'{id}' would exceed the {:.0} GB Scene budget with the safety margin",
|
||||
plan.budget_gb
|
||||
);
|
||||
}
|
||||
let pid = proc::spawn(root, id, model)?;
|
||||
state.seq += 1;
|
||||
let process = Proc::capture(pid, state.seq, model.serve.port)
|
||||
.with_context(|| format!("capturing ownership for newly started '{id}'"))?;
|
||||
state.procs.insert(id.clone(), process.clone());
|
||||
state.save(root)?;
|
||||
committed += model.footprint_gb;
|
||||
println!(" started {id} (pid {pid}); waiting for exact health");
|
||||
if wait_healthy && !governor::wait_healthy(model, health_timeout()) {
|
||||
let _ = proc::stop_owned(&process);
|
||||
state.procs.remove(id);
|
||||
state.save(root)?;
|
||||
bail!("'{id}' did not report its exact health marker before timeout");
|
||||
}
|
||||
}
|
||||
|
||||
let unhealthy: Vec<String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !governor::is_running(®istry.models[*id]))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !unhealthy.is_empty() {
|
||||
bail!(
|
||||
"Scene '{name}' is incomplete; exact health failed for: {}",
|
||||
unhealthy.join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> {
|
||||
let missing: Vec<&String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !registry.models.contains_key(*id))
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
bail!(
|
||||
"Scene '{}' references unknown model id(s): {}",
|
||||
scene.metadata.name,
|
||||
missing
|
||||
.iter()
|
||||
.map(|id| id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
}
|
||||
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
|
||||
let footprint: f64 = scene
|
||||
.models
|
||||
.iter()
|
||||
.map(|id| registry.models[id].footprint_gb)
|
||||
.sum();
|
||||
if footprint + governor::SAFETY_MARGIN_GB > budget {
|
||||
bail!(
|
||||
"Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget",
|
||||
scene.metadata.name,
|
||||
footprint + governor::SAFETY_MARGIN_GB,
|
||||
budget
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn health_timeout() -> Duration {
|
||||
Duration::from_secs(if cfg!(test) { 5 } else { 900 })
|
||||
}
|
||||
|
||||
fn format_ids(ids: &[String]) -> String {
|
||||
if ids.is_empty() {
|
||||
"(none)".to_string()
|
||||
} else {
|
||||
ids.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::TcpListener;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn fixture_root() -> (PathBuf, u16) {
|
||||
let unique = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}"));
|
||||
fs::create_dir_all(root.join("registry")).unwrap();
|
||||
fs::create_dir_all(root.join("scenes")).unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
drop(listener);
|
||||
let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let bad_port = bad_listener.local_addr().unwrap().port();
|
||||
drop(bad_listener);
|
||||
let registry = format!(
|
||||
"apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n"
|
||||
);
|
||||
fs::write(root.join("registry/models.yaml"), registry).unwrap();
|
||||
fs::write(
|
||||
root.join("scenes/test.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
(root, port)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn activation_persists_and_deactivation_stops_owned_process() {
|
||||
let (root, _port) = fixture_root();
|
||||
activate(&root, "test", false).unwrap();
|
||||
let state = State::load_checked(&root).unwrap();
|
||||
assert_eq!(state.desired_scene.as_deref(), Some("test"));
|
||||
assert_eq!(state.active_scene.as_deref(), Some("test"));
|
||||
assert!(state.procs["fake"].owned_alive());
|
||||
deactivate(&root).unwrap();
|
||||
assert!(State::load_checked(&root).unwrap().procs.is_empty());
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_serving_scene_requires_explicit_adoption() {
|
||||
let (root, port) = fixture_root();
|
||||
let registry = Registry::load(&root).unwrap();
|
||||
let model = ®istry.models["fake"];
|
||||
let pid = proc::spawn(&root, "fake", model).unwrap();
|
||||
assert!(governor::wait_healthy(model, Duration::from_secs(5)));
|
||||
let error = activate(&root, "test", true).unwrap_err().to_string();
|
||||
assert!(error.contains("adopt the active Scene"));
|
||||
let owned = Proc::capture(pid, 1, Some(port)).unwrap();
|
||||
proc::stop_owned(&owned).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_rejection_preserves_healthy_active_scene() {
|
||||
let (root, _port) = fixture_root();
|
||||
activate(&root, "test", false).unwrap();
|
||||
let before = State::load_checked(&root).unwrap();
|
||||
let pid = before.procs["fake"].pid;
|
||||
fs::write(
|
||||
root.join("scenes/invalid.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(activate(&root, "invalid", false).is_err());
|
||||
let after = State::load_checked(&root).unwrap();
|
||||
assert_eq!(after.active_scene.as_deref(), Some("test"));
|
||||
assert_eq!(after.desired_scene.as_deref(), Some("test"));
|
||||
assert_eq!(after.procs["fake"].pid, pid);
|
||||
assert!(after.procs["fake"].owned_alive());
|
||||
deactivate(&root).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_activation_cleans_up_partial_scene() {
|
||||
let (root, port) = fixture_root();
|
||||
fs::write(
|
||||
root.join("scenes/failing.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(activate(&root, "failing", false).is_err());
|
||||
let state = State::load_checked(&root).unwrap();
|
||||
assert!(state.procs.is_empty());
|
||||
assert!(state.active_scene.is_none());
|
||||
assert!(!governor::port_open(port));
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_state_load_rejects_malformed_yaml() {
|
||||
let (root, _port) = fixture_root();
|
||||
fs::create_dir_all(root.join(".kuda")).unwrap();
|
||||
fs::write(root.join(".kuda/state.yaml"), "desired_scene: [").unwrap();
|
||||
assert!(State::load_checked(&root).is_err());
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_write_replaces_complete_yaml_atomically() {
|
||||
let (root, _port) = fixture_root();
|
||||
let state = State {
|
||||
desired_scene: Some("test".to_string()),
|
||||
..State::default()
|
||||
};
|
||||
state.save(&root).unwrap();
|
||||
let loaded = State::load_checked(&root).unwrap();
|
||||
assert_eq!(loaded.desired_scene.as_deref(), Some("test"));
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
}
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
//! Lumbridge Compute — safe AI workload orchestration for accelerator nodes.
|
||||
|
||||
mod agent;
|
||||
mod config;
|
||||
mod eval;
|
||||
mod gateway;
|
||||
mod governor;
|
||||
mod lifecycle;
|
||||
mod mcp;
|
||||
mod mem;
|
||||
mod proc;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use config::{find_scene, load_scenes, Registry, Scene};
|
||||
use proc::State;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "lumbridge-compute",
|
||||
version,
|
||||
about = "Lumbridge Compute — safe AI workload orchestration for accelerator nodes."
|
||||
)]
|
||||
struct Cli {
|
||||
/// Root dir containing registry/ and scenes/ (default: $LUMBRIDGE_COMPUTE_ROOT, $KUDA_ROOT, or current dir)
|
||||
#[arg(long, global = true)]
|
||||
root: Option<PathBuf>,
|
||||
#[command(subcommand)]
|
||||
cmd: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Governor status: memory, budget, running set, headroom
|
||||
Status,
|
||||
/// Manage the model registry
|
||||
Model {
|
||||
#[command(subcommand)]
|
||||
cmd: ModelCmd,
|
||||
},
|
||||
/// Manage scenes (bundles of models)
|
||||
Scene {
|
||||
#[command(subcommand)]
|
||||
cmd: SceneCmd,
|
||||
},
|
||||
/// Run reproducible capability and serving-performance evaluations
|
||||
Eval {
|
||||
#[command(subcommand)]
|
||||
cmd: EvalCmd,
|
||||
},
|
||||
/// Run the memory watchdog in the foreground (kills the newest model before OOM-wedge)
|
||||
Watchdog {
|
||||
/// Kill the newest model if MemAvailable dips below this many GB
|
||||
#[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)]
|
||||
floor: f64,
|
||||
},
|
||||
/// Run the stable streaming gateway in the foreground
|
||||
Gateway {
|
||||
#[arg(long, default_value = "127.0.0.1:8011")]
|
||||
listen: String,
|
||||
#[arg(long, default_value = "127.0.0.1:8001")]
|
||||
upstream: String,
|
||||
},
|
||||
/// Serve the Governor to AI agents as MCP tools over stdio
|
||||
Mcp {
|
||||
/// Let an agent perform a real Scene transition, not just plan one.
|
||||
/// Off by default: the agent writes its own tool arguments, so the only
|
||||
/// meaningful gate on a production switch is one the operator sets here.
|
||||
#[arg(long)]
|
||||
allow_activate: bool,
|
||||
},
|
||||
/// Run the resident supervisor: resume desired Scene, gateway, and memory floor
|
||||
Agent {
|
||||
#[arg(long, default_value = "127.0.0.1:8011")]
|
||||
listen: String,
|
||||
#[arg(long, default_value = "127.0.0.1:8001")]
|
||||
upstream: String,
|
||||
#[arg(long, default_value_t = governor::WATCHDOG_FLOOR_GB)]
|
||||
floor: f64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum ModelCmd {
|
||||
/// List registered models and their live state
|
||||
Ls,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum SceneCmd {
|
||||
/// List scenes with total footprint
|
||||
Ls,
|
||||
/// Show a scene: models, footprints, and the Governor's admission verdict
|
||||
Show { name: String },
|
||||
/// Activate a scene: stop what's not in it, admit + start what is
|
||||
Activate {
|
||||
name: String,
|
||||
/// Print the plan without changing anything
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// Adopt an already-running exact Scene and bind legacy PIDs to process identities
|
||||
Adopt { name: String },
|
||||
/// Resume the persisted desired Scene, falling back to the previous known-good Scene
|
||||
Resume,
|
||||
/// Stop all Lumbridge Compute-managed models
|
||||
Deactivate,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum EvalCmd {
|
||||
/// List available evaluation suites
|
||||
Ls,
|
||||
/// Run a suite against an OpenAI-compatible endpoint
|
||||
Run {
|
||||
suite: String,
|
||||
#[arg(long, default_value = "http://127.0.0.1:8001/v1")]
|
||||
base_url: String,
|
||||
#[arg(long, default_value = "brain")]
|
||||
model: String,
|
||||
/// Override the suite's repetitions per case
|
||||
#[arg(long)]
|
||||
repeat: Option<u32>,
|
||||
},
|
||||
}
|
||||
|
||||
fn root_dir(cli: &Cli) -> PathBuf {
|
||||
cli.root
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
std::env::var("LUMBRIDGE_COMPUTE_ROOT")
|
||||
.ok()
|
||||
.map(PathBuf::from)
|
||||
})
|
||||
.or_else(|| std::env::var("KUDA_ROOT").ok().map(PathBuf::from))
|
||||
.unwrap_or_else(|| PathBuf::from("."))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let root = root_dir(&cli);
|
||||
match &cli.cmd {
|
||||
Cmd::Status => cmd_status(&root),
|
||||
Cmd::Model { cmd } => match cmd {
|
||||
ModelCmd::Ls => cmd_model_ls(&root),
|
||||
},
|
||||
Cmd::Scene { cmd } => match cmd {
|
||||
SceneCmd::Ls => cmd_scene_ls(&root),
|
||||
SceneCmd::Show { name } => cmd_scene_show(&root, name),
|
||||
SceneCmd::Activate { name, dry_run } => lifecycle::activate(&root, name, *dry_run),
|
||||
SceneCmd::Adopt { name } => lifecycle::adopt(&root, name),
|
||||
SceneCmd::Resume => lifecycle::resume(&root),
|
||||
SceneCmd::Deactivate => lifecycle::deactivate(&root),
|
||||
},
|
||||
Cmd::Eval { cmd } => match cmd {
|
||||
EvalCmd::Ls => eval::list(&root),
|
||||
EvalCmd::Run {
|
||||
suite,
|
||||
base_url,
|
||||
model,
|
||||
repeat,
|
||||
} => eval::run(&root, suite, base_url, model, *repeat),
|
||||
},
|
||||
Cmd::Watchdog { floor } => cmd_watchdog(&root, *floor),
|
||||
Cmd::Gateway { listen, upstream } => gateway::run(listen, upstream),
|
||||
Cmd::Mcp { allow_activate } => mcp::run(&root, *allow_activate),
|
||||
Cmd::Agent {
|
||||
listen,
|
||||
upstream,
|
||||
floor,
|
||||
} => agent::run(&root, listen, upstream, *floor),
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_status(root: &Path) -> Result<()> {
|
||||
let reg = Registry::load(root)?;
|
||||
let m = mem::read()?;
|
||||
let committed = governor::committed_gb(®);
|
||||
let running = governor::running_ids(®);
|
||||
let budget = governor::DEFAULT_BUDGET_GB;
|
||||
let headroom = (budget - committed - governor::SAFETY_MARGIN_GB).max(0.0);
|
||||
let managed = State::load_checked(root)?;
|
||||
|
||||
println!("Lumbridge Compute · governor");
|
||||
println!(
|
||||
" memory {:.1} GB total · {:.1} GB available",
|
||||
m.total_gb, m.available_gb
|
||||
);
|
||||
println!(
|
||||
" budget {:.0} GB (safety margin {:.0} · watchdog floor {:.0})",
|
||||
budget,
|
||||
governor::SAFETY_MARGIN_GB,
|
||||
governor::WATCHDOG_FLOOR_GB
|
||||
);
|
||||
println!(
|
||||
" committed {committed:.1} GB across {} model(s)",
|
||||
running.len()
|
||||
);
|
||||
println!(" headroom {headroom:.1} GB admittable\n");
|
||||
println!(
|
||||
" scene active={} desired={} fallback={}",
|
||||
managed.active_scene.as_deref().unwrap_or("-"),
|
||||
managed.desired_scene.as_deref().unwrap_or("-"),
|
||||
managed.last_known_good_scene.as_deref().unwrap_or("-")
|
||||
);
|
||||
if let Some(error) = &managed.last_error {
|
||||
println!(" last error {error}");
|
||||
}
|
||||
|
||||
if running.is_empty() {
|
||||
println!(" (no registered models currently serving)");
|
||||
} else {
|
||||
println!(" running:");
|
||||
for id in &running {
|
||||
let mdl = ®.models[id];
|
||||
let tag = if managed
|
||||
.procs
|
||||
.get(id)
|
||||
.is_some_and(|process| process.owned_alive())
|
||||
{
|
||||
"compute"
|
||||
} else {
|
||||
"ext "
|
||||
};
|
||||
println!(
|
||||
" ● [{tag}] {:<12} {:>5.0} GB :{:<5} {}",
|
||||
id,
|
||||
mdl.footprint_gb,
|
||||
port_str(mdl.serve.port),
|
||||
mdl.name
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_model_ls(root: &Path) -> Result<()> {
|
||||
let reg = Registry::load(root)?;
|
||||
println!(
|
||||
"{:<12} {:>6} {:<6} {:<6} MODEL",
|
||||
"ID", "GB", "STATE", "PORT"
|
||||
);
|
||||
for (id, m) in ®.models {
|
||||
println!(
|
||||
"{:<12} {:>6.0} {:<6} {:<6} {}",
|
||||
id,
|
||||
m.footprint_gb,
|
||||
if governor::is_running(m) {
|
||||
"up"
|
||||
} else {
|
||||
"down"
|
||||
},
|
||||
port_str(m.serve.port),
|
||||
m.name
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_scene_ls(root: &Path) -> Result<()> {
|
||||
let reg = Registry::load(root)?;
|
||||
let scenes = load_scenes(root)?;
|
||||
if scenes.is_empty() {
|
||||
println!("no scenes in {}/scenes", root.display());
|
||||
return Ok(());
|
||||
}
|
||||
println!("{:<12} {:>6} {:<30} MODELS", "SCENE", "GB", "DESCRIPTION");
|
||||
for s in &scenes {
|
||||
let total = scene_footprint(s, ®);
|
||||
println!(
|
||||
"{:<12} {:>6.0} {:<30} {}",
|
||||
s.metadata.name,
|
||||
total,
|
||||
truncate(&s.metadata.description, 30),
|
||||
s.models.join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_scene_show(root: &Path, name: &str) -> Result<()> {
|
||||
let reg = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
|
||||
|
||||
println!(
|
||||
"scene {} (v{})",
|
||||
scene.metadata.name, scene.metadata.version
|
||||
);
|
||||
if !scene.metadata.description.is_empty() {
|
||||
println!(" {}", scene.metadata.description);
|
||||
}
|
||||
println!(" budget {budget:.0} GB\n models:");
|
||||
|
||||
let mut total = 0.0;
|
||||
let mut missing = Vec::new();
|
||||
for id in &scene.models {
|
||||
match reg.models.get(id) {
|
||||
Some(m) => {
|
||||
total += m.footprint_gb;
|
||||
let state = if governor::is_running(m) {
|
||||
"up"
|
||||
} else {
|
||||
"down"
|
||||
};
|
||||
println!(
|
||||
" {:<12} {:>5.0} GB [{:<4}] {}",
|
||||
id, m.footprint_gb, state, m.name
|
||||
);
|
||||
}
|
||||
None => {
|
||||
missing.push(id.clone());
|
||||
println!(" {id:<12} ? [MISSING from registry]");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let needed = total + governor::SAFETY_MARGIN_GB;
|
||||
println!(
|
||||
"\n total footprint {total:.1} GB (+{:.0} safety = {needed:.1} GB)",
|
||||
governor::SAFETY_MARGIN_GB
|
||||
);
|
||||
let verdict = if !missing.is_empty() {
|
||||
format!("✗ {} model(s) missing from registry", missing.len())
|
||||
} else if needed <= budget {
|
||||
format!("✓ fits — {:.1} GB to spare", budget - needed)
|
||||
} else {
|
||||
format!("✗ exceeds budget by {:.1} GB", needed - budget)
|
||||
};
|
||||
println!(" admission {verdict}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_watchdog(root: &Path, floor: f64) -> Result<()> {
|
||||
println!(
|
||||
"Lumbridge Compute watchdog — killing the newest model if MemAvailable < {floor:.1} GB. Ctrl-C to stop."
|
||||
);
|
||||
loop {
|
||||
let m = mem::read()?;
|
||||
if m.available_gb < floor {
|
||||
let mut state = State::load_checked(root)?;
|
||||
match state.newest_alive() {
|
||||
Some((id, p)) => {
|
||||
eprintln!(
|
||||
"watchdog: MemAvailable {:.1} GB < floor {:.1} — killing newest '{id}' (pid {})",
|
||||
m.available_gb, floor, p.pid
|
||||
);
|
||||
proc::stop_owned(&p)?;
|
||||
state.procs.remove(&id);
|
||||
state.save(root)?;
|
||||
}
|
||||
None => eprintln!(
|
||||
"watchdog: MemAvailable {:.1} GB < floor {:.1} but no Lumbridge Compute-managed model to kill!",
|
||||
m.available_gb, floor
|
||||
),
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------
|
||||
|
||||
fn scene_footprint(s: &Scene, reg: &Registry) -> f64 {
|
||||
s.models
|
||||
.iter()
|
||||
.filter_map(|id| reg.models.get(id))
|
||||
.map(|m| m.footprint_gb)
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn port_str(p: Option<u16>) -> String {
|
||||
p.map(|p| p.to_string()).unwrap_or_else(|| "-".to_string())
|
||||
}
|
||||
|
||||
fn truncate(s: &str, n: usize) -> String {
|
||||
if s.chars().count() <= n {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!(
|
||||
"{}…",
|
||||
s.chars().take(n.saturating_sub(1)).collect::<String>()
|
||||
)
|
||||
}
|
||||
}
|
||||
+609
@@ -0,0 +1,609 @@
|
||||
//! MCP server: the Governor's surface, exposed to agents as tools.
|
||||
//!
|
||||
//! Agents already drive Compute by shelling out to the CLI and scraping the
|
||||
//! column-aligned output. That works until a column moves. Speaking MCP means
|
||||
//! the agent gets the same numbers the Governor reasons about, as data, with a
|
||||
//! schema attached — and it means the *shape* of what an agent may do becomes
|
||||
//! something this file decides rather than something `bash` decides.
|
||||
//!
|
||||
//! Two deliberate constraints shape everything below.
|
||||
//!
|
||||
//! **Read-first.** Every tool here except `scene_activate` is a pure read.
|
||||
//! Activating a Scene stops every registered model that is not in it, which on
|
||||
//! this box means taking down whatever is currently serving — a live voice
|
||||
//! pipeline included. There is no undo an agent can reach for: if the target
|
||||
//! Scene then fails to come up, recovery depends on rollback that may itself
|
||||
//! fail. That asymmetry is why `scene adopt`, `scene resume`, and
|
||||
//! `scene deactivate` are absent entirely; they are operator verbs whose
|
||||
//! correctness depends on knowing what the box was doing five minutes ago.
|
||||
//! `scene_activate` is exposed because planning a switch is genuinely the
|
||||
//! useful thing an agent wants, and it defaults to planning only.
|
||||
//!
|
||||
//! **The safety boundary is the operator's, not the agent's.** A `dry_run`
|
||||
//! argument defaulting to `true` documents intent but guards nothing: the agent
|
||||
//! writes the arguments, so it can write `false`. The only boundary an agent
|
||||
//! cannot cross is one set before it connects, so a real transition also
|
||||
//! requires `lumbridge-compute mcp --allow-activate`, chosen by the human who
|
||||
//! launched the server. Without that flag `dry_run: false` is refused, and the
|
||||
//! refusal says so rather than silently planning instead.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rmcp::handler::server::router::tool::ToolRouter;
|
||||
use rmcp::handler::server::wrapper::Parameters;
|
||||
use rmcp::model::{Implementation, ServerCapabilities, ServerInfo};
|
||||
use rmcp::{tool, tool_handler, tool_router, ErrorData, Json, ServerHandler, ServiceExt};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::os::fd::FromRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::config::{find_scene, load_scenes, Registry};
|
||||
use crate::proc::State;
|
||||
use crate::{eval, governor, lifecycle, mem};
|
||||
|
||||
pub fn run(root: &Path, allow_activate: bool) -> Result<()> {
|
||||
let transport_stdout = hand_over_stdout()?;
|
||||
// The banner has to go to stderr for the same reason everything else does;
|
||||
// it doubles as confirmation to the operator that the mutating tool is off.
|
||||
eprintln!(
|
||||
"Lumbridge Compute MCP server on stdio · root {} · scene activation {}",
|
||||
root.display(),
|
||||
if allow_activate {
|
||||
"ENABLED (--allow-activate)"
|
||||
} else {
|
||||
"disabled; plan only"
|
||||
}
|
||||
);
|
||||
|
||||
let server = ComputeMcp::new(root.to_path_buf(), allow_activate);
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.context("building the MCP runtime")?;
|
||||
runtime.block_on(async move {
|
||||
let transport = (
|
||||
tokio::io::stdin(),
|
||||
tokio::fs::File::from_std(transport_stdout),
|
||||
);
|
||||
let service = server
|
||||
.serve(transport)
|
||||
.await
|
||||
.context("negotiating the MCP stdio session")?;
|
||||
service.waiting().await.context("serving MCP over stdio")?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Hand the real stdout to the transport and point fd 1 at stderr.
|
||||
///
|
||||
/// On stdio transport fd 1 *is* the JSON-RPC framing, and this crate reports
|
||||
/// progress with `println!` throughout — `lifecycle::activate` narrates every
|
||||
/// stop and start. One such line interleaved into the framing desynchronises
|
||||
/// the client mid-transition, which is the worst possible moment to lose it.
|
||||
/// Rather than audit every print (and every future one), move the file
|
||||
/// descriptor: library output lands on stderr, where operators already read
|
||||
/// this server's logs, and the protocol gets a channel nothing else can write.
|
||||
fn hand_over_stdout() -> Result<std::fs::File> {
|
||||
let saved = unsafe { libc::dup(libc::STDOUT_FILENO) };
|
||||
if saved < 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.context("duplicating stdout for the MCP transport");
|
||||
}
|
||||
if unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) } < 0 {
|
||||
let error = std::io::Error::last_os_error();
|
||||
unsafe { libc::close(saved) };
|
||||
return Err(error).context("redirecting stdout to stderr");
|
||||
}
|
||||
// SAFETY: `saved` is a fresh descriptor from `dup` that nothing else owns.
|
||||
Ok(unsafe { std::fs::File::from_raw_fd(saved) })
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ComputeMcp {
|
||||
root: Arc<PathBuf>,
|
||||
allow_activate: bool,
|
||||
tool_router: ToolRouter<ComputeMcp>,
|
||||
}
|
||||
|
||||
#[tool_router]
|
||||
impl ComputeMcp {
|
||||
fn new(root: PathBuf, allow_activate: bool) -> Self {
|
||||
Self {
|
||||
root: Arc::new(root),
|
||||
allow_activate,
|
||||
tool_router: Self::tool_router(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Governor status for this node: unified-memory totals, the hard budget
|
||||
/// and its margins, how much is committed by models that are actually
|
||||
/// serving, how much is still admittable, the persisted Scene state, and
|
||||
/// the last transition or watchdog error. Read this before reasoning about
|
||||
/// whether anything else will fit.
|
||||
#[tool(
|
||||
name = "governor_status",
|
||||
annotations(
|
||||
title = "Governor status",
|
||||
read_only_hint = true,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
async fn governor_status(&self) -> Result<Json<StatusReport>, ErrorData> {
|
||||
let root = self.root.clone();
|
||||
offload(move || collect_status(&root)).await.map(Json)
|
||||
}
|
||||
|
||||
/// List every model in the node's registry with its declared worst-case
|
||||
/// footprint, serving port, and whether it is currently serving. Footprints
|
||||
/// are what admission control is decided against, not observed usage.
|
||||
#[tool(
|
||||
name = "model_list",
|
||||
annotations(
|
||||
title = "List registered models",
|
||||
read_only_hint = true,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
async fn model_list(&self) -> Result<Json<ModelList>, ErrorData> {
|
||||
let root = self.root.clone();
|
||||
offload(move || collect_models(&root)).await.map(Json)
|
||||
}
|
||||
|
||||
/// List the Scenes this node can activate, with each one's total footprint
|
||||
/// and member models. A Scene is a named set of models brought up as a
|
||||
/// single unit.
|
||||
#[tool(
|
||||
name = "scene_list",
|
||||
annotations(title = "List scenes", read_only_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn scene_list(&self) -> Result<Json<SceneList>, ErrorData> {
|
||||
let root = self.root.clone();
|
||||
offload(move || collect_scenes(&root)).await.map(Json)
|
||||
}
|
||||
|
||||
/// Show one Scene: its models, their footprints and live state, and the
|
||||
/// Governor's admission verdict — whether the Scene fits its budget once
|
||||
/// the safety margin is added, and by how much it fits or misses.
|
||||
#[tool(
|
||||
name = "scene_show",
|
||||
annotations(
|
||||
title = "Show a scene and its admission verdict",
|
||||
read_only_hint = true,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
async fn scene_show(
|
||||
&self,
|
||||
Parameters(params): Parameters<SceneNameParams>,
|
||||
) -> Result<Json<SceneDetail>, ErrorData> {
|
||||
let root = self.root.clone();
|
||||
offload(move || collect_scene(&root, ¶ms.name))
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
|
||||
/// List the evaluation suites available on this node, with their versions
|
||||
/// and case counts. Running a suite is deliberately not exposed: it drives
|
||||
/// real load against a serving model for an unbounded time.
|
||||
#[tool(
|
||||
name = "eval_list",
|
||||
annotations(
|
||||
title = "List eval suites",
|
||||
read_only_hint = true,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
async fn eval_list(&self) -> Result<Json<EvalList>, ErrorData> {
|
||||
let root = self.root.clone();
|
||||
offload(move || collect_evals(&root)).await.map(Json)
|
||||
}
|
||||
|
||||
/// Plan — or, if the operator allowed it, perform — a Scene transition.
|
||||
///
|
||||
/// With `dry_run` true (the default) this runs every check that guards a
|
||||
/// real transition and returns the models it would stop and start, changing
|
||||
/// nothing. With `dry_run` false it actually switches the node: models
|
||||
/// outside the Scene are stopped, which will interrupt anything they serve,
|
||||
/// and the call blocks until every Scene model reports healthy or the
|
||||
/// transition rolls back. A real transition additionally requires the
|
||||
/// server to have been started with `--allow-activate`; without it,
|
||||
/// `dry_run: false` is refused. Ask a human before setting it.
|
||||
#[tool(
|
||||
name = "scene_activate",
|
||||
annotations(
|
||||
title = "Activate a scene (plans by default)",
|
||||
read_only_hint = false,
|
||||
destructive_hint = true,
|
||||
idempotent_hint = false,
|
||||
open_world_hint = false
|
||||
)
|
||||
)]
|
||||
async fn scene_activate(
|
||||
&self,
|
||||
Parameters(params): Parameters<SceneActivateParams>,
|
||||
) -> Result<Json<ActivationReport>, ErrorData> {
|
||||
if !params.dry_run && !self.allow_activate {
|
||||
return Err(ErrorData::invalid_params(
|
||||
"this MCP server is running plan-only: a real Scene transition needs an \
|
||||
operator to restart it as `lumbridge-compute mcp --allow-activate`. \
|
||||
Re-run with dry_run: true to see the plan.",
|
||||
None,
|
||||
));
|
||||
}
|
||||
let root = self.root.clone();
|
||||
offload(move || activate_scene(&root, ¶ms.name, params.dry_run))
|
||||
.await
|
||||
.map(Json)
|
||||
}
|
||||
}
|
||||
|
||||
#[tool_handler(router = self.tool_router)]
|
||||
impl ServerHandler for ComputeMcp {
|
||||
fn get_info(&self) -> ServerInfo {
|
||||
ServerInfo::new(ServerCapabilities::builder().enable_tools().build())
|
||||
.with_server_info(Implementation::new(
|
||||
"lumbridge-compute",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
))
|
||||
.with_instructions(
|
||||
"Lumbridge Compute runs many AI models on one unified-memory box. There is a \
|
||||
single memory pool shared by CPU and GPU, so over-committing does not fail \
|
||||
gracefully — the machine thrashes and wedges before the OOM killer acts. The \
|
||||
Governor prevents that by admitting a model only if its declared footprint \
|
||||
plus a safety margin still fits a hard budget.\n\n\
|
||||
Start from `governor_status` for what is committed and what is admittable, \
|
||||
and `scene_show` for whether a named Scene would be admitted. Everything is \
|
||||
read-only except `scene_activate`, which plans by default. Activating a Scene \
|
||||
stops every registered model outside it, so treat it as a production change \
|
||||
and get a human's agreement before asking for a non-dry run.",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- tool parameters -------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct SceneNameParams {
|
||||
/// Scene name, as reported by `scene_list`.
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct SceneActivateParams {
|
||||
/// Scene name, as reported by `scene_list`.
|
||||
name: String,
|
||||
/// Validate and report the plan without changing anything. Defaults to
|
||||
/// true; set it to false only with a human's explicit agreement.
|
||||
#[serde(default = "yes")]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
fn yes() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ---- tool results ----------------------------------------------------------
|
||||
//
|
||||
// Lists are wrapped in objects rather than returned bare: MCP structured
|
||||
// content must be a JSON object, and a named field leaves room to report
|
||||
// alongside the list later without breaking a client's schema.
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct StatusReport {
|
||||
memory_total_gb: f64,
|
||||
memory_available_gb: f64,
|
||||
budget_gb: f64,
|
||||
safety_margin_gb: f64,
|
||||
watchdog_floor_gb: f64,
|
||||
/// Sum of the footprints of every registered model currently serving.
|
||||
committed_gb: f64,
|
||||
/// What a new model could still claim without breaching the safety margin.
|
||||
headroom_gb: f64,
|
||||
/// Scene whose exact model health was last verified.
|
||||
active_scene: Option<String>,
|
||||
/// Scene the node should return to after a restart.
|
||||
desired_scene: Option<String>,
|
||||
/// Previous proven Scene, used if the desired one cannot resume.
|
||||
last_known_good_scene: Option<String>,
|
||||
last_error: Option<String>,
|
||||
running: Vec<RunningModel>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct RunningModel {
|
||||
id: String,
|
||||
name: String,
|
||||
footprint_gb: f64,
|
||||
port: Option<u16>,
|
||||
/// True when Compute owns this process identity and may signal it. False
|
||||
/// means the model is serving but was started outside Compute, so a Scene
|
||||
/// transition will refuse to touch it until it is adopted.
|
||||
compute_owned: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct ModelList {
|
||||
models: Vec<ModelEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct ModelEntry {
|
||||
id: String,
|
||||
name: String,
|
||||
/// Declared worst-case unified memory once serving, not observed usage.
|
||||
footprint_gb: f64,
|
||||
port: Option<u16>,
|
||||
running: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct SceneList {
|
||||
scenes: Vec<SceneEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct SceneEntry {
|
||||
name: String,
|
||||
version: u32,
|
||||
description: String,
|
||||
/// Total footprint of the Scene's models that exist in the registry.
|
||||
footprint_gb: f64,
|
||||
models: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct SceneDetail {
|
||||
name: String,
|
||||
version: u32,
|
||||
description: String,
|
||||
budget_gb: f64,
|
||||
models: Vec<SceneModel>,
|
||||
/// Model ids the Scene references that the registry does not define.
|
||||
missing_models: Vec<String>,
|
||||
footprint_gb: f64,
|
||||
/// Footprint plus the Governor's safety margin — the figure compared
|
||||
/// against the budget.
|
||||
required_gb: f64,
|
||||
/// The Governor's verdict: would this Scene be admitted?
|
||||
admits: bool,
|
||||
verdict: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct SceneModel {
|
||||
id: String,
|
||||
/// Absent when the id is missing from the registry.
|
||||
name: Option<String>,
|
||||
footprint_gb: Option<f64>,
|
||||
running: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct EvalList {
|
||||
suites: Vec<EvalSuite>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct EvalSuite {
|
||||
name: String,
|
||||
version: u32,
|
||||
cases: usize,
|
||||
description: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, JsonSchema)]
|
||||
struct ActivationReport {
|
||||
scene: String,
|
||||
budget_gb: f64,
|
||||
/// Serving models outside the Scene, which the transition stops first.
|
||||
stop: Vec<String>,
|
||||
/// Scene models not yet serving, in the order they would be admitted.
|
||||
start: Vec<String>,
|
||||
/// False when this was a plan and the node was left untouched.
|
||||
applied: bool,
|
||||
}
|
||||
|
||||
// ---- collectors ------------------------------------------------------------
|
||||
//
|
||||
// These compose the same primitives the CLI prints from — `governor`, `mem`,
|
||||
// `config`, `proc::State` — rather than parsing the CLI's output. The numbers
|
||||
// an agent sees are therefore the numbers admission control uses, by
|
||||
// construction.
|
||||
|
||||
fn collect_status(root: &Path) -> Result<StatusReport> {
|
||||
let registry = Registry::load(root)?;
|
||||
let memory = mem::read()?;
|
||||
let committed = governor::committed_gb(®istry);
|
||||
let managed = State::load_checked(root)?;
|
||||
let budget = governor::DEFAULT_BUDGET_GB;
|
||||
|
||||
let running = governor::running_ids(®istry)
|
||||
.into_iter()
|
||||
.map(|id| {
|
||||
let model = ®istry.models[&id];
|
||||
RunningModel {
|
||||
name: model.name.clone(),
|
||||
footprint_gb: model.footprint_gb,
|
||||
port: model.serve.port,
|
||||
compute_owned: managed
|
||||
.procs
|
||||
.get(&id)
|
||||
.is_some_and(|process| process.owned_alive()),
|
||||
id,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(StatusReport {
|
||||
memory_total_gb: memory.total_gb,
|
||||
memory_available_gb: memory.available_gb,
|
||||
budget_gb: budget,
|
||||
safety_margin_gb: governor::SAFETY_MARGIN_GB,
|
||||
watchdog_floor_gb: governor::WATCHDOG_FLOOR_GB,
|
||||
committed_gb: committed,
|
||||
headroom_gb: (budget - committed - governor::SAFETY_MARGIN_GB).max(0.0),
|
||||
active_scene: managed.active_scene,
|
||||
desired_scene: managed.desired_scene,
|
||||
last_known_good_scene: managed.last_known_good_scene,
|
||||
last_error: managed.last_error,
|
||||
running,
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_models(root: &Path) -> Result<ModelList> {
|
||||
let registry = Registry::load(root)?;
|
||||
Ok(ModelList {
|
||||
models: registry
|
||||
.models
|
||||
.iter()
|
||||
.map(|(id, model)| ModelEntry {
|
||||
id: id.clone(),
|
||||
name: model.name.clone(),
|
||||
footprint_gb: model.footprint_gb,
|
||||
port: model.serve.port,
|
||||
running: governor::is_running(model),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_scenes(root: &Path) -> Result<SceneList> {
|
||||
let registry = Registry::load(root)?;
|
||||
Ok(SceneList {
|
||||
scenes: load_scenes(root)?
|
||||
.into_iter()
|
||||
.map(|scene| SceneEntry {
|
||||
name: scene.metadata.name,
|
||||
version: scene.metadata.version,
|
||||
description: scene.metadata.description,
|
||||
footprint_gb: scene
|
||||
.models
|
||||
.iter()
|
||||
.filter_map(|id| registry.models.get(id))
|
||||
.map(|model| model.footprint_gb)
|
||||
.sum(),
|
||||
models: scene.models,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_scene(root: &Path, name: &str) -> Result<SceneDetail> {
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
|
||||
|
||||
let mut footprint = 0.0;
|
||||
let mut missing = Vec::new();
|
||||
let mut models = Vec::new();
|
||||
for id in &scene.models {
|
||||
match registry.models.get(id) {
|
||||
Some(model) => {
|
||||
footprint += model.footprint_gb;
|
||||
models.push(SceneModel {
|
||||
id: id.clone(),
|
||||
name: Some(model.name.clone()),
|
||||
footprint_gb: Some(model.footprint_gb),
|
||||
running: governor::is_running(model),
|
||||
});
|
||||
}
|
||||
None => {
|
||||
missing.push(id.clone());
|
||||
models.push(SceneModel {
|
||||
id: id.clone(),
|
||||
name: None,
|
||||
footprint_gb: None,
|
||||
running: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same three-way verdict the CLI prints: a missing id is fatal regardless
|
||||
// of arithmetic, because the Scene cannot be resolved at all.
|
||||
let required = footprint + governor::SAFETY_MARGIN_GB;
|
||||
let (admits, verdict) = if !missing.is_empty() {
|
||||
(
|
||||
false,
|
||||
format!("{} model(s) missing from the registry", missing.len()),
|
||||
)
|
||||
} else if required <= budget {
|
||||
(
|
||||
true,
|
||||
format!("fits with {:.1} GB to spare", budget - required),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
false,
|
||||
format!("exceeds the budget by {:.1} GB", required - budget),
|
||||
)
|
||||
};
|
||||
|
||||
Ok(SceneDetail {
|
||||
name: scene.metadata.name,
|
||||
version: scene.metadata.version,
|
||||
description: scene.metadata.description,
|
||||
budget_gb: budget,
|
||||
models,
|
||||
missing_models: missing,
|
||||
footprint_gb: footprint,
|
||||
required_gb: required,
|
||||
admits,
|
||||
verdict,
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_evals(root: &Path) -> Result<EvalList> {
|
||||
Ok(EvalList {
|
||||
suites: eval::catalogue(root)?
|
||||
.into_iter()
|
||||
.map(|suite| EvalSuite {
|
||||
name: suite.name,
|
||||
version: suite.version,
|
||||
cases: suite.cases,
|
||||
description: suite.description,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn activate_scene(root: &Path, name: &str, dry_run: bool) -> Result<ActivationReport> {
|
||||
// Plan first either way. On a real transition this is the plan we report;
|
||||
// `activate` re-derives and re-validates its own under the transition lock,
|
||||
// so the report describes intent and the lock still owns the truth.
|
||||
let plan = lifecycle::plan(root, name)?;
|
||||
if !dry_run {
|
||||
lifecycle::activate(root, name, false)?;
|
||||
}
|
||||
Ok(ActivationReport {
|
||||
scene: plan.scene,
|
||||
budget_gb: plan.budget_gb,
|
||||
stop: plan.stop,
|
||||
start: plan.start,
|
||||
applied: !dry_run,
|
||||
})
|
||||
}
|
||||
|
||||
/// Run a synchronous Governor call off the transport's thread.
|
||||
///
|
||||
/// Every read here touches `/proc` and probes serving ports with blocking
|
||||
/// socket timeouts, and a real transition can sit for minutes waiting on model
|
||||
/// health. Doing that inline would stall the JSON-RPC reader for the duration,
|
||||
/// so the synchronous core stays on the blocking pool where it belongs.
|
||||
async fn offload<T, F>(work: F) -> Result<T, ErrorData>
|
||||
where
|
||||
F: FnOnce() -> Result<T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
match tokio::task::spawn_blocking(work).await {
|
||||
Ok(Ok(value)) => Ok(value),
|
||||
// `{:#}` keeps anyhow's context chain, which is where the actionable
|
||||
// half of a Compute error lives ("adopt the active Scene first").
|
||||
Ok(Err(error)) => Err(ErrorData::internal_error(format!("{error:#}"), None)),
|
||||
Err(join) => Err(ErrorData::internal_error(
|
||||
format!("Compute worker task failed: {join}"),
|
||||
None,
|
||||
)),
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
//! Unified-memory sensing. On a shared-memory box there's one pool, so `MemAvailable`
|
||||
//! from /proc/meminfo is the single source of truth the Governor guards.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs;
|
||||
|
||||
pub struct MemInfo {
|
||||
pub total_gb: f64,
|
||||
pub available_gb: f64,
|
||||
}
|
||||
|
||||
pub fn read() -> Result<MemInfo> {
|
||||
let s = fs::read_to_string("/proc/meminfo").context("reading /proc/meminfo")?;
|
||||
let mut total = 0.0;
|
||||
let mut available = 0.0;
|
||||
for line in s.lines() {
|
||||
if let Some(v) = line.strip_prefix("MemTotal:") {
|
||||
total = kb_to_gb(v);
|
||||
} else if let Some(v) = line.strip_prefix("MemAvailable:") {
|
||||
available = kb_to_gb(v);
|
||||
}
|
||||
}
|
||||
Ok(MemInfo {
|
||||
total_gb: total,
|
||||
available_gb: available,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse a " 123456 kB" meminfo value into GB.
|
||||
fn kb_to_gb(v: &str) -> f64 {
|
||||
v.split_whitespace()
|
||||
.next()
|
||||
.and_then(|n| n.parse::<f64>().ok())
|
||||
.unwrap_or(0.0)
|
||||
/ 1024.0
|
||||
/ 1024.0
|
||||
}
|
||||
+363
@@ -0,0 +1,363 @@
|
||||
//! Process lifecycle: spawn a model in its own process group, track it in a state
|
||||
//! file, and stop it (SIGTERM → SIGKILL to the whole group). The Governor decides
|
||||
//! *whether* to start; this module *how*.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{self, Command, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::config::Model;
|
||||
|
||||
/// One Lumbridge Compute-managed process. `seq` is a monotonic launch counter so the watchdog
|
||||
/// can always find the *newest* model to sacrifice first.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Proc {
|
||||
pub pid: i32,
|
||||
pub seq: u64,
|
||||
pub port: Option<u16>,
|
||||
/// Linux boot id plus process start ticks bind ownership to one process
|
||||
/// incarnation, preventing a reused PID from ever being signalled.
|
||||
#[serde(default)]
|
||||
pub boot_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub start_time_ticks: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct State {
|
||||
#[serde(default)]
|
||||
pub seq: u64,
|
||||
#[serde(default)]
|
||||
pub procs: BTreeMap<String, Proc>,
|
||||
#[serde(default)]
|
||||
pub desired_scene: Option<String>,
|
||||
#[serde(default)]
|
||||
pub active_scene: Option<String>,
|
||||
/// Previous proven scene used if the desired scene cannot resume.
|
||||
#[serde(default)]
|
||||
pub last_known_good_scene: Option<String>,
|
||||
#[serde(default)]
|
||||
pub transition_scene: Option<String>,
|
||||
#[serde(default)]
|
||||
pub last_error: Option<String>,
|
||||
}
|
||||
|
||||
fn state_dir(root: &Path) -> PathBuf {
|
||||
root.join(".kuda")
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub fn load_checked(root: &Path) -> Result<State> {
|
||||
let p = state_dir(root).join("state.yaml");
|
||||
match fs::read_to_string(&p) {
|
||||
Ok(s) => serde_yaml::from_str(&s)
|
||||
.with_context(|| format!("parsing persisted state {}", p.display())),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(State::default()),
|
||||
Err(error) => Err(error).with_context(|| format!("reading {}", p.display())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save(&self, root: &Path) -> Result<()> {
|
||||
let dir = state_dir(root);
|
||||
fs::create_dir_all(&dir)?;
|
||||
let p = dir.join("state.yaml");
|
||||
let tmp = dir.join(format!(".state.yaml.{}.tmp", process::id()));
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&tmp)
|
||||
.with_context(|| format!("creating {}", tmp.display()))?;
|
||||
file.write_all(serde_yaml::to_string(self)?.as_bytes())?;
|
||||
file.sync_all()?;
|
||||
fs::rename(&tmp, &p).with_context(|| format!("atomically replacing {}", p.display()))?;
|
||||
File::open(&dir)
|
||||
.with_context(|| format!("opening state directory {}", dir.display()))?
|
||||
.sync_all()
|
||||
.with_context(|| format!("syncing state directory {}", dir.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Newest still-alive managed model id, if any.
|
||||
pub fn newest_alive(&self) -> Option<(String, Proc)> {
|
||||
self.procs
|
||||
.iter()
|
||||
.filter(|(_, p)| p.owned_alive())
|
||||
.max_by_key(|(_, p)| p.seq)
|
||||
.map(|(id, p)| (id.clone(), p.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ProcessIdentity {
|
||||
boot_id: String,
|
||||
start_time_ticks: u64,
|
||||
process_group_id: i32,
|
||||
state: char,
|
||||
}
|
||||
|
||||
impl Proc {
|
||||
/// Capture the identity of a process that Lumbridge Compute just spawned or
|
||||
/// that an operator explicitly adopted from legacy state.
|
||||
pub fn capture(pid: i32, seq: u64, port: Option<u16>) -> Result<Proc> {
|
||||
let identity = process_identity(pid)?;
|
||||
if identity.process_group_id != pid {
|
||||
bail!(
|
||||
"refusing process {pid}: process group {} does not equal pid",
|
||||
identity.process_group_id
|
||||
);
|
||||
}
|
||||
Ok(Proc {
|
||||
pid,
|
||||
seq,
|
||||
port,
|
||||
boot_id: Some(identity.boot_id),
|
||||
start_time_ticks: Some(identity.start_time_ticks),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn owned_alive(&self) -> bool {
|
||||
self.validate_owned().is_ok()
|
||||
}
|
||||
|
||||
fn validate_owned(&self) -> Result<()> {
|
||||
let expected_boot = self
|
||||
.boot_id
|
||||
.as_deref()
|
||||
.context("legacy process record has no boot identity; adopt the active scene first")?;
|
||||
let expected_start = self
|
||||
.start_time_ticks
|
||||
.context("legacy process record has no start identity; adopt the active scene first")?;
|
||||
let current = process_identity(self.pid)?;
|
||||
if current.boot_id != expected_boot || current.start_time_ticks != expected_start {
|
||||
bail!(
|
||||
"pid {} no longer identifies the process Compute started",
|
||||
self.pid
|
||||
);
|
||||
}
|
||||
if current.state == 'Z' {
|
||||
bail!("pid {} is a zombie awaiting reap", self.pid);
|
||||
}
|
||||
if current.process_group_id != self.pid {
|
||||
bail!(
|
||||
"pid {} now belongs to process group {}; refusing group signal",
|
||||
self.pid,
|
||||
current.process_group_id
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn process_identity(pid: i32) -> Result<ProcessIdentity> {
|
||||
let boot_id = fs::read_to_string("/proc/sys/kernel/random/boot_id")
|
||||
.context("reading Linux boot id")?
|
||||
.trim()
|
||||
.to_string();
|
||||
let stat_path = format!("/proc/{pid}/stat");
|
||||
let stat = fs::read_to_string(&stat_path)
|
||||
.with_context(|| format!("reading process identity {stat_path}"))?;
|
||||
let (state, process_group_id, start_time_ticks) = parse_stat_identity(&stat)?;
|
||||
Ok(ProcessIdentity {
|
||||
boot_id,
|
||||
start_time_ticks,
|
||||
process_group_id,
|
||||
state,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_stat_identity(stat: &str) -> Result<(char, i32, u64)> {
|
||||
// `comm` is parenthesized and may contain spaces, so locate its final `)`
|
||||
// before indexing the fields that follow it. The remainder starts at field 3.
|
||||
let close = stat
|
||||
.rfind(')')
|
||||
.context("malformed /proc stat: missing process name terminator")?;
|
||||
let fields: Vec<&str> = stat[close + 1..].split_whitespace().collect();
|
||||
let state = fields
|
||||
.first()
|
||||
.and_then(|value| value.chars().next())
|
||||
.context("malformed /proc stat: missing process state")?;
|
||||
let process_group_id = fields
|
||||
.get(2)
|
||||
.context("malformed /proc stat: missing process group")?
|
||||
.parse::<i32>()
|
||||
.context("parsing process group id")?;
|
||||
let start_time_ticks = fields
|
||||
.get(19)
|
||||
.context("malformed /proc stat: missing start time")?
|
||||
.parse::<u64>()
|
||||
.context("parsing process start time")?;
|
||||
Ok((state, process_group_id, start_time_ticks))
|
||||
}
|
||||
|
||||
/// Expand a leading `~/` to `$HOME`.
|
||||
fn expand(s: &str) -> String {
|
||||
if let Some(rest) = s.strip_prefix("~/") {
|
||||
if let Ok(home) = std::env::var("HOME") {
|
||||
return format!("{home}/{rest}");
|
||||
}
|
||||
}
|
||||
s.to_string()
|
||||
}
|
||||
|
||||
/// Build the launch argv for a model: an explicit `command` wins; otherwise a
|
||||
/// `kind`-based builder. argv[0] is the program.
|
||||
pub fn build_argv(m: &Model) -> Result<Vec<String>> {
|
||||
if !m.serve.command.is_empty() {
|
||||
return Ok(m.serve.command.iter().map(|s| expand(s)).collect());
|
||||
}
|
||||
match m.serve.kind.as_str() {
|
||||
"vllm" => {
|
||||
let executable = m
|
||||
.serve
|
||||
.executable
|
||||
.as_deref()
|
||||
.map(expand)
|
||||
.unwrap_or_else(|| "vllm".to_string());
|
||||
let mut a = vec![executable, "serve".to_string()];
|
||||
if let Some(w) = &m.serve.weights {
|
||||
a.push(expand(w));
|
||||
}
|
||||
if !m.serve.served_name.is_empty() {
|
||||
a.push("--served-model-name".into());
|
||||
a.extend(m.serve.served_name.iter().cloned());
|
||||
}
|
||||
a.push("--host".into());
|
||||
a.push("0.0.0.0".into());
|
||||
if let Some(p) = m.serve.port {
|
||||
a.push("--port".into());
|
||||
a.push(p.to_string());
|
||||
}
|
||||
for (k, v) in &m.serve.args {
|
||||
render_arg(&mut a, k, v);
|
||||
}
|
||||
Ok(a)
|
||||
}
|
||||
other => bail!(
|
||||
"model '{}' has no explicit `command` and kind '{}' has no builder yet — \
|
||||
add a `command: [...]` to the registry entry",
|
||||
m.name,
|
||||
other
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn render_arg(out: &mut Vec<String>, k: &str, v: &serde_yaml::Value) {
|
||||
use serde_yaml::Value;
|
||||
let flag = format!("--{k}");
|
||||
match v {
|
||||
Value::Bool(true) => out.push(flag),
|
||||
Value::Bool(false) => {}
|
||||
Value::String(s) => {
|
||||
out.push(flag);
|
||||
out.push(s.clone());
|
||||
}
|
||||
Value::Number(n) => {
|
||||
out.push(flag);
|
||||
out.push(n.to_string());
|
||||
}
|
||||
_ => {
|
||||
out.push(flag);
|
||||
out.push(
|
||||
serde_yaml::to_string(v)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn `id`'s model detached in its own process group, logs → compatibility path `.kuda/logs/<id>.log`.
|
||||
/// Returns the child pid (also its process-group id).
|
||||
pub fn spawn(root: &Path, id: &str, m: &Model) -> Result<i32> {
|
||||
let argv = build_argv(m)?;
|
||||
let logdir = state_dir(root).join("logs");
|
||||
fs::create_dir_all(&logdir)?;
|
||||
let log = File::create(logdir.join(format!("{id}.log")))?;
|
||||
let errlog = log.try_clone()?;
|
||||
|
||||
let mut cmd = Command::new(&argv[0]);
|
||||
cmd.args(&argv[1..])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(log))
|
||||
.stderr(Stdio::from(errlog))
|
||||
.process_group(0); // own group → clean group-kill, immune to CLI's signals
|
||||
for (k, v) in &m.serve.env {
|
||||
cmd.env(k, expand(v));
|
||||
}
|
||||
let mut child = cmd
|
||||
.spawn()
|
||||
.with_context(|| format!("spawning '{id}': {}", argv.join(" ")))?;
|
||||
let pid = child.id() as i32;
|
||||
// A resident agent may outlive many model processes. Reap each child when
|
||||
// it exits so failed runtimes cannot accumulate as zombies under the agent.
|
||||
std::thread::spawn(move || {
|
||||
let _ = child.wait();
|
||||
});
|
||||
Ok(pid)
|
||||
}
|
||||
|
||||
/// Stop only the exact process group captured in this ownership record.
|
||||
pub fn stop_owned(proc: &Proc) -> Result<()> {
|
||||
if !proc.owned_alive() {
|
||||
return Ok(());
|
||||
}
|
||||
proc.validate_owned()?;
|
||||
let term = unsafe { libc::kill(-proc.pid, libc::SIGTERM) };
|
||||
if term != 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.with_context(|| format!("sending SIGTERM to owned process group {}", proc.pid));
|
||||
}
|
||||
for _ in 0..60 {
|
||||
if !proc.owned_alive() {
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
proc.validate_owned()?;
|
||||
let kill = unsafe { libc::kill(-proc.pid, libc::SIGKILL) };
|
||||
if kill != 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.with_context(|| format!("sending SIGKILL to owned process group {}", proc.pid));
|
||||
}
|
||||
for _ in 0..20 {
|
||||
if !proc.owned_alive() {
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
bail!("owned process group {} survived SIGKILL", proc.pid)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn proc_stat_parser_handles_spaces_in_process_name() {
|
||||
let stat = "123 (worker process) S 1 123 123 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 4567";
|
||||
let (state, pgrp, start) = parse_stat_identity(stat).unwrap();
|
||||
assert_eq!(state, 'S');
|
||||
assert_eq!(pgrp, 123);
|
||||
assert_eq!(start, 4567);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_records_are_never_treated_as_owned() {
|
||||
let proc = Proc {
|
||||
pid: std::process::id() as i32,
|
||||
seq: 1,
|
||||
port: None,
|
||||
boot_id: None,
|
||||
start_time_ticks: None,
|
||||
};
|
||||
assert!(!proc.owned_alive());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
[Unit]
|
||||
Description=Lumbridge Compute resident Scene supervisor and model gateway
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
# A dedicated unprivileged service account. Create it with:
|
||||
# sudo useradd --system --home-dir /var/lib/lumbridge --create-home lumbridge
|
||||
User=lumbridge
|
||||
Environment=HOME=/var/lib/lumbridge
|
||||
Environment=LUMBRIDGE_COMPUTE_ROOT=/var/lib/lumbridge/compute
|
||||
# The gateway binds loopback by design. It has no authentication and no TLS, so
|
||||
# listening on 0.0.0.0 would publish every model on this node to the whole
|
||||
# network. To reach it from elsewhere put a reverse proxy or an overlay network
|
||||
# in front of it, rather than widening this bind.
|
||||
ExecStart=/usr/local/bin/lumbridge-compute agent --listen 127.0.0.1:8011 --upstream 127.0.0.1:8001 --floor 3
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
# Model process groups are lifecycle-owned by Compute and must survive an agent
|
||||
# binary restart; the agent will revalidate their boot/start identities on resume.
|
||||
KillMode=process
|
||||
TimeoutStopSec=15
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue
Block a user