This commit is contained in:
Karti Tripathi
@@ -0,0 +1,915 @@
|
||||
//! Transactional Scene lifecycle and persisted desired-state recovery.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::config::{find_scene, Model, Registry, Scene};
|
||||
use crate::governor;
|
||||
use crate::mem;
|
||||
use crate::proc::{self, Proc, State};
|
||||
|
||||
/// What activating a Scene would do, decided before anything is mutated.
|
||||
///
|
||||
/// `activate` renders this for the CLI and the MCP server returns it verbatim,
|
||||
/// so a plan an operator reads and a plan an agent reads can never drift apart.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Plan {
|
||||
pub scene: String,
|
||||
pub budget_gb: f64,
|
||||
/// Registered models serving outside the target Scene; stopped first.
|
||||
pub stop: Vec<String>,
|
||||
/// Scene models not yet serving, in the order they would be admitted.
|
||||
pub start: Vec<String>,
|
||||
}
|
||||
|
||||
pub struct TransitionLock {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl TransitionLock {
|
||||
pub fn acquire(root: &Path) -> Result<Self> {
|
||||
let dir = root.join(".compute");
|
||||
fs::create_dir_all(&dir)?;
|
||||
let path = dir.join("transition.lock");
|
||||
let file = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.with_context(|| format!("opening transition lock {}", path.display()))?;
|
||||
let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if result != 0 {
|
||||
return Err(std::io::Error::last_os_error())
|
||||
.context("another Scene transition is already running; wait for it to finish");
|
||||
}
|
||||
Ok(Self { file })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TransitionLock {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
libc::flock(self.file.as_raw_fd(), libc::LOCK_UN);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn adopt(root: &Path, name: &str) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
|
||||
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
|
||||
let running = governor::running_ids(®istry);
|
||||
let extras: Vec<String> = running
|
||||
.iter()
|
||||
.filter(|id| !target.contains(id.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !extras.is_empty() {
|
||||
bail!(
|
||||
"cannot adopt '{name}': registered model(s) outside the Scene are serving: {}",
|
||||
extras.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
let previous = State::load_checked(root)?;
|
||||
let mut adopted = BTreeMap::new();
|
||||
for id in &scene.models {
|
||||
let model = ®istry.models[id];
|
||||
if !governor::is_running(model) {
|
||||
bail!("cannot adopt '{name}': exact model '{id}' is not healthy");
|
||||
}
|
||||
let legacy = previous.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute"
|
||||
)
|
||||
})?;
|
||||
let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port)
|
||||
.with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?;
|
||||
adopted.insert(id.clone(), captured);
|
||||
}
|
||||
|
||||
let mut state = previous;
|
||||
state.procs = adopted;
|
||||
state.desired_scene = Some(name.to_string());
|
||||
state.active_scene = Some(name.to_string());
|
||||
state.last_known_good_scene = Some(name.to_string());
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("adopted exact running Scene '{name}' and captured process identities");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let prior = State::load_checked(root)?;
|
||||
let prior_active = prior.active_scene.clone();
|
||||
|
||||
if dry_run {
|
||||
return activate_once(root, name, true);
|
||||
}
|
||||
|
||||
match activate_once(root, name, false) {
|
||||
Ok(()) => {
|
||||
let mut state = State::load_checked(root)?;
|
||||
if prior_active.as_deref() != Some(name) {
|
||||
if let Some(previous) = prior_active {
|
||||
state.last_known_good_scene = Some(previous);
|
||||
}
|
||||
}
|
||||
state.desired_scene = Some(name.to_string());
|
||||
state.active_scene = Some(name.to_string());
|
||||
if state.last_known_good_scene.is_none() {
|
||||
state.last_known_good_scene = Some(name.to_string());
|
||||
}
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("Scene '{name}' is active and persisted as desired");
|
||||
Ok(())
|
||||
}
|
||||
Err(target_error) => {
|
||||
let target_message = format!("activating '{name}' failed: {target_error:#}");
|
||||
eprintln!("{target_message}");
|
||||
let transition_started =
|
||||
State::load_checked(root)?.transition_scene.as_deref() == Some(name);
|
||||
// Where a failed transition would roll back to. Absent when the scene being
|
||||
// activated is already the active one, which is the ordinary case for
|
||||
// "start the models in this scene that are currently down".
|
||||
let rollback_target = prior_active.as_deref().filter(|previous| *previous != name);
|
||||
// Only tear the box down when there is somewhere to put it back.
|
||||
//
|
||||
// This used to run whenever a transition had begun, including when the target
|
||||
// scene *was* the prior scene — so a refused admission stopped every running
|
||||
// model and then skipped the rollback, because the rollback target had been
|
||||
// filtered out for being the same scene. Twice on 2026-08-14 that turned one
|
||||
// model failing admission into an empty box, taking brain-nemotron and music
|
||||
// down with it.
|
||||
//
|
||||
// With no rollback target, leaving the partially-started scene running is
|
||||
// strictly better: those models are healthy and they belong to the scene that
|
||||
// was asked for. The refusal is reported either way.
|
||||
let cleanup_error = if transition_started && rollback_target.is_some() {
|
||||
stop_all_owned(root).err()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let rollback = if transition_started && cleanup_error.is_none() {
|
||||
rollback_target
|
||||
.map(|previous| (previous.to_string(), activate_once(root, previous, false)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.transition_scene = None;
|
||||
state.last_error = Some(target_message.clone());
|
||||
match rollback {
|
||||
Some((previous, Ok(()))) => {
|
||||
state.desired_scene = Some(previous.clone());
|
||||
state.active_scene = Some(previous.clone());
|
||||
if state.last_known_good_scene.is_none() {
|
||||
state.last_known_good_scene = Some(previous.clone());
|
||||
}
|
||||
state.save(root)?;
|
||||
eprintln!("rolled back to Scene '{previous}'");
|
||||
bail!("{target_message}; rolled back to '{previous}'")
|
||||
}
|
||||
Some((previous, Err(rollback_error))) => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
bail!(
|
||||
"{target_message}; rollback to '{previous}' also failed: {rollback_error:#}"
|
||||
)
|
||||
}
|
||||
None if !transition_started => {
|
||||
state.save(root)?;
|
||||
bail!("{target_message}")
|
||||
}
|
||||
None if cleanup_error.is_none() => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
bail!("{target_message}")
|
||||
}
|
||||
None => {
|
||||
state.active_scene = None;
|
||||
state.save(root)?;
|
||||
let cleanup_error = cleanup_error.expect("guarded by match condition");
|
||||
bail!("{target_message}; cleanup also failed: {cleanup_error:#}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_all_owned(root: &Path) -> Result<()> {
|
||||
let mut state = State::load_checked(root)?;
|
||||
let owned: Vec<(String, Proc)> = state
|
||||
.procs
|
||||
.iter()
|
||||
.filter(|(_, process)| process.owned_alive())
|
||||
.map(|(id, process)| (id.clone(), process.clone()))
|
||||
.collect();
|
||||
for (id, process) in owned {
|
||||
proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?;
|
||||
state.procs.remove(&id);
|
||||
state.save(root)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn resume(root: &Path) -> Result<()> {
|
||||
let state = State::load_checked(root)?;
|
||||
let desired = state
|
||||
.desired_scene
|
||||
.clone()
|
||||
.or_else(|| state.last_known_good_scene.clone())
|
||||
.context("no desired Scene is persisted; activate or adopt one first")?;
|
||||
let fallback = state.last_known_good_scene.clone();
|
||||
|
||||
match activate(root, &desired, false) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(desired_error) => {
|
||||
let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else {
|
||||
return Err(desired_error).context("resuming desired Scene");
|
||||
};
|
||||
eprintln!(
|
||||
"desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'"
|
||||
);
|
||||
activate(root, &fallback, false).with_context(|| {
|
||||
format!(
|
||||
"desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed"
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Restart one explicitly supervised model without disturbing the other models
|
||||
/// in the desired Scene.
|
||||
///
|
||||
/// This is deliberately narrower than `activate`: a resident recovery attempt
|
||||
/// must not turn one failed voice runtime into a stop-all rollback of an
|
||||
/// otherwise healthy Scene. The same transition lock, identity ownership,
|
||||
/// declared/observed admission checks and exact health gate still apply.
|
||||
/// Returns `true` when a new process was started and `false` when the model
|
||||
/// recovered before the lock was acquired.
|
||||
pub fn restart_supervised_model(root: &Path, id: &str, timeout: Duration) -> Result<bool> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let result = restart_supervised_model_locked(root, id, timeout);
|
||||
if let Err(error) = &result {
|
||||
if let Ok(mut state) = State::load_checked(root) {
|
||||
state.active_scene = None;
|
||||
state.last_error = Some(format!("supervision could not restart '{id}': {error:#}"));
|
||||
let _ = state.save(root);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn restart_supervised_model_locked(root: &Path, id: &str, timeout: Duration) -> Result<bool> {
|
||||
let registry = Registry::load(root)?;
|
||||
let model = registry
|
||||
.models
|
||||
.get(id)
|
||||
.with_context(|| format!("no registered model '{id}'"))?;
|
||||
model
|
||||
.supervision
|
||||
.as_ref()
|
||||
.with_context(|| format!("model '{id}' did not opt into supervision"))?;
|
||||
let port = model
|
||||
.serve
|
||||
.port
|
||||
.with_context(|| format!("supervised model '{id}' has no serving port"))?;
|
||||
|
||||
let mut state = State::load_checked(root)?;
|
||||
let desired = state
|
||||
.desired_scene
|
||||
.clone()
|
||||
.context("no desired Scene is persisted")?;
|
||||
let scene = find_scene(root, &desired)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
if !scene.models.iter().any(|candidate| candidate == id) {
|
||||
bail!("model '{id}' is not part of desired Scene '{desired}'");
|
||||
}
|
||||
|
||||
// The probe that triggered supervision ran before the transition lock. A
|
||||
// late request or a runtime finishing its own recovery may have repaired the
|
||||
// model in that gap, so re-check before signalling anything.
|
||||
if governor::is_running(model) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match state.procs.get(id).cloned() {
|
||||
Some(process) if process.owned_alive() => {
|
||||
// Health is bad but this is still the exact process Compute started.
|
||||
// Stop only its process group; no sibling in the Scene is touched.
|
||||
proc::stop_owned(&process)?;
|
||||
}
|
||||
Some(_) if governor::port_open(port) => {
|
||||
bail!(
|
||||
"port {port} is occupied but the ownership record for '{id}' is stale; refusing to signal it"
|
||||
);
|
||||
}
|
||||
None if governor::port_open(port) => {
|
||||
bail!(
|
||||
"port {port} is occupied by an unowned process; refusing to replace supervised model '{id}'"
|
||||
);
|
||||
}
|
||||
Some(_) | None => {}
|
||||
}
|
||||
|
||||
state.procs.remove(id);
|
||||
state.active_scene = None;
|
||||
state.last_error = Some(format!("supervision is restarting '{id}'"));
|
||||
state.save(root)?;
|
||||
|
||||
// Re-read both ceilings after the old process has released its memory. The
|
||||
// registry number prevents paper overcommit; MemAvailable prevents an
|
||||
// unregistered workload from turning a nominally valid restart into a wedged
|
||||
// unified-memory node.
|
||||
let committed = governor::committed_gb(®istry);
|
||||
let available = mem::read().ok().map(|memory| memory.available_gb);
|
||||
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
|
||||
if !governor::can_admit(model.footprint_gb, committed, budget, available) {
|
||||
match available {
|
||||
Some(available) => bail!(
|
||||
"'{id}' ({:.0} GB) restart refused: {:.0} GB committed against a {:.0} GB budget, \
|
||||
{:.1} GB actually available, {:.0} GB margin required",
|
||||
model.footprint_gb,
|
||||
committed,
|
||||
budget,
|
||||
available,
|
||||
governor::SAFETY_MARGIN_GB
|
||||
),
|
||||
None => bail!(
|
||||
"'{id}' restart would exceed the {:.0} GB Scene budget with the safety margin",
|
||||
budget
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
let pid = proc::spawn(root, id, model)?;
|
||||
state.seq += 1;
|
||||
let process = Proc::capture(pid, state.seq, model.serve.port)
|
||||
.with_context(|| format!("capturing ownership for supervised '{id}'"))?;
|
||||
state.procs.insert(id.to_string(), process.clone());
|
||||
state.save(root)?;
|
||||
|
||||
if !governor::wait_healthy(model, timeout) {
|
||||
let _ = proc::stop_owned(&process);
|
||||
state.procs.remove(id);
|
||||
state.last_error = Some(format!(
|
||||
"supervised model '{id}' did not report exact health within {} seconds",
|
||||
timeout.as_secs()
|
||||
));
|
||||
state.save(root)?;
|
||||
bail!(
|
||||
"supervised model '{id}' did not report exact health within {} seconds",
|
||||
timeout.as_secs()
|
||||
);
|
||||
}
|
||||
|
||||
let unhealthy: Vec<String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|candidate| !governor::is_running(®istry.models[*candidate]))
|
||||
.cloned()
|
||||
.collect();
|
||||
if unhealthy.is_empty() {
|
||||
state.active_scene = Some(desired);
|
||||
state.last_error = None;
|
||||
} else {
|
||||
state.active_scene = None;
|
||||
state.last_error = Some(format!(
|
||||
"supervised model '{id}' recovered, but desired Scene is still unhealthy: {}",
|
||||
unhealthy.join(", ")
|
||||
));
|
||||
}
|
||||
state.save(root)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub fn deactivate(root: &Path) -> Result<()> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let mut state = State::load_checked(root)?;
|
||||
let ids: Vec<String> = state.procs.keys().cloned().collect();
|
||||
for id in ids {
|
||||
let process = state.procs[&id].clone();
|
||||
if process.owned_alive() {
|
||||
println!(" stopping {id} (pid {})", process.pid);
|
||||
proc::stop_owned(&process)?;
|
||||
}
|
||||
state.procs.remove(&id);
|
||||
state.save(root)?;
|
||||
}
|
||||
state.desired_scene = None;
|
||||
state.active_scene = None;
|
||||
state.transition_scene = None;
|
||||
state.last_error = None;
|
||||
state.save(root)?;
|
||||
println!("all Compute-managed models stopped; no Scene is desired");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Diff the target Scene against what is actually serving. Pure: it decides
|
||||
/// *what* would change, never *whether* the change is allowed.
|
||||
fn plan_transition(registry: &Registry, scene: &Scene) -> Plan {
|
||||
let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
|
||||
let running = governor::running_ids(registry);
|
||||
let stop: Vec<String> = running
|
||||
.iter()
|
||||
.filter(|id| !target.contains(id.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut start: Vec<String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !governor::is_running(®istry.models[*id]))
|
||||
.cloned()
|
||||
.collect();
|
||||
let listed = scene
|
||||
.activation
|
||||
.as_ref()
|
||||
.and_then(|activation| activation.order.as_deref())
|
||||
.map(|order| order == "listed")
|
||||
.unwrap_or(false);
|
||||
if !listed {
|
||||
start.sort_by(|a, b| {
|
||||
registry.models[a]
|
||||
.footprint_gb
|
||||
.partial_cmp(®istry.models[b].footprint_gb)
|
||||
.unwrap()
|
||||
});
|
||||
}
|
||||
Plan {
|
||||
scene: scene.metadata.name.clone(),
|
||||
budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB),
|
||||
stop,
|
||||
start,
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything that must hold before the first process is signalled. Runs to
|
||||
/// completion with nothing mutated, so a rejection here leaves the currently
|
||||
/// active Scene exactly as it was.
|
||||
fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> {
|
||||
for id in &scene.models {
|
||||
if governor::is_running(®istry.models[id]) {
|
||||
let process = state.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first"
|
||||
)
|
||||
})?;
|
||||
if !process.owned_alive() {
|
||||
bail!(
|
||||
"ownership record for serving model '{id}' is stale; adopt the active Scene first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id in &plan.stop {
|
||||
let process = state.procs.get(id).with_context(|| {
|
||||
format!(
|
||||
"'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching"
|
||||
)
|
||||
})?;
|
||||
if !process.owned_alive() {
|
||||
bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid");
|
||||
}
|
||||
}
|
||||
for id in &plan.start {
|
||||
let model = ®istry.models[id];
|
||||
if let Some(port) = model.serve.port {
|
||||
// A model in `plan.stop` may currently hold this port. Every stop runs before
|
||||
// any start, so that is a handoff, not a conflict. Without this exemption every
|
||||
// same-port swap is rejected — including brain -> brain-laguna/brain-gemma,
|
||||
// which share :8001 by design because the alias downstream agents call must
|
||||
// survive a weight swap.
|
||||
let freed_by_stop = plan.stop.iter().any(|stopping| {
|
||||
registry
|
||||
.models
|
||||
.get(stopping)
|
||||
.and_then(|stopped| stopped.serve.port)
|
||||
== Some(port)
|
||||
});
|
||||
if !freed_by_stop && governor::port_open(port) && !governor::is_running(model) {
|
||||
bail!("port {port} is occupied by a different model; refusing to start '{id}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `scene activate --dry-run` as data rather than as printed lines: the same
|
||||
/// validation, the same plan, nothing written. Callers that need the plan
|
||||
/// programmatically use this instead of scraping stdout.
|
||||
pub fn plan(root: &Path, name: &str) -> Result<Plan> {
|
||||
let _lock = TransitionLock::acquire(root)?;
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
let plan = plan_transition(®istry, &scene);
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.procs.retain(|_, process| process.owned_alive());
|
||||
preflight(®istry, &scene, &plan, &state)?;
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> {
|
||||
let registry = Registry::load(root)?;
|
||||
let scene = find_scene(root, name)?;
|
||||
validate_scene(®istry, &scene)?;
|
||||
let plan = plan_transition(®istry, &scene);
|
||||
|
||||
println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb);
|
||||
println!(" stop : {}", format_ids(&plan.stop));
|
||||
println!(" start: {}", format_ids(&plan.start));
|
||||
|
||||
let mut state = State::load_checked(root)?;
|
||||
state.procs.retain(|_, process| process.owned_alive());
|
||||
preflight(®istry, &scene, &plan, &state)?;
|
||||
|
||||
if dry_run {
|
||||
println!(" dry run: validation passed; nothing changed");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
state.transition_scene = Some(name.to_string());
|
||||
state.save(root)?;
|
||||
|
||||
for id in &plan.stop {
|
||||
let process = state.procs[id].clone();
|
||||
println!(" stopping {id} (pid {})", process.pid);
|
||||
proc::stop_owned(&process)?;
|
||||
state.procs.remove(id);
|
||||
state.save(root)?;
|
||||
}
|
||||
|
||||
let mut committed: f64 = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| governor::is_running(®istry.models[*id]))
|
||||
.map(|id| registry.models[id].footprint_gb)
|
||||
.sum();
|
||||
let wait_healthy = scene
|
||||
.activation
|
||||
.as_ref()
|
||||
.and_then(|activation| activation.wait_healthy)
|
||||
.unwrap_or(true);
|
||||
|
||||
for id in &plan.start {
|
||||
let model: &Model = ®istry.models[id];
|
||||
// Re-read the pool before every start rather than once per activation: each model
|
||||
// that comes up consumes real memory, and its true appetite is only knowable after
|
||||
// it has allocated. A footprint that was optimistic shows up here, on the next
|
||||
// model, instead of taking the box down.
|
||||
let available_gb = mem::read().ok().map(|m| m.available_gb);
|
||||
if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) {
|
||||
match available_gb {
|
||||
Some(available) => bail!(
|
||||
"'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \
|
||||
{:.1} GB actually available, {:.0} GB margin required",
|
||||
model.footprint_gb,
|
||||
committed,
|
||||
plan.budget_gb,
|
||||
available,
|
||||
governor::SAFETY_MARGIN_GB
|
||||
),
|
||||
None => bail!(
|
||||
"'{id}' would exceed the {:.0} GB Scene budget with the safety margin",
|
||||
plan.budget_gb
|
||||
),
|
||||
}
|
||||
}
|
||||
let pid = proc::spawn(root, id, model)?;
|
||||
state.seq += 1;
|
||||
let process = Proc::capture(pid, state.seq, model.serve.port)
|
||||
.with_context(|| format!("capturing ownership for newly started '{id}'"))?;
|
||||
state.procs.insert(id.clone(), process.clone());
|
||||
state.save(root)?;
|
||||
committed += model.footprint_gb;
|
||||
println!(" started {id} (pid {pid}); waiting for exact health");
|
||||
if wait_healthy && !governor::wait_healthy(model, health_timeout()) {
|
||||
let _ = proc::stop_owned(&process);
|
||||
state.procs.remove(id);
|
||||
state.save(root)?;
|
||||
bail!("'{id}' did not report its exact health marker before timeout");
|
||||
}
|
||||
}
|
||||
|
||||
let unhealthy: Vec<String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !governor::is_running(®istry.models[*id]))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !unhealthy.is_empty() {
|
||||
bail!(
|
||||
"Scene '{name}' is incomplete; exact health failed for: {}",
|
||||
unhealthy.join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> {
|
||||
let missing: Vec<&String> = scene
|
||||
.models
|
||||
.iter()
|
||||
.filter(|id| !registry.models.contains_key(*id))
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
bail!(
|
||||
"Scene '{}' references unknown model id(s): {}",
|
||||
scene.metadata.name,
|
||||
missing
|
||||
.iter()
|
||||
.map(|id| id.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
}
|
||||
let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
|
||||
let footprint: f64 = scene
|
||||
.models
|
||||
.iter()
|
||||
.map(|id| registry.models[id].footprint_gb)
|
||||
.sum();
|
||||
if footprint + governor::SAFETY_MARGIN_GB > budget {
|
||||
bail!(
|
||||
"Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget",
|
||||
scene.metadata.name,
|
||||
footprint + governor::SAFETY_MARGIN_GB,
|
||||
budget
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn health_timeout() -> Duration {
|
||||
Duration::from_secs(if cfg!(test) { 5 } else { 900 })
|
||||
}
|
||||
|
||||
fn format_ids(ids: &[String]) -> String {
|
||||
if ids.is_empty() {
|
||||
"(none)".to_string()
|
||||
} else {
|
||||
ids.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::TcpListener;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn fixture_root() -> (PathBuf, u16) {
|
||||
let unique = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}"));
|
||||
fs::create_dir_all(root.join("registry")).unwrap();
|
||||
fs::create_dir_all(root.join("scenes")).unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
drop(listener);
|
||||
let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let bad_port = bad_listener.local_addr().unwrap().port();
|
||||
drop(bad_listener);
|
||||
let registry = format!(
|
||||
"apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n supervision:\n restart: always\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n"
|
||||
);
|
||||
fs::write(root.join("registry/models.yaml"), registry).unwrap();
|
||||
fs::write(
|
||||
root.join("scenes/test.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
(root, port)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn activation_persists_and_deactivation_stops_owned_process() {
|
||||
let (root, _port) = fixture_root();
|
||||
activate(&root, "test", false).unwrap();
|
||||
let state = State::load_checked(&root).unwrap();
|
||||
assert_eq!(state.desired_scene.as_deref(), Some("test"));
|
||||
assert_eq!(state.active_scene.as_deref(), Some("test"));
|
||||
assert!(state.procs["fake"].owned_alive());
|
||||
deactivate(&root).unwrap();
|
||||
assert!(State::load_checked(&root).unwrap().procs.is_empty());
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supervision_restarts_only_the_failed_owned_model() {
|
||||
let (root, _port) = fixture_root();
|
||||
activate(&root, "test", false).unwrap();
|
||||
let before = State::load_checked(&root).unwrap();
|
||||
let old = before.procs["fake"].clone();
|
||||
proc::stop_owned(&old).unwrap();
|
||||
|
||||
assert!(restart_supervised_model(&root, "fake", Duration::from_secs(5)).unwrap());
|
||||
let after = State::load_checked(&root).unwrap();
|
||||
assert_ne!(after.procs["fake"].pid, old.pid);
|
||||
assert!(after.procs["fake"].owned_alive());
|
||||
assert_eq!(after.desired_scene.as_deref(), Some("test"));
|
||||
assert_eq!(after.active_scene.as_deref(), Some("test"));
|
||||
|
||||
deactivate(&root).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supervision_refuses_to_replace_an_unowned_process() {
|
||||
let (root, port) = fixture_root();
|
||||
let registry = Registry::load(&root).unwrap();
|
||||
let model = ®istry.models["fake"];
|
||||
let pid = proc::spawn(&root, "fake", model).unwrap();
|
||||
assert!(governor::wait_healthy(model, Duration::from_secs(5)));
|
||||
|
||||
// Make exact health fail while the unowned server continues to hold the
|
||||
// port. Supervision must not infer ownership from a port number.
|
||||
let registry_path = root.join("registry/models.yaml");
|
||||
let contents = fs::read_to_string(®istry_path).unwrap();
|
||||
fs::write(
|
||||
®istry_path,
|
||||
contents.replace(
|
||||
&format!("health: http://localhost:{port}/"),
|
||||
&format!(
|
||||
"health: http://localhost:{port}/\n health_contains: marker-that-is-not-served"
|
||||
),
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(root.join(".compute")).unwrap();
|
||||
State {
|
||||
desired_scene: Some("test".to_string()),
|
||||
active_scene: Some("test".to_string()),
|
||||
last_known_good_scene: Some("test".to_string()),
|
||||
..State::default()
|
||||
}
|
||||
.save(&root)
|
||||
.unwrap();
|
||||
|
||||
let error = restart_supervised_model(&root, "fake", Duration::from_millis(50))
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("unowned process"));
|
||||
assert!(governor::port_open(port));
|
||||
|
||||
let owned = Proc::capture(pid, 1, Some(port)).unwrap();
|
||||
proc::stop_owned(&owned).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
/// Two models sharing one port, distinguishable by `health_contains` — the
|
||||
/// brain/brain-laguna shape. Each serves its own directory so the health probe
|
||||
/// can tell which one is actually up.
|
||||
fn same_port_root() -> PathBuf {
|
||||
let unique = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let root = std::env::temp_dir().join(format!("lumbridge-compute-swap-{unique}"));
|
||||
fs::create_dir_all(root.join("registry")).unwrap();
|
||||
fs::create_dir_all(root.join("scenes")).unwrap();
|
||||
let dir_a = root.join("srv-alpha");
|
||||
let dir_b = root.join("srv-beta");
|
||||
fs::create_dir_all(&dir_a).unwrap();
|
||||
fs::create_dir_all(&dir_b).unwrap();
|
||||
fs::write(dir_a.join("alpha-marker.txt"), "a").unwrap();
|
||||
fs::write(dir_b.join("beta-marker.txt"), "b").unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
drop(listener);
|
||||
let (a, b) = (dir_a.display(), dir_b.display());
|
||||
let registry = format!(
|
||||
"apiVersion: lumbridge/v1\nmodels:\n alpha:\n name: Alpha\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: alpha-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{a}']\n beta:\n name: Beta\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: beta-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{b}']\n"
|
||||
);
|
||||
fs::write(root.join("registry/models.yaml"), registry).unwrap();
|
||||
fs::write(
|
||||
root.join("scenes/a.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: a\n version: 1\nmodels: [alpha]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(
|
||||
root.join("scenes/b.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: b\n version: 1\nmodels: [beta]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
root
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_port_swap_is_allowed_when_the_occupant_is_being_stopped() {
|
||||
let root = same_port_root();
|
||||
activate(&root, "a", false).unwrap();
|
||||
assert!(State::load_checked(&root)
|
||||
.unwrap()
|
||||
.procs
|
||||
.contains_key("alpha"));
|
||||
|
||||
// Regression: this used to fail with "port N is occupied by a different model;
|
||||
// refusing to start 'beta'". The pre-flight port check ran over `plan.start`
|
||||
// without exempting ports released by `plan.stop`, so every same-port swap was
|
||||
// rejected even though stops precede starts.
|
||||
activate(&root, "b", false).unwrap();
|
||||
|
||||
let state = State::load_checked(&root).unwrap();
|
||||
assert_eq!(state.active_scene.as_deref(), Some("b"));
|
||||
assert!(state.procs.contains_key("beta"));
|
||||
assert!(!state.procs.contains_key("alpha"));
|
||||
|
||||
deactivate(&root).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_serving_scene_requires_explicit_adoption() {
|
||||
let (root, port) = fixture_root();
|
||||
let registry = Registry::load(&root).unwrap();
|
||||
let model = ®istry.models["fake"];
|
||||
let pid = proc::spawn(&root, "fake", model).unwrap();
|
||||
assert!(governor::wait_healthy(model, Duration::from_secs(5)));
|
||||
let error = activate(&root, "test", true).unwrap_err().to_string();
|
||||
assert!(error.contains("adopt the active Scene"));
|
||||
let owned = Proc::capture(pid, 1, Some(port)).unwrap();
|
||||
proc::stop_owned(&owned).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preflight_rejection_preserves_healthy_active_scene() {
|
||||
let (root, _port) = fixture_root();
|
||||
activate(&root, "test", false).unwrap();
|
||||
let before = State::load_checked(&root).unwrap();
|
||||
let pid = before.procs["fake"].pid;
|
||||
fs::write(
|
||||
root.join("scenes/invalid.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(activate(&root, "invalid", false).is_err());
|
||||
let after = State::load_checked(&root).unwrap();
|
||||
assert_eq!(after.active_scene.as_deref(), Some("test"));
|
||||
assert_eq!(after.desired_scene.as_deref(), Some("test"));
|
||||
assert_eq!(after.procs["fake"].pid, pid);
|
||||
assert!(after.procs["fake"].owned_alive());
|
||||
deactivate(&root).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_activation_keeps_healthy_partial_scene_without_rollback_target() {
|
||||
let (root, port) = fixture_root();
|
||||
fs::write(
|
||||
root.join("scenes/failing.scene.yaml"),
|
||||
"apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(activate(&root, "failing", false).is_err());
|
||||
let state = State::load_checked(&root).unwrap();
|
||||
assert!(state.procs.contains_key("fake"));
|
||||
assert!(!state.procs.contains_key("bad"));
|
||||
assert!(state.active_scene.is_none());
|
||||
assert!(governor::port_open(port));
|
||||
deactivate(&root).unwrap();
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_state_load_rejects_malformed_yaml() {
|
||||
let (root, _port) = fixture_root();
|
||||
fs::create_dir_all(root.join(".compute")).unwrap();
|
||||
fs::write(root.join(".compute/state.yaml"), "desired_scene: [").unwrap();
|
||||
assert!(State::load_checked(&root).is_err());
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_write_replaces_complete_yaml_atomically() {
|
||||
let (root, _port) = fixture_root();
|
||||
let state = State {
|
||||
desired_scene: Some("test".to_string()),
|
||||
..State::default()
|
||||
};
|
||||
state.save(&root).unwrap();
|
||||
let loaded = State::load_checked(&root).unwrap();
|
||||
assert_eq!(loaded.desired_scene.as_deref(), Some("test"));
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user