//! Transactional Scene lifecycle and persisted desired-state recovery. use anyhow::{bail, Context, Result}; use std::collections::{BTreeMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::os::fd::AsRawFd; use std::path::Path; use std::time::Duration; use crate::config::{find_scene, Model, Registry, Scene}; use crate::governor; use crate::mem; use crate::proc::{self, Proc, State}; /// What activating a Scene would do, decided before anything is mutated. /// /// `activate` renders this for the CLI and the MCP server returns it verbatim, /// so a plan an operator reads and a plan an agent reads can never drift apart. #[derive(Debug, Clone)] pub struct Plan { pub scene: String, pub budget_gb: f64, /// Registered models serving outside the target Scene; stopped first. pub stop: Vec, /// Scene models not yet serving, in the order they would be admitted. pub start: Vec, } pub struct TransitionLock { file: File, } impl TransitionLock { pub fn acquire(root: &Path) -> Result { let dir = root.join(".compute"); fs::create_dir_all(&dir)?; let path = dir.join("transition.lock"); let file = OpenOptions::new() .create(true) .truncate(false) .read(true) .write(true) .open(&path) .with_context(|| format!("opening transition lock {}", path.display()))?; let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; if result != 0 { return Err(std::io::Error::last_os_error()) .context("another Scene transition is already running; wait for it to finish"); } Ok(Self { file }) } } impl Drop for TransitionLock { fn drop(&mut self) { unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN); } } } pub fn adopt(root: &Path, name: &str) -> Result<()> { let _lock = TransitionLock::acquire(root)?; let registry = Registry::load(root)?; let scene = find_scene(root, name)?; validate_scene(®istry, &scene)?; let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); let running = governor::running_ids(®istry); let extras: Vec = running .iter() .filter(|id| !target.contains(id.as_str())) .cloned() .collect(); if !extras.is_empty() { bail!( "cannot adopt '{name}': registered model(s) outside the Scene are serving: {}", extras.join(", ") ); } let previous = State::load_checked(root)?; let mut adopted = BTreeMap::new(); for id in &scene.models { let model = ®istry.models[id]; if !governor::is_running(model) { bail!("cannot adopt '{name}': exact model '{id}' is not healthy"); } let legacy = previous.procs.get(id).with_context(|| { format!( "cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute" ) })?; let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port) .with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?; adopted.insert(id.clone(), captured); } let mut state = previous; state.procs = adopted; state.desired_scene = Some(name.to_string()); state.active_scene = Some(name.to_string()); state.last_known_good_scene = Some(name.to_string()); state.transition_scene = None; state.last_error = None; state.save(root)?; println!("adopted exact running Scene '{name}' and captured process identities"); Ok(()) } pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> { let _lock = TransitionLock::acquire(root)?; let prior = State::load_checked(root)?; let prior_active = prior.active_scene.clone(); if dry_run { return activate_once(root, name, true); } match activate_once(root, name, false) { Ok(()) => { let mut state = State::load_checked(root)?; if prior_active.as_deref() != Some(name) { if let Some(previous) = prior_active { state.last_known_good_scene = Some(previous); } } state.desired_scene = Some(name.to_string()); state.active_scene = Some(name.to_string()); if state.last_known_good_scene.is_none() { state.last_known_good_scene = Some(name.to_string()); } state.transition_scene = None; state.last_error = None; state.save(root)?; println!("Scene '{name}' is active and persisted as desired"); Ok(()) } Err(target_error) => { let target_message = format!("activating '{name}' failed: {target_error:#}"); eprintln!("{target_message}"); let transition_started = State::load_checked(root)?.transition_scene.as_deref() == Some(name); // Where a failed transition would roll back to. Absent when the scene being // activated is already the active one, which is the ordinary case for // "start the models in this scene that are currently down". let rollback_target = prior_active.as_deref().filter(|previous| *previous != name); // Only tear the box down when there is somewhere to put it back. // // This used to run whenever a transition had begun, including when the target // scene *was* the prior scene — so a refused admission stopped every running // model and then skipped the rollback, because the rollback target had been // filtered out for being the same scene. Twice on 2026-08-14 that turned one // model failing admission into an empty box, taking brain-nemotron and music // down with it. // // With no rollback target, leaving the partially-started scene running is // strictly better: those models are healthy and they belong to the scene that // was asked for. The refusal is reported either way. let cleanup_error = if transition_started && rollback_target.is_some() { stop_all_owned(root).err() } else { None }; let rollback = if transition_started && cleanup_error.is_none() { rollback_target .map(|previous| (previous.to_string(), activate_once(root, previous, false))) } else { None }; let mut state = State::load_checked(root)?; state.transition_scene = None; state.last_error = Some(target_message.clone()); match rollback { Some((previous, Ok(()))) => { state.desired_scene = Some(previous.clone()); state.active_scene = Some(previous.clone()); if state.last_known_good_scene.is_none() { state.last_known_good_scene = Some(previous.clone()); } state.save(root)?; eprintln!("rolled back to Scene '{previous}'"); bail!("{target_message}; rolled back to '{previous}'") } Some((previous, Err(rollback_error))) => { state.active_scene = None; state.save(root)?; bail!( "{target_message}; rollback to '{previous}' also failed: {rollback_error:#}" ) } None if !transition_started => { state.save(root)?; bail!("{target_message}") } None if cleanup_error.is_none() => { state.active_scene = None; state.save(root)?; bail!("{target_message}") } None => { state.active_scene = None; state.save(root)?; let cleanup_error = cleanup_error.expect("guarded by match condition"); bail!("{target_message}; cleanup also failed: {cleanup_error:#}") } } } } } fn stop_all_owned(root: &Path) -> Result<()> { let mut state = State::load_checked(root)?; let owned: Vec<(String, Proc)> = state .procs .iter() .filter(|(_, process)| process.owned_alive()) .map(|(id, process)| (id.clone(), process.clone())) .collect(); for (id, process) in owned { proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?; state.procs.remove(&id); state.save(root)?; } Ok(()) } pub fn resume(root: &Path) -> Result<()> { let state = State::load_checked(root)?; let desired = state .desired_scene .clone() .or_else(|| state.last_known_good_scene.clone()) .context("no desired Scene is persisted; activate or adopt one first")?; let fallback = state.last_known_good_scene.clone(); match activate(root, &desired, false) { Ok(()) => Ok(()), Err(desired_error) => { let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else { return Err(desired_error).context("resuming desired Scene"); }; eprintln!( "desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'" ); activate(root, &fallback, false).with_context(|| { format!( "desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed" ) }) } } } /// Restart one explicitly supervised model without disturbing the other models /// in the desired Scene. /// /// This is deliberately narrower than `activate`: a resident recovery attempt /// must not turn one failed voice runtime into a stop-all rollback of an /// otherwise healthy Scene. The same transition lock, identity ownership, /// declared/observed admission checks and exact health gate still apply. /// Returns `true` when a new process was started and `false` when the model /// recovered before the lock was acquired. pub fn restart_supervised_model(root: &Path, id: &str, timeout: Duration) -> Result { let _lock = TransitionLock::acquire(root)?; let result = restart_supervised_model_locked(root, id, timeout); if let Err(error) = &result { if let Ok(mut state) = State::load_checked(root) { state.active_scene = None; state.last_error = Some(format!("supervision could not restart '{id}': {error:#}")); let _ = state.save(root); } } result } fn restart_supervised_model_locked(root: &Path, id: &str, timeout: Duration) -> Result { let registry = Registry::load(root)?; let model = registry .models .get(id) .with_context(|| format!("no registered model '{id}'"))?; model .supervision .as_ref() .with_context(|| format!("model '{id}' did not opt into supervision"))?; let port = model .serve .port .with_context(|| format!("supervised model '{id}' has no serving port"))?; let mut state = State::load_checked(root)?; let desired = state .desired_scene .clone() .context("no desired Scene is persisted")?; let scene = find_scene(root, &desired)?; validate_scene(®istry, &scene)?; if !scene.models.iter().any(|candidate| candidate == id) { bail!("model '{id}' is not part of desired Scene '{desired}'"); } // The probe that triggered supervision ran before the transition lock. A // late request or a runtime finishing its own recovery may have repaired the // model in that gap, so re-check before signalling anything. if governor::is_running(model) { return Ok(false); } match state.procs.get(id).cloned() { Some(process) if process.owned_alive() => { // Health is bad but this is still the exact process Compute started. // Stop only its process group; no sibling in the Scene is touched. proc::stop_owned(&process)?; } Some(_) if governor::port_open(port) => { bail!( "port {port} is occupied but the ownership record for '{id}' is stale; refusing to signal it" ); } None if governor::port_open(port) => { bail!( "port {port} is occupied by an unowned process; refusing to replace supervised model '{id}'" ); } Some(_) | None => {} } state.procs.remove(id); state.active_scene = None; state.last_error = Some(format!("supervision is restarting '{id}'")); state.save(root)?; // Re-read both ceilings after the old process has released its memory. The // registry number prevents paper overcommit; MemAvailable prevents an // unregistered workload from turning a nominally valid restart into a wedged // unified-memory node. let committed = governor::committed_gb(®istry); let available = mem::read().ok().map(|memory| memory.available_gb); let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); if !governor::can_admit(model.footprint_gb, committed, budget, available) { match available { Some(available) => bail!( "'{id}' ({:.0} GB) restart refused: {:.0} GB committed against a {:.0} GB budget, \ {:.1} GB actually available, {:.0} GB margin required", model.footprint_gb, committed, budget, available, governor::SAFETY_MARGIN_GB ), None => bail!( "'{id}' restart would exceed the {:.0} GB Scene budget with the safety margin", budget ), } } let pid = proc::spawn(root, id, model)?; state.seq += 1; let process = Proc::capture(pid, state.seq, model.serve.port) .with_context(|| format!("capturing ownership for supervised '{id}'"))?; state.procs.insert(id.to_string(), process.clone()); state.save(root)?; if !governor::wait_healthy(model, timeout) { let _ = proc::stop_owned(&process); state.procs.remove(id); state.last_error = Some(format!( "supervised model '{id}' did not report exact health within {} seconds", timeout.as_secs() )); state.save(root)?; bail!( "supervised model '{id}' did not report exact health within {} seconds", timeout.as_secs() ); } let unhealthy: Vec = scene .models .iter() .filter(|candidate| !governor::is_running(®istry.models[*candidate])) .cloned() .collect(); if unhealthy.is_empty() { state.active_scene = Some(desired); state.last_error = None; } else { state.active_scene = None; state.last_error = Some(format!( "supervised model '{id}' recovered, but desired Scene is still unhealthy: {}", unhealthy.join(", ") )); } state.save(root)?; Ok(true) } pub fn deactivate(root: &Path) -> Result<()> { let _lock = TransitionLock::acquire(root)?; let mut state = State::load_checked(root)?; let ids: Vec = state.procs.keys().cloned().collect(); for id in ids { let process = state.procs[&id].clone(); if process.owned_alive() { println!(" stopping {id} (pid {})", process.pid); proc::stop_owned(&process)?; } state.procs.remove(&id); state.save(root)?; } state.desired_scene = None; state.active_scene = None; state.transition_scene = None; state.last_error = None; state.save(root)?; println!("all Compute-managed models stopped; no Scene is desired"); Ok(()) } /// Diff the target Scene against what is actually serving. Pure: it decides /// *what* would change, never *whether* the change is allowed. fn plan_transition(registry: &Registry, scene: &Scene) -> Plan { let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect(); let running = governor::running_ids(registry); let stop: Vec = running .iter() .filter(|id| !target.contains(id.as_str())) .cloned() .collect(); let mut start: Vec = scene .models .iter() .filter(|id| !governor::is_running(®istry.models[*id])) .cloned() .collect(); let listed = scene .activation .as_ref() .and_then(|activation| activation.order.as_deref()) .map(|order| order == "listed") .unwrap_or(false); if !listed { start.sort_by(|a, b| { registry.models[a] .footprint_gb .partial_cmp(®istry.models[b].footprint_gb) .unwrap() }); } Plan { scene: scene.metadata.name.clone(), budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB), stop, start, } } /// Everything that must hold before the first process is signalled. Runs to /// completion with nothing mutated, so a rejection here leaves the currently /// active Scene exactly as it was. fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> { for id in &scene.models { if governor::is_running(®istry.models[id]) { let process = state.procs.get(id).with_context(|| { format!( "'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first" ) })?; if !process.owned_alive() { bail!( "ownership record for serving model '{id}' is stale; adopt the active Scene first" ); } } } for id in &plan.stop { let process = state.procs.get(id).with_context(|| { format!( "'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching" ) })?; if !process.owned_alive() { bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid"); } } for id in &plan.start { let model = ®istry.models[id]; if let Some(port) = model.serve.port { // A model in `plan.stop` may currently hold this port. Every stop runs before // any start, so that is a handoff, not a conflict. Without this exemption every // same-port swap is rejected — including brain -> brain-laguna/brain-gemma, // which share :8001 by design because the alias downstream agents call must // survive a weight swap. let freed_by_stop = plan.stop.iter().any(|stopping| { registry .models .get(stopping) .and_then(|stopped| stopped.serve.port) == Some(port) }); if !freed_by_stop && governor::port_open(port) && !governor::is_running(model) { bail!("port {port} is occupied by a different model; refusing to start '{id}'"); } } } Ok(()) } /// `scene activate --dry-run` as data rather than as printed lines: the same /// validation, the same plan, nothing written. Callers that need the plan /// programmatically use this instead of scraping stdout. pub fn plan(root: &Path, name: &str) -> Result { let _lock = TransitionLock::acquire(root)?; let registry = Registry::load(root)?; let scene = find_scene(root, name)?; validate_scene(®istry, &scene)?; let plan = plan_transition(®istry, &scene); let mut state = State::load_checked(root)?; state.procs.retain(|_, process| process.owned_alive()); preflight(®istry, &scene, &plan, &state)?; Ok(plan) } fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> { let registry = Registry::load(root)?; let scene = find_scene(root, name)?; validate_scene(®istry, &scene)?; let plan = plan_transition(®istry, &scene); println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb); println!(" stop : {}", format_ids(&plan.stop)); println!(" start: {}", format_ids(&plan.start)); let mut state = State::load_checked(root)?; state.procs.retain(|_, process| process.owned_alive()); preflight(®istry, &scene, &plan, &state)?; if dry_run { println!(" dry run: validation passed; nothing changed"); return Ok(()); } state.transition_scene = Some(name.to_string()); state.save(root)?; for id in &plan.stop { let process = state.procs[id].clone(); println!(" stopping {id} (pid {})", process.pid); proc::stop_owned(&process)?; state.procs.remove(id); state.save(root)?; } let mut committed: f64 = scene .models .iter() .filter(|id| governor::is_running(®istry.models[*id])) .map(|id| registry.models[id].footprint_gb) .sum(); let wait_healthy = scene .activation .as_ref() .and_then(|activation| activation.wait_healthy) .unwrap_or(true); for id in &plan.start { let model: &Model = ®istry.models[id]; // Re-read the pool before every start rather than once per activation: each model // that comes up consumes real memory, and its true appetite is only knowable after // it has allocated. A footprint that was optimistic shows up here, on the next // model, instead of taking the box down. let available_gb = mem::read().ok().map(|m| m.available_gb); if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) { match available_gb { Some(available) => bail!( "'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \ {:.1} GB actually available, {:.0} GB margin required", model.footprint_gb, committed, plan.budget_gb, available, governor::SAFETY_MARGIN_GB ), None => bail!( "'{id}' would exceed the {:.0} GB Scene budget with the safety margin", plan.budget_gb ), } } let pid = proc::spawn(root, id, model)?; state.seq += 1; let process = Proc::capture(pid, state.seq, model.serve.port) .with_context(|| format!("capturing ownership for newly started '{id}'"))?; state.procs.insert(id.clone(), process.clone()); state.save(root)?; committed += model.footprint_gb; println!(" started {id} (pid {pid}); waiting for exact health"); if wait_healthy && !governor::wait_healthy(model, health_timeout()) { let _ = proc::stop_owned(&process); state.procs.remove(id); state.save(root)?; bail!("'{id}' did not report its exact health marker before timeout"); } } let unhealthy: Vec = scene .models .iter() .filter(|id| !governor::is_running(®istry.models[*id])) .cloned() .collect(); if !unhealthy.is_empty() { bail!( "Scene '{name}' is incomplete; exact health failed for: {}", unhealthy.join(", ") ); } Ok(()) } fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> { let missing: Vec<&String> = scene .models .iter() .filter(|id| !registry.models.contains_key(*id)) .collect(); if !missing.is_empty() { bail!( "Scene '{}' references unknown model id(s): {}", scene.metadata.name, missing .iter() .map(|id| id.as_str()) .collect::>() .join(", ") ); } let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB); let footprint: f64 = scene .models .iter() .map(|id| registry.models[id].footprint_gb) .sum(); if footprint + governor::SAFETY_MARGIN_GB > budget { bail!( "Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget", scene.metadata.name, footprint + governor::SAFETY_MARGIN_GB, budget ); } Ok(()) } fn health_timeout() -> Duration { Duration::from_secs(if cfg!(test) { 5 } else { 900 }) } fn format_ids(ids: &[String]) -> String { if ids.is_empty() { "(none)".to_string() } else { ids.join(", ") } } #[cfg(test)] mod tests { use super::*; use std::net::TcpListener; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; fn fixture_root() -> (PathBuf, u16) { let unique = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}")); fs::create_dir_all(root.join("registry")).unwrap(); fs::create_dir_all(root.join("scenes")).unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); drop(listener); let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap(); let bad_port = bad_listener.local_addr().unwrap().port(); drop(bad_listener); let registry = format!( "apiVersion: lumbridge/v1\nmodels:\n fake:\n name: Fake\n footprint_gb: 1\n health: http://localhost:{port}/\n supervision:\n restart: always\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n bad:\n name: Bad\n footprint_gb: 2\n health: http://localhost:{bad_port}/\n serve:\n kind: exec\n port: {bad_port}\n command: [/usr/bin/false]\n" ); fs::write(root.join("registry/models.yaml"), registry).unwrap(); fs::write( root.join("scenes/test.scene.yaml"), "apiVersion: lumbridge/v1\nmetadata:\n name: test\n version: 1\nmodels: [fake]\nbudget_gb: 100\n", ) .unwrap(); (root, port) } #[test] fn activation_persists_and_deactivation_stops_owned_process() { let (root, _port) = fixture_root(); activate(&root, "test", false).unwrap(); let state = State::load_checked(&root).unwrap(); assert_eq!(state.desired_scene.as_deref(), Some("test")); assert_eq!(state.active_scene.as_deref(), Some("test")); assert!(state.procs["fake"].owned_alive()); deactivate(&root).unwrap(); assert!(State::load_checked(&root).unwrap().procs.is_empty()); fs::remove_dir_all(root).unwrap(); } #[test] fn supervision_restarts_only_the_failed_owned_model() { let (root, _port) = fixture_root(); activate(&root, "test", false).unwrap(); let before = State::load_checked(&root).unwrap(); let old = before.procs["fake"].clone(); proc::stop_owned(&old).unwrap(); assert!(restart_supervised_model(&root, "fake", Duration::from_secs(5)).unwrap()); let after = State::load_checked(&root).unwrap(); assert_ne!(after.procs["fake"].pid, old.pid); assert!(after.procs["fake"].owned_alive()); assert_eq!(after.desired_scene.as_deref(), Some("test")); assert_eq!(after.active_scene.as_deref(), Some("test")); deactivate(&root).unwrap(); fs::remove_dir_all(root).unwrap(); } #[test] fn supervision_refuses_to_replace_an_unowned_process() { let (root, port) = fixture_root(); let registry = Registry::load(&root).unwrap(); let model = ®istry.models["fake"]; let pid = proc::spawn(&root, "fake", model).unwrap(); assert!(governor::wait_healthy(model, Duration::from_secs(5))); // Make exact health fail while the unowned server continues to hold the // port. Supervision must not infer ownership from a port number. let registry_path = root.join("registry/models.yaml"); let contents = fs::read_to_string(®istry_path).unwrap(); fs::write( ®istry_path, contents.replace( &format!("health: http://localhost:{port}/"), &format!( "health: http://localhost:{port}/\n health_contains: marker-that-is-not-served" ), ), ) .unwrap(); fs::create_dir_all(root.join(".compute")).unwrap(); State { desired_scene: Some("test".to_string()), active_scene: Some("test".to_string()), last_known_good_scene: Some("test".to_string()), ..State::default() } .save(&root) .unwrap(); let error = restart_supervised_model(&root, "fake", Duration::from_millis(50)) .unwrap_err() .to_string(); assert!(error.contains("unowned process")); assert!(governor::port_open(port)); let owned = Proc::capture(pid, 1, Some(port)).unwrap(); proc::stop_owned(&owned).unwrap(); fs::remove_dir_all(root).unwrap(); } /// Two models sharing one port, distinguishable by `health_contains` — the /// brain/brain-laguna shape. Each serves its own directory so the health probe /// can tell which one is actually up. fn same_port_root() -> PathBuf { let unique = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); let root = std::env::temp_dir().join(format!("lumbridge-compute-swap-{unique}")); fs::create_dir_all(root.join("registry")).unwrap(); fs::create_dir_all(root.join("scenes")).unwrap(); let dir_a = root.join("srv-alpha"); let dir_b = root.join("srv-beta"); fs::create_dir_all(&dir_a).unwrap(); fs::create_dir_all(&dir_b).unwrap(); fs::write(dir_a.join("alpha-marker.txt"), "a").unwrap(); fs::write(dir_b.join("beta-marker.txt"), "b").unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); drop(listener); let (a, b) = (dir_a.display(), dir_b.display()); let registry = format!( "apiVersion: lumbridge/v1\nmodels:\n alpha:\n name: Alpha\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: alpha-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{a}']\n beta:\n name: Beta\n footprint_gb: 1\n health: http://localhost:{port}/\n health_contains: beta-marker\n serve:\n kind: exec\n port: {port}\n command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{b}']\n" ); fs::write(root.join("registry/models.yaml"), registry).unwrap(); fs::write( root.join("scenes/a.scene.yaml"), "apiVersion: lumbridge/v1\nmetadata:\n name: a\n version: 1\nmodels: [alpha]\nbudget_gb: 100\n", ) .unwrap(); fs::write( root.join("scenes/b.scene.yaml"), "apiVersion: lumbridge/v1\nmetadata:\n name: b\n version: 1\nmodels: [beta]\nbudget_gb: 100\n", ) .unwrap(); root } #[test] fn same_port_swap_is_allowed_when_the_occupant_is_being_stopped() { let root = same_port_root(); activate(&root, "a", false).unwrap(); assert!(State::load_checked(&root) .unwrap() .procs .contains_key("alpha")); // Regression: this used to fail with "port N is occupied by a different model; // refusing to start 'beta'". The pre-flight port check ran over `plan.start` // without exempting ports released by `plan.stop`, so every same-port swap was // rejected even though stops precede starts. activate(&root, "b", false).unwrap(); let state = State::load_checked(&root).unwrap(); assert_eq!(state.active_scene.as_deref(), Some("b")); assert!(state.procs.contains_key("beta")); assert!(!state.procs.contains_key("alpha")); deactivate(&root).unwrap(); fs::remove_dir_all(root).unwrap(); } #[test] fn already_serving_scene_requires_explicit_adoption() { let (root, port) = fixture_root(); let registry = Registry::load(&root).unwrap(); let model = ®istry.models["fake"]; let pid = proc::spawn(&root, "fake", model).unwrap(); assert!(governor::wait_healthy(model, Duration::from_secs(5))); let error = activate(&root, "test", true).unwrap_err().to_string(); assert!(error.contains("adopt the active Scene")); let owned = Proc::capture(pid, 1, Some(port)).unwrap(); proc::stop_owned(&owned).unwrap(); fs::remove_dir_all(root).unwrap(); } #[test] fn preflight_rejection_preserves_healthy_active_scene() { let (root, _port) = fixture_root(); activate(&root, "test", false).unwrap(); let before = State::load_checked(&root).unwrap(); let pid = before.procs["fake"].pid; fs::write( root.join("scenes/invalid.scene.yaml"), "apiVersion: lumbridge/v1\nmetadata:\n name: invalid\n version: 1\nmodels: [missing]\nbudget_gb: 100\n", ) .unwrap(); assert!(activate(&root, "invalid", false).is_err()); let after = State::load_checked(&root).unwrap(); assert_eq!(after.active_scene.as_deref(), Some("test")); assert_eq!(after.desired_scene.as_deref(), Some("test")); assert_eq!(after.procs["fake"].pid, pid); assert!(after.procs["fake"].owned_alive()); deactivate(&root).unwrap(); fs::remove_dir_all(root).unwrap(); } #[test] fn failed_activation_keeps_healthy_partial_scene_without_rollback_target() { let (root, port) = fixture_root(); fs::write( root.join("scenes/failing.scene.yaml"), "apiVersion: lumbridge/v1\nmetadata:\n name: failing\n version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n", ) .unwrap(); assert!(activate(&root, "failing", false).is_err()); let state = State::load_checked(&root).unwrap(); assert!(state.procs.contains_key("fake")); assert!(!state.procs.contains_key("bad")); assert!(state.active_scene.is_none()); assert!(governor::port_open(port)); deactivate(&root).unwrap(); fs::remove_dir_all(root).unwrap(); } #[test] fn checked_state_load_rejects_malformed_yaml() { let (root, _port) = fixture_root(); fs::create_dir_all(root.join(".compute")).unwrap(); fs::write(root.join(".compute/state.yaml"), "desired_scene: [").unwrap(); assert!(State::load_checked(&root).is_err()); fs::remove_dir_all(root).unwrap(); } #[test] fn state_write_replaces_complete_yaml_atomically() { let (root, _port) = fixture_root(); let state = State { desired_scene: Some("test".to_string()), ..State::default() }; state.save(&root).unwrap(); let loaded = State::load_checked(&root).unwrap(); assert_eq!(loaded.desired_scene.as_deref(), Some("test")); fs::remove_dir_all(root).unwrap(); } }