compute/src/lifecycle.rs

//! Transactional Scene lifecycle and persisted desired-state recovery.

use anyhow::{bail, Context, Result};
use std::collections::{BTreeMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::os::fd::AsRawFd;
use std::path::Path;
use std::time::Duration;

use crate::config::{find_scene, Model, Registry, Scene};
use crate::governor;
use crate::mem;
use crate::proc::{self, Proc, State};

/// What activating a Scene would do, decided before anything is mutated.
///
/// `activate` renders this for the CLI and the MCP server returns it verbatim,
/// so a plan an operator reads and a plan an agent reads can never drift apart.
#[derive(Debug, Clone)]
pub struct Plan {
    pub scene: String,
    pub budget_gb: f64,
    /// Registered models serving outside the target Scene; stopped first.
    pub stop: Vec<String>,
    /// Scene models not yet serving, in the order they would be admitted.
    pub start: Vec<String>,
}

pub struct TransitionLock {
    file: File,
}

impl TransitionLock {
    pub fn acquire(root: &Path) -> Result<Self> {
        let dir = root.join(".compute");
        fs::create_dir_all(&dir)?;
        let path = dir.join("transition.lock");
        let file = OpenOptions::new()
            .create(true)
            .truncate(false)
            .read(true)
            .write(true)
            .open(&path)
            .with_context(|| format!("opening transition lock {}", path.display()))?;
        let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
        if result != 0 {
            return Err(std::io::Error::last_os_error())
                .context("another Scene transition is already running; wait for it to finish");
        }
        Ok(Self { file })
    }
}

impl Drop for TransitionLock {
    fn drop(&mut self) {
        unsafe {
            libc::flock(self.file.as_raw_fd(), libc::LOCK_UN);
        }
    }
}

pub fn adopt(root: &Path, name: &str) -> Result<()> {
    let _lock = TransitionLock::acquire(root)?;
    let registry = Registry::load(root)?;
    let scene = find_scene(root, name)?;
    validate_scene(&registry, &scene)?;

    let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
    let running = governor::running_ids(&registry);
    let extras: Vec<String> = running
        .iter()
        .filter(|id| !target.contains(id.as_str()))
        .cloned()
        .collect();
    if !extras.is_empty() {
        bail!(
            "cannot adopt '{name}': registered model(s) outside the Scene are serving: {}",
            extras.join(", ")
        );
    }

    let previous = State::load_checked(root)?;
    let mut adopted = BTreeMap::new();
    for id in &scene.models {
        let model = &registry.models[id];
        if !governor::is_running(model) {
            bail!("cannot adopt '{name}': exact model '{id}' is not healthy");
        }
        let legacy = previous.procs.get(id).with_context(|| {
            format!(
                "cannot adopt '{name}': no legacy process record for '{id}'; start it through Compute"
            )
        })?;
        let captured = Proc::capture(legacy.pid, legacy.seq, model.serve.port)
            .with_context(|| format!("adopting '{id}' pid {}", legacy.pid))?;
        adopted.insert(id.clone(), captured);
    }

    let mut state = previous;
    state.procs = adopted;
    state.desired_scene = Some(name.to_string());
    state.active_scene = Some(name.to_string());
    state.last_known_good_scene = Some(name.to_string());
    state.transition_scene = None;
    state.last_error = None;
    state.save(root)?;
    println!("adopted exact running Scene '{name}' and captured process identities");
    Ok(())
}

pub fn activate(root: &Path, name: &str, dry_run: bool) -> Result<()> {
    let _lock = TransitionLock::acquire(root)?;
    let prior = State::load_checked(root)?;
    let prior_active = prior.active_scene.clone();

    if dry_run {
        return activate_once(root, name, true);
    }

    match activate_once(root, name, false) {
        Ok(()) => {
            let mut state = State::load_checked(root)?;
            if prior_active.as_deref() != Some(name) {
                if let Some(previous) = prior_active {
                    state.last_known_good_scene = Some(previous);
                }
            }
            state.desired_scene = Some(name.to_string());
            state.active_scene = Some(name.to_string());
            if state.last_known_good_scene.is_none() {
                state.last_known_good_scene = Some(name.to_string());
            }
            state.transition_scene = None;
            state.last_error = None;
            state.save(root)?;
            println!("Scene '{name}' is active and persisted as desired");
            Ok(())
        }
        Err(target_error) => {
            let target_message = format!("activating '{name}' failed: {target_error:#}");
            eprintln!("{target_message}");
            let transition_started =
                State::load_checked(root)?.transition_scene.as_deref() == Some(name);
            // Where a failed transition would roll back to. Absent when the scene being
            // activated is already the active one, which is the ordinary case for
            // "start the models in this scene that are currently down".
            let rollback_target = prior_active.as_deref().filter(|previous| *previous != name);
            // Only tear the box down when there is somewhere to put it back.
            //
            // This used to run whenever a transition had begun, including when the target
            // scene *was* the prior scene — so a refused admission stopped every running
            // model and then skipped the rollback, because the rollback target had been
            // filtered out for being the same scene. Twice on 2026-08-14 that turned one
            // model failing admission into an empty box, taking brain-nemotron and music
            // down with it.
            //
            // With no rollback target, leaving the partially-started scene running is
            // strictly better: those models are healthy and they belong to the scene that
            // was asked for. The refusal is reported either way.
            let cleanup_error = if transition_started && rollback_target.is_some() {
                stop_all_owned(root).err()
            } else {
                None
            };
            let rollback = if transition_started && cleanup_error.is_none() {
                rollback_target
                    .map(|previous| (previous.to_string(), activate_once(root, previous, false)))
            } else {
                None
            };

            let mut state = State::load_checked(root)?;
            state.transition_scene = None;
            state.last_error = Some(target_message.clone());
            match rollback {
                Some((previous, Ok(()))) => {
                    state.desired_scene = Some(previous.clone());
                    state.active_scene = Some(previous.clone());
                    if state.last_known_good_scene.is_none() {
                        state.last_known_good_scene = Some(previous.clone());
                    }
                    state.save(root)?;
                    eprintln!("rolled back to Scene '{previous}'");
                    bail!("{target_message}; rolled back to '{previous}'")
                }
                Some((previous, Err(rollback_error))) => {
                    state.active_scene = None;
                    state.save(root)?;
                    bail!(
                        "{target_message}; rollback to '{previous}' also failed: {rollback_error:#}"
                    )
                }
                None if !transition_started => {
                    state.save(root)?;
                    bail!("{target_message}")
                }
                None if cleanup_error.is_none() => {
                    state.active_scene = None;
                    state.save(root)?;
                    bail!("{target_message}")
                }
                None => {
                    state.active_scene = None;
                    state.save(root)?;
                    let cleanup_error = cleanup_error.expect("guarded by match condition");
                    bail!("{target_message}; cleanup also failed: {cleanup_error:#}")
                }
            }
        }
    }
}

fn stop_all_owned(root: &Path) -> Result<()> {
    let mut state = State::load_checked(root)?;
    let owned: Vec<(String, Proc)> = state
        .procs
        .iter()
        .filter(|(_, process)| process.owned_alive())
        .map(|(id, process)| (id.clone(), process.clone()))
        .collect();
    for (id, process) in owned {
        proc::stop_owned(&process).with_context(|| format!("cleaning up '{id}'"))?;
        state.procs.remove(&id);
        state.save(root)?;
    }
    Ok(())
}

pub fn resume(root: &Path) -> Result<()> {
    let state = State::load_checked(root)?;
    let desired = state
        .desired_scene
        .clone()
        .or_else(|| state.last_known_good_scene.clone())
        .context("no desired Scene is persisted; activate or adopt one first")?;
    let fallback = state.last_known_good_scene.clone();

    match activate(root, &desired, false) {
        Ok(()) => Ok(()),
        Err(desired_error) => {
            let Some(fallback) = fallback.filter(|fallback| fallback != &desired) else {
                return Err(desired_error).context("resuming desired Scene");
            };
            eprintln!(
                "desired Scene '{desired}' did not resume; trying last-known-good '{fallback}'"
            );
            activate(root, &fallback, false).with_context(|| {
                format!(
                    "desired Scene '{desired}' failed ({desired_error:#}) and fallback '{fallback}' failed"
                )
            })
        }
    }
}

/// Restart one explicitly supervised model without disturbing the other models
/// in the desired Scene.
///
/// This is deliberately narrower than `activate`: a resident recovery attempt
/// must not turn one failed voice runtime into a stop-all rollback of an
/// otherwise healthy Scene. The same transition lock, identity ownership,
/// declared/observed admission checks and exact health gate still apply.
/// Returns `true` when a new process was started and `false` when the model
/// recovered before the lock was acquired.
pub fn restart_supervised_model(root: &Path, id: &str, timeout: Duration) -> Result<bool> {
    let _lock = TransitionLock::acquire(root)?;
    let result = restart_supervised_model_locked(root, id, timeout);
    if let Err(error) = &result {
        if let Ok(mut state) = State::load_checked(root) {
            state.active_scene = None;
            state.last_error = Some(format!("supervision could not restart '{id}': {error:#}"));
            let _ = state.save(root);
        }
    }
    result
}

fn restart_supervised_model_locked(root: &Path, id: &str, timeout: Duration) -> Result<bool> {
    let registry = Registry::load(root)?;
    let model = registry
        .models
        .get(id)
        .with_context(|| format!("no registered model '{id}'"))?;
    model
        .supervision
        .as_ref()
        .with_context(|| format!("model '{id}' did not opt into supervision"))?;
    let port = model
        .serve
        .port
        .with_context(|| format!("supervised model '{id}' has no serving port"))?;

    let mut state = State::load_checked(root)?;
    let desired = state
        .desired_scene
        .clone()
        .context("no desired Scene is persisted")?;
    let scene = find_scene(root, &desired)?;
    validate_scene(&registry, &scene)?;
    if !scene.models.iter().any(|candidate| candidate == id) {
        bail!("model '{id}' is not part of desired Scene '{desired}'");
    }

    // The probe that triggered supervision ran before the transition lock. A
    // late request or a runtime finishing its own recovery may have repaired the
    // model in that gap, so re-check before signalling anything.
    if governor::is_running(model) {
        return Ok(false);
    }

    match state.procs.get(id).cloned() {
        Some(process) if process.owned_alive() => {
            // Health is bad but this is still the exact process Compute started.
            // Stop only its process group; no sibling in the Scene is touched.
            proc::stop_owned(&process)?;
        }
        Some(_) if governor::port_open(port) => {
            bail!(
                "port {port} is occupied but the ownership record for '{id}' is stale; refusing to signal it"
            );
        }
        None if governor::port_open(port) => {
            bail!(
                "port {port} is occupied by an unowned process; refusing to replace supervised model '{id}'"
            );
        }
        Some(_) | None => {}
    }

    state.procs.remove(id);
    state.active_scene = None;
    state.last_error = Some(format!("supervision is restarting '{id}'"));
    state.save(root)?;

    // Re-read both ceilings after the old process has released its memory. The
    // registry number prevents paper overcommit; MemAvailable prevents an
    // unregistered workload from turning a nominally valid restart into a wedged
    // unified-memory node.
    let committed = governor::committed_gb(&registry);
    let available = mem::read().ok().map(|memory| memory.available_gb);
    let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
    if !governor::can_admit(model.footprint_gb, committed, budget, available) {
        match available {
            Some(available) => bail!(
                "'{id}' ({:.0} GB) restart refused: {:.0} GB committed against a {:.0} GB budget, \
                 {:.1} GB actually available, {:.0} GB margin required",
                model.footprint_gb,
                committed,
                budget,
                available,
                governor::SAFETY_MARGIN_GB
            ),
            None => bail!(
                "'{id}' restart would exceed the {:.0} GB Scene budget with the safety margin",
                budget
            ),
        }
    }

    let pid = proc::spawn(root, id, model)?;
    state.seq += 1;
    let process = Proc::capture(pid, state.seq, model.serve.port)
        .with_context(|| format!("capturing ownership for supervised '{id}'"))?;
    state.procs.insert(id.to_string(), process.clone());
    state.save(root)?;

    if !governor::wait_healthy(model, timeout) {
        let _ = proc::stop_owned(&process);
        state.procs.remove(id);
        state.last_error = Some(format!(
            "supervised model '{id}' did not report exact health within {} seconds",
            timeout.as_secs()
        ));
        state.save(root)?;
        bail!(
            "supervised model '{id}' did not report exact health within {} seconds",
            timeout.as_secs()
        );
    }

    let unhealthy: Vec<String> = scene
        .models
        .iter()
        .filter(|candidate| !governor::is_running(&registry.models[*candidate]))
        .cloned()
        .collect();
    if unhealthy.is_empty() {
        state.active_scene = Some(desired);
        state.last_error = None;
    } else {
        state.active_scene = None;
        state.last_error = Some(format!(
            "supervised model '{id}' recovered, but desired Scene is still unhealthy: {}",
            unhealthy.join(", ")
        ));
    }
    state.save(root)?;
    Ok(true)
}

pub fn deactivate(root: &Path) -> Result<()> {
    let _lock = TransitionLock::acquire(root)?;
    let mut state = State::load_checked(root)?;
    let ids: Vec<String> = state.procs.keys().cloned().collect();
    for id in ids {
        let process = state.procs[&id].clone();
        if process.owned_alive() {
            println!("  stopping {id} (pid {})", process.pid);
            proc::stop_owned(&process)?;
        }
        state.procs.remove(&id);
        state.save(root)?;
    }
    state.desired_scene = None;
    state.active_scene = None;
    state.transition_scene = None;
    state.last_error = None;
    state.save(root)?;
    println!("all Compute-managed models stopped; no Scene is desired");
    Ok(())
}

/// Diff the target Scene against what is actually serving. Pure: it decides
/// *what* would change, never *whether* the change is allowed.
fn plan_transition(registry: &Registry, scene: &Scene) -> Plan {
    let target: HashSet<&str> = scene.models.iter().map(String::as_str).collect();
    let running = governor::running_ids(registry);
    let stop: Vec<String> = running
        .iter()
        .filter(|id| !target.contains(id.as_str()))
        .cloned()
        .collect();
    let mut start: Vec<String> = scene
        .models
        .iter()
        .filter(|id| !governor::is_running(&registry.models[*id]))
        .cloned()
        .collect();
    let listed = scene
        .activation
        .as_ref()
        .and_then(|activation| activation.order.as_deref())
        .map(|order| order == "listed")
        .unwrap_or(false);
    if !listed {
        start.sort_by(|a, b| {
            registry.models[a]
                .footprint_gb
                .partial_cmp(&registry.models[b].footprint_gb)
                .unwrap()
        });
    }
    Plan {
        scene: scene.metadata.name.clone(),
        budget_gb: scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB),
        stop,
        start,
    }
}

/// Everything that must hold before the first process is signalled. Runs to
/// completion with nothing mutated, so a rejection here leaves the currently
/// active Scene exactly as it was.
fn preflight(registry: &Registry, scene: &Scene, plan: &Plan, state: &State) -> Result<()> {
    for id in &scene.models {
        if governor::is_running(&registry.models[id]) {
            let process = state.procs.get(id).with_context(|| {
                format!(
                    "'{id}' is already serving but is not identity-owned by Compute; adopt the active Scene first"
                )
            })?;
            if !process.owned_alive() {
                bail!(
                    "ownership record for serving model '{id}' is stale; adopt the active Scene first"
                );
            }
        }
    }

    for id in &plan.stop {
        let process = state.procs.get(id).with_context(|| {
            format!(
                "'{id}' is serving but is not identity-owned by Compute; adopt the active Scene before switching"
            )
        })?;
        if !process.owned_alive() {
            bail!("ownership record for serving model '{id}' is stale; refusing to signal its pid");
        }
    }
    for id in &plan.start {
        let model = &registry.models[id];
        if let Some(port) = model.serve.port {
            // A model in `plan.stop` may currently hold this port. Every stop runs before
            // any start, so that is a handoff, not a conflict. Without this exemption every
            // same-port swap is rejected — including brain -> brain-laguna/brain-gemma,
            // which share :8001 by design because the alias downstream agents call must
            // survive a weight swap.
            let freed_by_stop = plan.stop.iter().any(|stopping| {
                registry
                    .models
                    .get(stopping)
                    .and_then(|stopped| stopped.serve.port)
                    == Some(port)
            });
            if !freed_by_stop && governor::port_open(port) && !governor::is_running(model) {
                bail!("port {port} is occupied by a different model; refusing to start '{id}'");
            }
        }
    }
    Ok(())
}

/// `scene activate --dry-run` as data rather than as printed lines: the same
/// validation, the same plan, nothing written. Callers that need the plan
/// programmatically use this instead of scraping stdout.
pub fn plan(root: &Path, name: &str) -> Result<Plan> {
    let _lock = TransitionLock::acquire(root)?;
    let registry = Registry::load(root)?;
    let scene = find_scene(root, name)?;
    validate_scene(&registry, &scene)?;
    let plan = plan_transition(&registry, &scene);
    let mut state = State::load_checked(root)?;
    state.procs.retain(|_, process| process.owned_alive());
    preflight(&registry, &scene, &plan, &state)?;
    Ok(plan)
}

fn activate_once(root: &Path, name: &str, dry_run: bool) -> Result<()> {
    let registry = Registry::load(root)?;
    let scene = find_scene(root, name)?;
    validate_scene(&registry, &scene)?;
    let plan = plan_transition(&registry, &scene);

    println!("activate '{name}' (budget {:.0} GB)", plan.budget_gb);
    println!("  stop : {}", format_ids(&plan.stop));
    println!("  start: {}", format_ids(&plan.start));

    let mut state = State::load_checked(root)?;
    state.procs.retain(|_, process| process.owned_alive());
    preflight(&registry, &scene, &plan, &state)?;

    if dry_run {
        println!("  dry run: validation passed; nothing changed");
        return Ok(());
    }

    state.transition_scene = Some(name.to_string());
    state.save(root)?;

    for id in &plan.stop {
        let process = state.procs[id].clone();
        println!("  stopping {id} (pid {})", process.pid);
        proc::stop_owned(&process)?;
        state.procs.remove(id);
        state.save(root)?;
    }

    let mut committed: f64 = scene
        .models
        .iter()
        .filter(|id| governor::is_running(&registry.models[*id]))
        .map(|id| registry.models[id].footprint_gb)
        .sum();
    let wait_healthy = scene
        .activation
        .as_ref()
        .and_then(|activation| activation.wait_healthy)
        .unwrap_or(true);

    for id in &plan.start {
        let model: &Model = &registry.models[id];
        // Re-read the pool before every start rather than once per activation: each model
        // that comes up consumes real memory, and its true appetite is only knowable after
        // it has allocated. A footprint that was optimistic shows up here, on the next
        // model, instead of taking the box down.
        let available_gb = mem::read().ok().map(|m| m.available_gb);
        if !governor::can_admit(model.footprint_gb, committed, plan.budget_gb, available_gb) {
            match available_gb {
                Some(available) => bail!(
                    "'{id}' ({:.0} GB) refused: {:.0} GB committed against a {:.0} GB budget, \
                     {:.1} GB actually available, {:.0} GB margin required",
                    model.footprint_gb,
                    committed,
                    plan.budget_gb,
                    available,
                    governor::SAFETY_MARGIN_GB
                ),
                None => bail!(
                    "'{id}' would exceed the {:.0} GB Scene budget with the safety margin",
                    plan.budget_gb
                ),
            }
        }
        let pid = proc::spawn(root, id, model)?;
        state.seq += 1;
        let process = Proc::capture(pid, state.seq, model.serve.port)
            .with_context(|| format!("capturing ownership for newly started '{id}'"))?;
        state.procs.insert(id.clone(), process.clone());
        state.save(root)?;
        committed += model.footprint_gb;
        println!("  started {id} (pid {pid}); waiting for exact health");
        if wait_healthy && !governor::wait_healthy(model, health_timeout()) {
            let _ = proc::stop_owned(&process);
            state.procs.remove(id);
            state.save(root)?;
            bail!("'{id}' did not report its exact health marker before timeout");
        }
    }

    let unhealthy: Vec<String> = scene
        .models
        .iter()
        .filter(|id| !governor::is_running(&registry.models[*id]))
        .cloned()
        .collect();
    if !unhealthy.is_empty() {
        bail!(
            "Scene '{name}' is incomplete; exact health failed for: {}",
            unhealthy.join(", ")
        );
    }
    Ok(())
}

fn validate_scene(registry: &Registry, scene: &Scene) -> Result<()> {
    let missing: Vec<&String> = scene
        .models
        .iter()
        .filter(|id| !registry.models.contains_key(*id))
        .collect();
    if !missing.is_empty() {
        bail!(
            "Scene '{}' references unknown model id(s): {}",
            scene.metadata.name,
            missing
                .iter()
                .map(|id| id.as_str())
                .collect::<Vec<_>>()
                .join(", ")
        );
    }
    let budget = scene.budget_gb.unwrap_or(governor::DEFAULT_BUDGET_GB);
    let footprint: f64 = scene
        .models
        .iter()
        .map(|id| registry.models[id].footprint_gb)
        .sum();
    if footprint + governor::SAFETY_MARGIN_GB > budget {
        bail!(
            "Scene '{}' needs {:.1} GB including safety margin, above its {:.1} GB budget",
            scene.metadata.name,
            footprint + governor::SAFETY_MARGIN_GB,
            budget
        );
    }
    Ok(())
}

fn health_timeout() -> Duration {
    Duration::from_secs(if cfg!(test) { 5 } else { 900 })
}

fn format_ids(ids: &[String]) -> String {
    if ids.is_empty() {
        "(none)".to_string()
    } else {
        ids.join(", ")
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::net::TcpListener;
    use std::path::PathBuf;
    use std::time::{SystemTime, UNIX_EPOCH};

    fn fixture_root() -> (PathBuf, u16) {
        let unique = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let root = std::env::temp_dir().join(format!("lumbridge-compute-{unique}"));
        fs::create_dir_all(root.join("registry")).unwrap();
        fs::create_dir_all(root.join("scenes")).unwrap();
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let port = listener.local_addr().unwrap().port();
        drop(listener);
        let bad_listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let bad_port = bad_listener.local_addr().unwrap().port();
        drop(bad_listener);
        let registry = format!(
            "apiVersion: lumbridge/v1\nmodels:\n  fake:\n    name: Fake\n    footprint_gb: 1\n    health: http://localhost:{port}/\n    supervision:\n      restart: always\n    serve:\n      kind: exec\n      port: {port}\n      command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1]\n  bad:\n    name: Bad\n    footprint_gb: 2\n    health: http://localhost:{bad_port}/\n    serve:\n      kind: exec\n      port: {bad_port}\n      command: [/usr/bin/false]\n"
        );
        fs::write(root.join("registry/models.yaml"), registry).unwrap();
        fs::write(
            root.join("scenes/test.scene.yaml"),
            "apiVersion: lumbridge/v1\nmetadata:\n  name: test\n  version: 1\nmodels: [fake]\nbudget_gb: 100\n",
        )
        .unwrap();
        (root, port)
    }

    #[test]
    fn activation_persists_and_deactivation_stops_owned_process() {
        let (root, _port) = fixture_root();
        activate(&root, "test", false).unwrap();
        let state = State::load_checked(&root).unwrap();
        assert_eq!(state.desired_scene.as_deref(), Some("test"));
        assert_eq!(state.active_scene.as_deref(), Some("test"));
        assert!(state.procs["fake"].owned_alive());
        deactivate(&root).unwrap();
        assert!(State::load_checked(&root).unwrap().procs.is_empty());
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn supervision_restarts_only_the_failed_owned_model() {
        let (root, _port) = fixture_root();
        activate(&root, "test", false).unwrap();
        let before = State::load_checked(&root).unwrap();
        let old = before.procs["fake"].clone();
        proc::stop_owned(&old).unwrap();

        assert!(restart_supervised_model(&root, "fake", Duration::from_secs(5)).unwrap());
        let after = State::load_checked(&root).unwrap();
        assert_ne!(after.procs["fake"].pid, old.pid);
        assert!(after.procs["fake"].owned_alive());
        assert_eq!(after.desired_scene.as_deref(), Some("test"));
        assert_eq!(after.active_scene.as_deref(), Some("test"));

        deactivate(&root).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn supervision_refuses_to_replace_an_unowned_process() {
        let (root, port) = fixture_root();
        let registry = Registry::load(&root).unwrap();
        let model = &registry.models["fake"];
        let pid = proc::spawn(&root, "fake", model).unwrap();
        assert!(governor::wait_healthy(model, Duration::from_secs(5)));

        // Make exact health fail while the unowned server continues to hold the
        // port. Supervision must not infer ownership from a port number.
        let registry_path = root.join("registry/models.yaml");
        let contents = fs::read_to_string(&registry_path).unwrap();
        fs::write(
            &registry_path,
            contents.replace(
                &format!("health: http://localhost:{port}/"),
                &format!(
                    "health: http://localhost:{port}/\n    health_contains: marker-that-is-not-served"
                ),
            ),
        )
        .unwrap();
        fs::create_dir_all(root.join(".compute")).unwrap();
        State {
            desired_scene: Some("test".to_string()),
            active_scene: Some("test".to_string()),
            last_known_good_scene: Some("test".to_string()),
            ..State::default()
        }
        .save(&root)
        .unwrap();

        let error = restart_supervised_model(&root, "fake", Duration::from_millis(50))
            .unwrap_err()
            .to_string();
        assert!(error.contains("unowned process"));
        assert!(governor::port_open(port));

        let owned = Proc::capture(pid, 1, Some(port)).unwrap();
        proc::stop_owned(&owned).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    /// Two models sharing one port, distinguishable by `health_contains` — the
    /// brain/brain-laguna shape. Each serves its own directory so the health probe
    /// can tell which one is actually up.
    fn same_port_root() -> PathBuf {
        let unique = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let root = std::env::temp_dir().join(format!("lumbridge-compute-swap-{unique}"));
        fs::create_dir_all(root.join("registry")).unwrap();
        fs::create_dir_all(root.join("scenes")).unwrap();
        let dir_a = root.join("srv-alpha");
        let dir_b = root.join("srv-beta");
        fs::create_dir_all(&dir_a).unwrap();
        fs::create_dir_all(&dir_b).unwrap();
        fs::write(dir_a.join("alpha-marker.txt"), "a").unwrap();
        fs::write(dir_b.join("beta-marker.txt"), "b").unwrap();
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let port = listener.local_addr().unwrap().port();
        drop(listener);
        let (a, b) = (dir_a.display(), dir_b.display());
        let registry = format!(
            "apiVersion: lumbridge/v1\nmodels:\n  alpha:\n    name: Alpha\n    footprint_gb: 1\n    health: http://localhost:{port}/\n    health_contains: alpha-marker\n    serve:\n      kind: exec\n      port: {port}\n      command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{a}']\n  beta:\n    name: Beta\n    footprint_gb: 1\n    health: http://localhost:{port}/\n    health_contains: beta-marker\n    serve:\n      kind: exec\n      port: {port}\n      command: [/usr/bin/python3, -m, http.server, '{port}', --bind, 127.0.0.1, --directory, '{b}']\n"
        );
        fs::write(root.join("registry/models.yaml"), registry).unwrap();
        fs::write(
            root.join("scenes/a.scene.yaml"),
            "apiVersion: lumbridge/v1\nmetadata:\n  name: a\n  version: 1\nmodels: [alpha]\nbudget_gb: 100\n",
        )
        .unwrap();
        fs::write(
            root.join("scenes/b.scene.yaml"),
            "apiVersion: lumbridge/v1\nmetadata:\n  name: b\n  version: 1\nmodels: [beta]\nbudget_gb: 100\n",
        )
        .unwrap();
        root
    }

    #[test]
    fn same_port_swap_is_allowed_when_the_occupant_is_being_stopped() {
        let root = same_port_root();
        activate(&root, "a", false).unwrap();
        assert!(State::load_checked(&root)
            .unwrap()
            .procs
            .contains_key("alpha"));

        // Regression: this used to fail with "port N is occupied by a different model;
        // refusing to start 'beta'". The pre-flight port check ran over `plan.start`
        // without exempting ports released by `plan.stop`, so every same-port swap was
        // rejected even though stops precede starts.
        activate(&root, "b", false).unwrap();

        let state = State::load_checked(&root).unwrap();
        assert_eq!(state.active_scene.as_deref(), Some("b"));
        assert!(state.procs.contains_key("beta"));
        assert!(!state.procs.contains_key("alpha"));

        deactivate(&root).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn already_serving_scene_requires_explicit_adoption() {
        let (root, port) = fixture_root();
        let registry = Registry::load(&root).unwrap();
        let model = &registry.models["fake"];
        let pid = proc::spawn(&root, "fake", model).unwrap();
        assert!(governor::wait_healthy(model, Duration::from_secs(5)));
        let error = activate(&root, "test", true).unwrap_err().to_string();
        assert!(error.contains("adopt the active Scene"));
        let owned = Proc::capture(pid, 1, Some(port)).unwrap();
        proc::stop_owned(&owned).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn preflight_rejection_preserves_healthy_active_scene() {
        let (root, _port) = fixture_root();
        activate(&root, "test", false).unwrap();
        let before = State::load_checked(&root).unwrap();
        let pid = before.procs["fake"].pid;
        fs::write(
            root.join("scenes/invalid.scene.yaml"),
            "apiVersion: lumbridge/v1\nmetadata:\n  name: invalid\n  version: 1\nmodels: [missing]\nbudget_gb: 100\n",
        )
        .unwrap();
        assert!(activate(&root, "invalid", false).is_err());
        let after = State::load_checked(&root).unwrap();
        assert_eq!(after.active_scene.as_deref(), Some("test"));
        assert_eq!(after.desired_scene.as_deref(), Some("test"));
        assert_eq!(after.procs["fake"].pid, pid);
        assert!(after.procs["fake"].owned_alive());
        deactivate(&root).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn failed_activation_keeps_healthy_partial_scene_without_rollback_target() {
        let (root, port) = fixture_root();
        fs::write(
            root.join("scenes/failing.scene.yaml"),
            "apiVersion: lumbridge/v1\nmetadata:\n  name: failing\n  version: 1\nmodels: [fake, bad]\nbudget_gb: 100\n",
        )
        .unwrap();
        assert!(activate(&root, "failing", false).is_err());
        let state = State::load_checked(&root).unwrap();
        assert!(state.procs.contains_key("fake"));
        assert!(!state.procs.contains_key("bad"));
        assert!(state.active_scene.is_none());
        assert!(governor::port_open(port));
        deactivate(&root).unwrap();
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn checked_state_load_rejects_malformed_yaml() {
        let (root, _port) = fixture_root();
        fs::create_dir_all(root.join(".compute")).unwrap();
        fs::write(root.join(".compute/state.yaml"), "desired_scene: [").unwrap();
        assert!(State::load_checked(&root).is_err());
        fs::remove_dir_all(root).unwrap();
    }

    #[test]
    fn state_write_replaces_complete_yaml_atomically() {
        let (root, _port) = fixture_root();
        let state = State {
            desired_scene: Some("test".to_string()),
            ..State::default()
        };
        state.save(&root).unwrap();
        let loaded = State::load_checked(&root).unwrap();
        assert_eq!(loaded.desired_scene.as_deref(), Some("test"));
        fs::remove_dir_all(root).unwrap();
    }
}