sausage/cli/src/services.rs

//! Ordered boot calls and the single restart owner for each service (PROTOCOL.md section 8).
use crate::{Host, host::State};
use serde_json::{Value, json};
use std::{collections::VecDeque, sync::atomic::Ordering, time::Duration};
use tokio::time::Instant;

// Agreed systemd-derived values: /docs/design/plugin-handover/design.md,
// https://www.man7.org/linux/man-pages/man5/systemd-system.conf.5.html.
const CRASH_LIMIT: usize = 5;
const CRASH_WINDOW: Duration = Duration::from_secs(10);

pub(crate) struct Service {
    pub(crate) method: String,
    pub(crate) params: Value,
    pub(crate) request: Value,
    pub(crate) owner: Option<String>,
    pub(crate) pending: bool,
    pub(crate) version: Option<String>,
    pub(crate) enabled: bool,
    crashes: VecDeque<Instant>,
}
impl Service {
    fn crashed(&mut self, now: Instant) -> bool {
        while self
            .crashes
            .front()
            .is_some_and(|t| now.duration_since(*t) >= CRASH_WINDOW)
        {
            self.crashes.pop_front();
        }
        self.crashes.push_back(now);
        self.crashes.len() <= CRASH_LIMIT
    }
}
impl Host {
    /// Starts discovery in the background, then waits for each boot service in order.
    pub(crate) async fn boot(&self) -> anyhow::Result<()> {
        self.watch_plugins()?;
        let boot: Vec<Value> = serde_yaml_ng::from_slice(
            &tokio::fs::read(self.0.world.root.join("config/boot.yaml")).await?,
        )?;
        for service in boot {
            self.start_service(
                service["method"].as_str().unwrap_or("").into(),
                service["params"].clone(),
            )
            .await;
        }
        Ok(())
    }
    /// Register once, before admission or readiness. Restarts reuse this registration.
    pub(crate) async fn start_service(&self, method: String, params: Value) {
        let request = json!(uuid::Uuid::now_v7().to_string());
        {
            let mut state = self.0.state.lock().unwrap();
            if self.0.shutdown.load(Ordering::SeqCst) {
                return;
            }
            state.services.push(Service {
                method,
                params,
                request: request.clone(),
                owner: None,
                enabled: true,
                pending: true,
                version: None,
                crashes: VecDeque::new(),
            });
        }
        self.run_service(request).await;
    }
    /// Await readiness only; process cleanup, not this response, decides whether to restart.
    async fn run_service(&self, request: Value) {
        let (method, params) = {
            let state = self.0.state.lock().unwrap();
            if self.0.shutdown.load(Ordering::SeqCst) {
                return;
            }
            let Some(service) = state
                .services
                .iter()
                .find(|s| s.request == request && s.enabled)
            else {
                return;
            };
            (service.method.clone(), service.params.clone())
        };
        self.0
            .journal
            .write("core", "core", "core", &format!("service start {method}"));
        let result = self
            .route("core".into(), Some(request.clone()), method, params, true)
            .await;
        {
            let mut state = self.0.state.lock().unwrap();
            if let Some(service) = state.services.iter_mut().find(|s| s.request == request) {
                service.pending = false;
            }
        }
        if let Err(error) = result {
            // Admission failures have no process lifetime to supervise. A process crash
            // before readiness is handled by finish just like one after readiness.
            self.0
                .journal
                .write("core", "core", "core", &format!("service failed {error}"));
        }
    }
    /// Called under the state lock after the old owner and its files/locks are gone.
    pub(crate) fn service_ended(
        &self,
        state: &mut State,
        pid: &str,
        status: &str,
        stopped: bool,
        superseded: bool,
    ) {
        let Some(service) = state
            .services
            .iter_mut()
            .find(|s| s.owner.as_deref() == Some(pid))
        else {
            return;
        };
        service.owner = None;
        if self.0.shutdown.load(Ordering::SeqCst) || !service.enabled {
            return;
        }
        if !superseded {
            if stopped {
                service.enabled = false;
                return;
            }
            if status == "exit 0" {
                return;
            }
            if !service.crashed(Instant::now()) {
                service.enabled = false;
                self.0.journal.write(
                    "core",
                    "core",
                    "core",
                    &format!(
                        "service gave up {}: more than {CRASH_LIMIT} crashes within {} seconds",
                        service.method,
                        CRASH_WINDOW.as_secs(),
                    ),
                );
                return;
            }
        }
        self.queue_service(service);
    }
    /// Reserve the next attempt before spawning it; upgrades and crashes share this owner.
    pub(crate) fn queue_service(&self, service: &mut Service) {
        service.request = json!(uuid::Uuid::now_v7().to_string());
        service.pending = true;
        let request = service.request.clone();
        let host = self.clone();
        tokio::spawn(async move {
            host.run_service(request).await;
        });
    }
}

#[cfg(test)]
mod tests;