//! Ordered boot calls and the single restart owner for each service (PROTOCOL.md section 8). use crate::{Host, host::State}; use serde_json::{Value, json}; use std::{collections::VecDeque, sync::atomic::Ordering, time::Duration}; use tokio::time::Instant; // Agreed systemd-derived values: /docs/design/plugin-handover/design.md, // https://www.man7.org/linux/man-pages/man5/systemd-system.conf.5.html. const CRASH_LIMIT: usize = 5; const CRASH_WINDOW: Duration = Duration::from_secs(10); pub(crate) struct Service { pub(crate) method: String, pub(crate) params: Value, pub(crate) request: Value, pub(crate) owner: Option, pub(crate) pending: bool, pub(crate) version: Option, pub(crate) enabled: bool, crashes: VecDeque, } impl Service { fn crashed(&mut self, now: Instant) -> bool { while self .crashes .front() .is_some_and(|t| now.duration_since(*t) >= CRASH_WINDOW) { self.crashes.pop_front(); } self.crashes.push_back(now); self.crashes.len() <= CRASH_LIMIT } } impl Host { /// Starts discovery in the background, then waits for each boot service in order. pub(crate) async fn boot(&self) -> anyhow::Result<()> { self.watch_plugins()?; let boot: Vec = serde_yaml_ng::from_slice( &tokio::fs::read(self.0.world.root.join("config/boot.yaml")).await?, )?; for service in boot { self.start_service( service["method"].as_str().unwrap_or("").into(), service["params"].clone(), ) .await; } Ok(()) } /// Register once, before admission or readiness. Restarts reuse this registration. pub(crate) async fn start_service(&self, method: String, params: Value) { let request = json!(uuid::Uuid::now_v7().to_string()); { let mut state = self.0.state.lock().unwrap(); if self.0.shutdown.load(Ordering::SeqCst) { return; } state.services.push(Service { method, params, request: request.clone(), owner: None, enabled: true, pending: true, version: None, crashes: VecDeque::new(), }); } self.run_service(request).await; } /// Await readiness only; process cleanup, not this response, decides whether to restart. async fn run_service(&self, request: Value) { let (method, params) = { let state = self.0.state.lock().unwrap(); if self.0.shutdown.load(Ordering::SeqCst) { return; } let Some(service) = state .services .iter() .find(|s| s.request == request && s.enabled) else { return; }; (service.method.clone(), service.params.clone()) }; self.0 .journal .write("core", "core", "core", &format!("service start {method}")); let result = self .route("core".into(), Some(request.clone()), method, params, true) .await; { let mut state = self.0.state.lock().unwrap(); if let Some(service) = state.services.iter_mut().find(|s| s.request == request) { service.pending = false; } } if let Err(error) = result { // Admission failures have no process lifetime to supervise. A process crash // before readiness is handled by finish just like one after readiness. self.0 .journal .write("core", "core", "core", &format!("service failed {error}")); } } /// Called under the state lock after the old owner and its files/locks are gone. pub(crate) fn service_ended( &self, state: &mut State, pid: &str, status: &str, stopped: bool, superseded: bool, ) { let Some(service) = state .services .iter_mut() .find(|s| s.owner.as_deref() == Some(pid)) else { return; }; service.owner = None; if self.0.shutdown.load(Ordering::SeqCst) || !service.enabled { return; } if !superseded { if stopped { service.enabled = false; return; } if status == "exit 0" { return; } if !service.crashed(Instant::now()) { service.enabled = false; self.0.journal.write( "core", "core", "core", &format!( "service gave up {}: more than {CRASH_LIMIT} crashes within {} seconds", service.method, CRASH_WINDOW.as_secs(), ), ); return; } } self.queue_service(service); } /// Reserve the next attempt before spawning it; upgrades and crashes share this owner. pub(crate) fn queue_service(&self, service: &mut Service) { service.request = json!(uuid::Uuid::now_v7().to_string()); service.pending = true; let request = service.request.clone(); let host = self.clone(); tokio::spawn(async move { host.run_service(request).await; }); } } #[cfg(test)] mod tests;