//! The supervisor of one WASM actor: it runs the instance, feeds it the //! mailbox, routes its output, restarts it after a crash, and swaps versions. //! //! The broker talks to a supervisor only through `Control` requests. use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{Result, bail}; use serde_json::{Value, json}; use tokio::sync::{mpsc, oneshot}; use wasmtime::component::Component; use crate::broker::{Broker, Envelope, SHUTDOWN_DEADLINE, shutdown_line}; use crate::runtime::{Exit, Instance}; /// A new version that crashes this often within `CRASH_WINDOW` after an /// upgrade is rolled back automatically. const CRASH_LIMIT: usize = 3; const CRASH_WINDOW: Duration = Duration::from_secs(30); pub type Reply = oneshot::Sender>; pub enum Control { /// Swap to `version` (already in `versions/`, compiled as `component`). Upgrade { version: u64, component: Component, requester: String, reply: Reply }, Rollback { version: u64, reply: Reply }, Stop { reply: Reply }, Start { reply: Reply }, Shutdown { reply: Reply }, } /// The time after an upgrade in which crashes count toward a rollback. struct Probation { previous: u64, since: Instant, crashes: usize, requester: String, } pub struct Supervisor { broker: Arc, id: String, version: u64, running: bool, mailbox: mpsc::Receiver, control: mpsc::Receiver, component: Option, instance: Option, started_at: Instant, restarts: u32, restart_at: Option, probation: Option, /// A control request that arrived during a blocked delivery. next_control: Option, } async fn next_line(instance: &mut Option) -> Option { match instance { Some(i) => i.stdout.recv().await, None => std::future::pending().await, } } async fn sleep_until(t: Option) { match t { Some(t) => tokio::time::sleep_until(t.into()).await, None => std::future::pending().await, } } impl Supervisor { pub fn new( broker: Arc, id: String, version: u64, running: bool, mailbox: mpsc::Receiver, control: mpsc::Receiver, ) -> Self { Self { broker, id, version, running, mailbox, control, component: None, instance: None, started_at: Instant::now(), restarts: 0, restart_at: None, probation: None, next_control: None, } } pub async fn run(mut self) { loop { if let Some(ctl) = self.next_control.take() { if self.handle(ctl).await { break; } continue; } if self.running && self.instance.is_none() && self.restart_at.is_none_or(|t| Instant::now() >= t) { self.restart_at = None; let _ = self.start().await; } let has_instance = self.instance.is_some(); tokio::select! { ctl = self.control.recv() => { let Some(ctl) = ctl else { break }; if self.handle(ctl).await { break; } } msg = self.mailbox.recv(), if has_instance => { let Some(msg) = msg else { break }; self.deliver(msg).await; } line = next_line(&mut self.instance) => match line { Some(line) => self.broker.route_line(&self.id, &line), None => { let exit = self.instance.as_mut().unwrap().wait().await; self.instance = None; self.on_exit(exit).await; } }, _ = sleep_until(self.restart_at), if !has_instance && self.running => {} } } } /// Write a message to the instance's stdin, and route its output /// meanwhile. If a control request arrives before the actor reads the /// message, the instance is killed (a partly written line would corrupt /// the next message) and the request runs. async fn deliver(&mut self, msg: Envelope) { let Some(inst) = self.instance.as_mut() else { return }; let (broker, id, line) = (self.broker.clone(), self.id.clone(), msg.to_line()); let stuck = tokio::select! { written = inst.send_line(&line, |l| broker.route_line(&id, &l)) => match written { Ok(()) => return, // The instance is ending. The main loop sees its closed stdout. Err(_) => false, }, Some(ctl) = self.control.recv() => { self.next_control = Some(ctl); true } }; if msg.from != "broker" { let body = json!({"type": "undeliverable", "reason": "the actor stopped", "to": self.id}); self.broker.send("broker", Some(&msg.from), None, Some(&msg.id), body); } if stuck && let Some(inst) = &self.instance { self.broker.log(&self.id, "killed: it did not read stdin when the broker needed it"); inst.kill(); } } /// Start an instance of the current version. If that fails, the actor /// stays stopped until the next start, upgrade, or rollback. async fn start(&mut self) -> Result<()> { let started = self.try_start(); match &started { Ok(()) => self.started_at = Instant::now(), Err(e) => { self.broker.log(&self.id, &format!("cannot start version {}: {e:#}", self.version)); self.running = false; } } started } fn try_start(&mut self) -> Result<()> { if self.component.is_none() { self.component = Some(self.broker.load_version(&self.id, self.version)?); } let cfg = self.broker.instance_config(&self.id, self.version, None); self.instance = Some(self.broker.runtime.start(self.component.as_ref().unwrap(), cfg)?); Ok(()) } /// Stop the instance gracefully. Its last messages still get routed. async fn stop(&mut self) { let Some(inst) = self.instance.take() else { return }; let (broker, id) = (self.broker.clone(), self.id.clone()); let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |line| broker.route_line(&id, &line)).await; self.broker.log(&self.id, &format!("stopped: {exit}")); } async fn on_exit(&mut self, exit: Exit) { self.broker.log(&self.id, &format!("instance ended unexpectedly: {exit}")); if let Some(p) = &mut self.probation && p.since.elapsed() < CRASH_WINDOW { p.crashes += 1; if p.crashes >= CRASH_LIMIT { let (from, to, requester) = (self.version, p.previous, p.requester.clone()); self.broker.log(&self.id, &format!("version {from} crashed {CRASH_LIMIT} times after the upgrade; rolling back to {to}")); let body = match self.rollback(to).await { Ok(_) => json!({"type": "rolled_back", "actor": self.id, "from": from, "to": to, "reason": "crash loop after upgrade", "last_exit": exit.to_string()}), Err(e) => json!({"type": "error", "actor": self.id, "reason": format!("automatic rollback failed: {e:#}")}), }; self.broker.send("broker", Some(&requester), None, None, body); return; } } // Restart after 0.5 s, doubling up to 30 s; a minute of uptime resets it. if self.started_at.elapsed() > Duration::from_secs(60) { self.restarts = 0; } let delay = Duration::from_millis(500 << self.restarts.min(6)).min(Duration::from_secs(30)); self.restarts += 1; self.restart_at = Some(Instant::now() + delay); } /// Make `version` current. It is compiled first (unless `component` is /// given): if that fails, nothing changes. `data/` and `public/` stay as /// they are. A stopped actor stays stopped. async fn install(&mut self, version: u64, component: Option) -> Result<()> { let component = match component { Some(c) => c, None => self.broker.load_version(&self.id, version)?, }; self.stop().await; self.broker.install(&self.id, version, self.running)?; self.version = version; self.component = Some(component); self.restarts = 0; self.restart_at = None; Ok(()) } async fn upgrade(&mut self, version: u64, component: Component, requester: String) -> Result { let previous = self.version; self.install(version, Some(component)).await?; if !self.running { // Installed, not started: main runs and mail is read at the next `start`. return Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous, "running": false})); } self.probation = Some(Probation { previous, since: Instant::now(), crashes: 0, requester }); if let Err(e) = self.start().await { self.rollback(previous).await?; bail!("version {version} did not start ({e:#}); rolled back to {previous}"); } Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous, "running": true})) } async fn rollback(&mut self, version: u64) -> Result { let from = self.version; self.install(version, None).await?; self.probation = None; if self.running { self.start().await?; } Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": version, "running": self.running})) } /// Handle a control request. Returns `true` when the supervisor ends. async fn handle(&mut self, ctl: Control) -> bool { match ctl { Control::Upgrade { version, component, requester, reply } => { let _ = reply.send(self.upgrade(version, component, requester).await); } Control::Rollback { version, reply } => { let _ = reply.send(self.rollback(version).await); } Control::Stop { reply } => { self.running = false; self.stop().await; let r = self.broker.set_running(&self.id, false); let _ = reply.send(r.map(|_| json!({"type": "stopped", "actor": self.id}))); } Control::Start { reply } => { self.running = true; self.restart_at = None; let r = self.broker.set_running(&self.id, true); let _ = reply.send(r.map(|_| json!({"type": "started", "actor": self.id}))); } Control::Shutdown { reply } => { self.stop().await; let _ = reply.send(Ok(json!({"type": "shut_down"}))); return true; } } false } }