Merge pull request 'broker: upgrade and rollback keep a stopped actor stopped' (#3) from upgrade-keeps-stopped into main

Reviewed-on: #3

PROTOCOL.md

@@ -125,14 +125,16 @@ the request: the request's `type` in past tense (`spawned`, `upgraded`,
 | `type` | Fields | Effect |
 | --- | --- | --- |
 | `spawn` | `actor`, `wasm`, `source`?, `built_by`? | test, then start a new actor |
-| `upgrade` | `actor`, `wasm`, `source`?, `built_by`? | test, then swap the version |
+| `upgrade` | `actor`, `wasm`, `source`?, `built_by`? | test, then swap the version: a running actor is stopped (normal shutdown) and starts the new version; a stopped actor gets the new version and stays stopped, with its mailbox untouched, until `start`. The answer has `running` |
-| `rollback` | `actor`, `version` | go back to an older version |
+| `rollback` | `actor`, `version` | go back to an older version; like `upgrade`, a stopped actor stays stopped. The answer has `running` |
 | `stop`, `start` | `actor` | stop or start an actor |
 | `remove` | `actor` | stop the actor (normal shutdown), then delete its whole folder `/actor/<id>/` (all versions, `data/`, `public/`, logs); answer `{"type": "removed", "actor"}`. Messages to it are then undeliverable, and the ID can be spawned again. An unknown actor or a native one (`broker`, `console`, `cargo`, `fs`) is an `error` |
 | `subscribe`, `unsubscribe` | `actor` | receive (or not) what `actor` publishes |
 | `tap`, `untap` | | receive (or not) a copy of every message the broker routes |
 | `watch`, `unwatch` | `path` | receive (or not) `fs_changed` for every change of `path` or anything under it; answer `{"type": "watching" \| "unwatched", "path"}` with the path as an `/actor/...` path |
-| `exit` | | answer `exiting`, stop every actor, and end the `sausage` process (a wrapper like `nix run .#loop` starts it again, updated) |
+| `exit` | | answer `exiting`, stop every actor, and end the `sausage` process with exit code 75 (a wrapper like `nix run .#loop` starts it again, updated) |
+
+**Headless.** `sausage run --headless [dir]` (or `sausage --headless`, or `SAUSAGE_HEADLESS=1`) turns the console input off: the host does not read stdin, so end of input (`/dev/null`) does not stop the world. `nix run .#loop -- --headless <world>` passes it through the loop. It runs until SIGTERM or SIGINT (stops every actor, exit code 0) or a broker `exit` request (exit code 75, as above). Without the flag, EOF on stdin or `/quit` ends the host.
 
 Paths are `/actor/...`, or the sender's `/public/...` or `/data/...`.
 
@@ -210,7 +212,8 @@ instance. A version without test files passes.
 
 **Swap.** The broker holds the mailbox, stops the old instance, makes the
 top-level `actor.wasm` and `source/` a copy of the new version, and starts
-it. A rollback does the same with the older version.
+it. A stopped actor is not started: it keeps its mailbox and runs the new
+version at the next `start`. A rollback does the same with the older version.
 
 **Crashes.** The broker restarts a crashed actor after 0.5 s, then doubles
 the delay up to 30 s. A new version that crashes 3 times within 30 seconds

README.md

@@ -48,6 +48,18 @@ From a clone (`git clone https://gitea.surma.technology/surma/sausage`), the
 same commands are `nix run`, `nix run -- init`, `nix run -- run` and
 `nix run .#loop`.
 
+### Headless
+
+`--headless` (or `SAUSAGE_HEADLESS=1`) turns the console input off: the host
+never reads stdin, so end of input does not stop the world. It runs until
+SIGTERM/SIGINT (clean stop, exit code 0) or a broker `exit` request (exit
+code 75, so the loop starts the newest version). Create the world first
+(`init`), then pass the flag through the loop:
+
+```sh
+nix run git+https://gitea.surma.technology/surma/sausage#loop -- --headless my-world
+```
+
 In the console:
 
 ```text

cli/src/broker.rs

@@ -331,12 +331,13 @@ impl Broker {
         Ok((component, tests))
     }
 
-    /// Make `version` the current `actor.wasm`, `source/`, and `actor.json`.
+    /// Make `version` the current `actor.wasm`, `source/`, and `actor.json`,
-    pub fn install(&self, id: &str, version: u64) -> Result<()> {
+    /// and record whether the actor runs.
+    pub fn install(&self, id: &str, version: u64, running: bool) -> Result<()> {
         self.world.install_files(id, version)?;
         self.update_meta(id, |m| {
             m.version = Some(version);
-            m.running = true;
+            m.running = running;
         })
     }
 
@@ -360,7 +361,7 @@ impl Broker {
 
     async fn create_version(self: &Arc<Self>, id: &str, meta: &VersionMeta, wasm: &Path, source: Option<&Path>) -> Result<Value> {
         let (_, tests) = self.try_version(id, meta, wasm, source).await?;
-        self.install(id, meta.version)?;
+        self.install(id, meta.version, true)?;
         self.start_supervisor(id, meta.version, true);
         Ok(json!({"type": "spawned", "actor": id, "version": meta.version, "tests": tests}))
     }

cli/src/console.rs

@@ -3,7 +3,8 @@
 //! - `@<actor> <json>` sends a JSON body. `@<actor> <text>` sends
 //!   `{"type": "text", "text": ...}`.
 //! - A line without `@` goes to `agent` as text.
-//! - `/help`, `/quit`. EOF on stdin also quits.
+//! - `/help`, `/quit`. EOF on stdin also quits (not with `--headless`, which
+//!   never reads stdin).
 
 use std::sync::Arc;
 

cli/src/main.rs

@@ -5,6 +5,7 @@ use anyhow::{Context, Result, bail};
 use clap::{Parser, Subcommand};
 use include_dir::{Dir, include_dir};
 use serde_json::json;
+use tokio::signal::unix::Signal;
 
 use sausage::world::{ActorMeta, World};
 use sausage::{Host, cargo_actor, console};
@@ -17,6 +18,11 @@ static AGENT_SOURCE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../agent");
 #[derive(Parser)]
 #[command(version, about)]
 struct Cli {
+    /// Do not read stdin, so end of input does not stop the world. It runs
+    /// until SIGTERM (or SIGINT), or until an actor sends
+    /// `@broker {"type":"exit"}`. Also set by SAUSAGE_HEADLESS=1.
+    #[arg(long, global = true)]
+    headless: bool,
     #[command(subcommand)]
     command: Option<Cmd>,
 }
@@ -91,13 +97,41 @@ async fn init(dir: PathBuf) -> Result<()> {
     Ok(())
 }
 
-async fn run(dir: PathBuf) -> Result<()> {
+/// Install the SIGTERM and SIGINT handlers now (not when first polled).
+fn termination_signals() -> Result<(Signal, Signal)> {
+    use tokio::signal::unix::{SignalKind, signal};
+    let term = signal(SignalKind::terminate()).context("install SIGTERM handler")?;
+    let int = signal(SignalKind::interrupt()).context("install SIGINT handler")?;
+    Ok((term, int))
+}
+
+/// Resolves on SIGTERM or SIGINT.
+async fn termination((mut term, mut int): (Signal, Signal)) {
+    tokio::select! {
+        _ = term.recv() => {}
+        _ = int.recv() => {}
+    }
+}
+
+async fn run(dir: PathBuf, headless: bool) -> Result<()> {
+    // Register the signal handlers before the host starts, so a signal that
+    // arrives early is never the default (kill) action.
+    let signals = if headless { Some(termination_signals()?) } else { None };
     let host = Host::start(&dir).await?;
     tokio::spawn(console::run_mailbox(host.mailbox(console::ID)));
     let broker = host.broker.clone();
-    let exit = tokio::select! {
+    let exit = match signals {
+        Some(signals) => {
+            println!("Sausage headless: running until SIGTERM or an exit request.");
+            tokio::select! {
+                _ = termination(signals) => false,
+                _ = broker.exit_requested() => true,
+            }
+        }
+        None => tokio::select! {
             _ = console::run_input(host.broker.clone()) => false,
             _ = broker.exit_requested() => true,
+        },
     };
     println!("Stopping actors …");
     host.stop().await;
@@ -113,16 +147,18 @@ async fn run(dir: PathBuf) -> Result<()> {
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    match Cli::parse().command {
+    let cli = Cli::parse();
+    let headless = cli.headless || std::env::var("SAUSAGE_HEADLESS").is_ok_and(|v| !v.is_empty() && v != "0");
+    match cli.command {
         Some(Cmd::Init { dir }) => init(dir).await,
-        Some(Cmd::Run { dir }) => run(dir).await,
+        Some(Cmd::Run { dir }) => run(dir, headless).await,
         None => {
             let dir = PathBuf::from("world");
             if !dir.join("actor").is_dir() {
                 println!("No world in ./world yet, creating it (same as `sausage init`).");
                 init(dir.clone()).await?;
             }
-            run(dir).await
+            run(dir, headless).await
         }
     }
 }

cli/src/supervisor.rs

@@ -220,17 +220,16 @@ impl Supervisor {
 
     /// Make `version` current. It is compiled first (unless `component` is
     /// given): if that fails, nothing changes. `data/` and `public/` stay as
-    /// they are.
+    /// they are. A stopped actor stays stopped.
     async fn install(&mut self, version: u64, component: Option<Component>) -> Result<()> {
         let component = match component {
             Some(c) => c,
             None => self.broker.load_version(&self.id, version)?,
         };
         self.stop().await;
-        self.broker.install(&self.id, version)?;
+        self.broker.install(&self.id, version, self.running)?;
         self.version = version;
         self.component = Some(component);
-        self.running = true;
         self.restarts = 0;
         self.restart_at = None;
         Ok(())
@@ -239,20 +238,26 @@ impl Supervisor {
     async fn upgrade(&mut self, version: u64, component: Component, requester: String) -> Result<Value> {
         let previous = self.version;
         self.install(version, Some(component)).await?;
+        if !self.running {
+            // Installed, not started: main runs and mail is read at the next `start`.
+            return Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous, "running": false}));
+        }
         self.probation = Some(Probation { previous, since: Instant::now(), crashes: 0, requester });
         if let Err(e) = self.start().await {
             self.rollback(previous).await?;
             bail!("version {version} did not start ({e:#}); rolled back to {previous}");
         }
-        Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous}))
+        Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous, "running": true}))
     }
 
     async fn rollback(&mut self, version: u64) -> Result<Value> {
         let from = self.version;
         self.install(version, None).await?;
         self.probation = None;
+        if self.running {
             self.start().await?;
-        Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": version}))
+        }
+        Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": version, "running": self.running}))
     }
 
     /// Handle a control request. Returns `true` when the supervisor ends.

cli/tests/cli.rs

@@ -39,3 +39,47 @@ fn bare_command_without_a_world_starts_init() {
     // It asks before it creates anything.
     assert!(!dir.join("world").exists());
 }
+
+/// Start a headless host on a fresh world with stdin at EOF; returns once it
+/// printed its ready line (printed after the signal handlers are installed).
+fn start_headless(label: &str) -> (std::process::Child, std::io::Lines<std::io::BufReader<std::process::ChildStdout>>) {
+    use std::io::{BufRead, BufReader};
+    let dir = empty_dir(label);
+    let world = dir.join("world");
+    sausage::World::create(&world).unwrap();
+    let mut child = Command::new(env!("CARGO_BIN_EXE_sausage"))
+        .args(["run", "--headless"])
+        .arg(&world)
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .spawn()
+        .unwrap();
+    let mut lines = BufReader::new(child.stdout.take().unwrap()).lines();
+    let line = lines.next().expect("host exited before it started").unwrap();
+    assert!(line.contains("headless"), "unexpected first line: {line}");
+    (child, lines)
+}
+
+/// With stdin at EOF (as soon as the host starts), a headless host stays up.
+#[test]
+fn headless_ignores_stdin_eof() {
+    let (mut child, _lines) = start_headless("headless-eof");
+    std::thread::sleep(std::time::Duration::from_millis(500));
+    let alive = child.try_wait().unwrap().is_none();
+    let _ = child.kill();
+    let _ = child.wait();
+    assert!(alive, "headless host exited on stdin EOF");
+}
+
+/// SIGTERM right after the ready line stops the actors and exits with 0.
+#[test]
+fn headless_stops_cleanly_on_sigterm_right_after_ready() {
+    let (mut child, lines) = start_headless("headless-term");
+    let pid = child.id().to_string();
+    assert!(Command::new("kill").args(["-TERM", &pid]).status().unwrap().success());
+    let rest: Vec<String> = lines.map(|l| l.unwrap()).collect();
+    let status = child.wait().unwrap();
+    assert!(status.success(), "exit status {status:?}, output {rest:?}");
+    assert!(rest.iter().any(|l| l.contains("Stopping actors")), "output {rest:?}");
+}

cli/tests/host.rs

@@ -506,6 +506,50 @@ async fn rolls_back_a_crash_loop_to_the_previous_version_without_restoring_data(
     host.stop().await;
 }
 
+#[tokio::test]
+async fn upgrade_and_rollback_keep_a_stopped_actor_stopped() {
+    let world = create_world("upgrade-stopped");
+    let wasm = toolbox_wasm().clone();
+    let (host, mut mailbox) = start(&world).await;
+    let source = passing_source(&world, "source");
+    spawn_type(&host, &mut mailbox, "sleeper", &wasm, &source).await;
+    let stopped = request(&host, &mut mailbox, json!({"type": "stop", "actor": "sleeper"})).await;
+    assert_eq!(stopped.body["type"], "stopped");
+
+    let upgraded = upgrade_actor(&host, &mut mailbox, "sleeper", &wasm, &source).await;
+    assert_eq!(upgraded.body["type"], "upgraded", "{upgraded:?}");
+    assert_eq!(upgraded.body["version"], 2);
+    assert_eq!(upgraded.body["running"], false);
+    let meta = world.read_meta("sleeper");
+    assert_eq!((meta.version, meta.running), (Some(2), false));
+
+    // Mail waits in the mailbox: the stopped actor reads nothing.
+    let waiting = host.broker.send("console", Some("sleeper"), None, None, json!({"type": "version"}));
+    assert!(
+        tokio::time::timeout(Duration::from_millis(500), receive_reply(&mut mailbox, &waiting, Duration::from_secs(10)))
+            .await
+            .is_err(),
+        "a stopped actor must not answer"
+    );
+
+    let rolled_back = request(&host, &mut mailbox, json!({"type": "rollback", "actor": "sleeper", "version": 1})).await;
+    assert_eq!(rolled_back.body["type"], "rolled_back", "{rolled_back:?}");
+    assert_eq!(rolled_back.body["running"], false);
+    assert!(!world.read_meta("sleeper").running);
+
+    // `start` runs the installed version and delivers the waiting message.
+    let started = request(&host, &mut mailbox, json!({"type": "start", "actor": "sleeper"})).await;
+    assert_eq!(started.body["type"], "started");
+    let answer = receive_reply(&mut mailbox, &waiting, Duration::from_secs(10)).await;
+    assert_eq!(answer.body["version"], "1");
+
+    // A running actor still restarts on the new version.
+    let running = upgrade_actor(&host, &mut mailbox, "sleeper", &wasm, &source).await;
+    assert_eq!(running.body["running"], true);
+    assert_eq!(ask(&host, &mut mailbox, "sleeper", json!({"type": "version"})).await.body["version"], "3");
+    host.stop().await;
+}
+
 #[tokio::test]
 async fn explicit_rollback_keeps_current_data() {
     let world = create_world("explicit-rollback");