Host: fixes from the review, with regression tests

- one live instance per actor: check after the lock; dropping an instance kills it
- state snapshots keep target/ and .git/; only source copies skip them
- one shutdown deadline; output during shutdown is still routed
- deliveries route stdout meanwhile; a stuck stdin kills the instance
  (no partial lines) and the sender gets undeliverable; control requests
  are not blocked by a stuck delivery
- Host::stop aborts running broker and cargo requests first
- compile a version before installing it; roll back if it does not start
- unique archive names; a failed archive keeps the actor manageable
- commit on close: truncate without create; directory renames
- candidates see SAUSAGE_VERSION and the existing state of a new actor
- the host publishes PROTOCOL.md at /actor/broker/public/
This commit is contained in:
2026-10-15 12:03:26 +01:00
parent 69cf80a114
commit 9d575b6a62
11 changed files with 811 additions and 220 deletions
+65 -16
View File
@@ -1,19 +1,21 @@
use std::collections::HashMap;
use std::io::{self, BufRead, Write};
use std::time::Duration;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde_json::{Value, json};
fn reply(out: &mut impl Write, message: &Value, body: Value) {
writeln!(
out,
"{}",
json!({"to": message["from"], "reply_to": message["id"], "body": body})
)
.unwrap();
fn send(out: &mut impl Write, message: Value) {
writeln!(out, "{message}").unwrap();
out.flush().unwrap();
}
fn reply(out: &mut impl Write, message: &Value, body: Value) {
send(
out,
json!({"to": message["from"], "reply_to": message["id"], "body": body}),
);
}
fn io_error(error: io::Error) -> Value {
json!({"type": "error", "error": error.to_string()})
}
@@ -21,7 +23,8 @@ fn io_error(error: io::Error) -> Value {
fn main() {
let actor = std::env::var("SAUSAGE_ACTOR").unwrap_or_default();
let version = std::env::var("SAUSAGE_VERSION").unwrap_or_else(|_| "candidate".to_string());
if actor == "crashloop" && version == "2" {
let candidate = std::env::var_os("SAUSAGE_CANDIDATE").is_some();
if actor == "crashloop" && version == "2" && !candidate {
std::fs::write("/data/state", "bad-v2").unwrap();
panic!("version 2 crashes on startup");
}
@@ -29,6 +32,14 @@ fn main() {
let mut open = HashMap::<String, std::fs::File>::new();
let stdin = io::stdin();
let mut out = io::stdout().lock();
let mut emit_shutdown = false;
if actor == "spawn-race" && !candidate {
let stamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
std::fs::write(format!("/data/start-{stamp}"), "started").unwrap();
}
for line in stdin.lock().lines() {
let Ok(line) = line else { break };
let Ok(message) = serde_json::from_str::<Value>(&line) else {
@@ -43,17 +54,21 @@ fn main() {
let _ = std::fs::write("/data/shutdown", "stopping");
std::thread::sleep(Duration::from_millis(800));
}
if emit_shutdown {
send(
&mut out,
json!({"to": "console", "reply_to": message["id"], "body": {"type": "shutdown-reply"}}),
);
send(&mut out, json!({"body": {"type": "shutdown-publish"}}));
}
break;
}
Some("echo") => json!({"type": "echo", "value": body["value"]}),
Some("publish") => {
writeln!(
out,
"{}",
json!({"body": {"type": "published", "value": body["value"]}})
)
.unwrap();
out.flush().unwrap();
send(
&mut out,
json!({"body": {"type": "published", "value": body["value"]}}),
);
continue;
}
Some("version") => json!({"type": "version", "version": version}),
@@ -62,6 +77,15 @@ fn main() {
Ok(()) => json!({"type": "ok"}),
Err(error) => io_error(error),
},
Some("mkdir") => match std::fs::create_dir_all(&path) {
Ok(()) => json!({"type": "ok"}),
Err(error) => io_error(error),
},
Some("rename") => match std::fs::rename(&path, body["to"].as_str().unwrap_or_default())
{
Ok(()) => json!({"type": "ok"}),
Err(error) => io_error(error),
},
Some("open") => {
let result = (|| -> io::Result<()> {
let mut file = std::fs::File::create(&path)?;
@@ -74,6 +98,21 @@ fn main() {
Err(error) => io_error(error),
}
}
Some("open-truncate") => {
let result = (|| -> io::Result<()> {
let mut file = std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.open(&path)?;
file.write_all(text.as_bytes())?;
open.insert(path.clone(), file);
Ok(())
})();
match result {
Ok(()) => json!({"type": "opened"}),
Err(error) => io_error(error),
}
}
Some("append") => {
let result = (|| -> io::Result<()> {
let mut file = std::fs::OpenOptions::new().append(true).open(&path)?;
@@ -87,6 +126,11 @@ fn main() {
}
}
Some("close") => json!({"type": "closed", "closed": open.remove(&path).is_some()}),
Some("arm-shutdown") => {
emit_shutdown = true;
json!({"type": "armed"})
}
Some("ignore-stdin") => json!({"type": "ignoring"}),
Some("read") => match std::fs::read_to_string(&path) {
Ok(text) => json!({"type": "content", "text": text}),
Err(error) => io_error(error),
@@ -107,5 +151,10 @@ fn main() {
_ => json!({"type": "error", "error": "unknown message"}),
};
reply(&mut out, &message, response);
if body["type"] == "ignore-stdin" {
loop {
std::thread::sleep(Duration::from_secs(60));
}
}
}
}
+62 -7
View File
@@ -142,6 +142,28 @@ pub async fn receive_reply(
.unwrap_or_else(|_| panic!("timed out waiting for reply to {id}"))
}
pub async fn receive_replies(
mailbox: &mut mpsc::Receiver<Envelope>,
ids: &[String],
timeout: Duration,
) -> Vec<Envelope> {
let mut replies = Vec::with_capacity(ids.len());
tokio::time::timeout(timeout, async {
while replies.len() < ids.len() {
let message = mailbox.recv().await.expect("console mailbox closed");
assert!(
ids.iter()
.any(|id| message.reply_to.as_deref() == Some(id.as_str())),
"unexpected message while waiting for replies: {message:?}"
);
replies.push(message);
}
})
.await
.expect("timed out waiting for replies");
replies
}
pub async fn ask(
host: &Host,
mailbox: &mut mpsc::Receiver<Envelope>,
@@ -152,16 +174,24 @@ pub async fn ask(
receive_reply(mailbox, &id, REPLY_TIMEOUT).await
}
pub async fn spawn_actor(
pub async fn ask_type(
host: &Host,
mailbox: &mut mpsc::Receiver<Envelope>,
actor: &str,
wasm: &Path,
source: &Path,
body: Value,
expected: &str,
) -> Envelope {
request(
host,
mailbox,
let reply = ask(host, mailbox, actor, body).await;
assert_eq!(reply.body["type"], expected, "unexpected reply: {reply:?}");
reply
}
pub fn spawn_request(host: &Host, actor: &str, wasm: &Path, source: &Path) -> String {
host.broker.send(
"console",
Some("broker"),
None,
None,
json!({
"type": "spawn",
"actor": actor,
@@ -169,7 +199,32 @@ pub async fn spawn_actor(
"source": source.display().to_string()
}),
)
.await
}
pub async fn spawn_actor(
host: &Host,
mailbox: &mut mpsc::Receiver<Envelope>,
actor: &str,
wasm: &Path,
source: &Path,
) -> Envelope {
let id = spawn_request(host, actor, wasm, source);
receive_reply(mailbox, &id, REPLY_TIMEOUT).await
}
pub async fn spawn_type(
host: &Host,
mailbox: &mut mpsc::Receiver<Envelope>,
actor: &str,
wasm: &Path,
source: &Path,
) -> Envelope {
let reply = spawn_actor(host, mailbox, actor, wasm, source).await;
assert_eq!(
reply.body["type"], "spawned",
"unexpected spawn reply: {reply:?}"
);
reply
}
pub async fn upgrade_actor(
+328 -1
View File
@@ -1,6 +1,6 @@
mod common;
use std::time::Duration;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use common::*;
use sausage::fs::TMP_PREFIX;
@@ -20,6 +20,41 @@ async fn filesystem_world(label: &str) -> (World, Host, mpsc::Receiver<Envelope>
(world, host, mailbox)
}
async fn receive_shutdown_messages(
mailbox: &mut mpsc::Receiver<Envelope>,
id: &str,
expected: &str,
) {
let (mut result, mut reply, mut publish) = (false, false, false);
tokio::time::timeout(Duration::from_secs(10), async {
while !result || !reply || !publish {
let message = mailbox.recv().await.expect("console mailbox closed");
if message.reply_to.as_deref() == Some(id) {
assert_eq!(message.body["type"], expected);
result = true;
} else if message.reply_to.as_deref() == Some("broker/shutdown") {
assert_eq!(message.from, "shutdown-output");
assert_eq!(message.body["type"], "shutdown-reply");
reply = true;
} else {
assert_eq!(message.from, "shutdown-output");
assert_eq!(message.reply_to, None);
assert_eq!(message.body["type"], "shutdown-publish");
publish = true;
}
}
})
.await
.expect("shutdown output or lifecycle reply did not arrive");
}
fn epoch_second() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs()
}
#[tokio::test]
async fn routes_replies_publishes_and_reports_undeliverable() {
let world = create_world("routing");
@@ -639,3 +674,295 @@ async fn rejects_parent_paths() {
);
host.stop().await;
}
#[tokio::test]
async fn concurrent_spawns_of_one_id_leave_one_live_instance() {
let world = create_world("concurrent-spawn");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
let ids = [
spawn_request(&host, "spawn-race", &wasm, &source),
spawn_request(&host, "spawn-race", &wasm, &source),
];
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
let spawned = replies
.iter()
.filter(|r| r.body["type"] == "spawned")
.count();
let failed = replies.iter().filter(|r| r.body["type"] == "error").count();
assert_eq!((spawned, failed), (1, 1));
let echo = ask_type(
&host,
&mut mailbox,
"spawn-race",
json!({"type": "echo", "value": "live"}),
"echo",
)
.await;
assert_eq!(echo.body["value"], "live");
tokio::time::sleep(Duration::from_millis(100)).await;
let list = ask_type(
&host,
&mut mailbox,
"spawn-race",
json!({"type": "list", "path": "/data"}),
"entries",
)
.await;
let starts = list.body["names"]
.as_array()
.unwrap()
.iter()
.filter(|n| n.as_str().unwrap().starts_with("start-"))
.count();
assert_eq!(starts, 1);
host.stop().await;
}
#[tokio::test]
async fn rollback_preserves_target_and_git_files_in_data_and_public() {
let world = create_world("snapshot-special-directories");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "snapshot", &wasm, &source).await;
let files = [
("/data/target", "/data/target/x", "data-target"),
("/data/.git", "/data/.git/x", "data-git"),
("/public/target", "/public/target/x", "public-target"),
("/public/.git", "/public/.git/x", "public-git"),
];
for (dir, path, text) in files {
let mkdir = json!({"type": "mkdir", "path": dir});
ask_type(&host, &mut mailbox, "snapshot", mkdir, "ok").await;
let write = json!({"type": "write", "path": path, "text": text});
ask_type(&host, &mut mailbox, "snapshot", write, "ok").await;
}
let upgrade = json!({"type": "upgrade", "actor": "snapshot", "wasm": wasm.display().to_string(), "source": source.display().to_string()});
ask_type(&host, &mut mailbox, "broker", upgrade, "upgraded").await;
let rollback = json!({"type": "rollback", "actor": "snapshot", "version": 1});
ask_type(&host, &mut mailbox, "broker", rollback, "rolled_back").await;
for (_, path, text) in files {
let read = json!({"type": "read", "path": path});
let reply = ask_type(&host, &mut mailbox, "snapshot", read, "content").await;
assert_eq!(reply.body["text"], text);
}
host.stop().await;
}
#[tokio::test]
async fn stop_finishes_when_actor_stdin_is_full() {
let world = create_world("full-stdin-stop");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "full-stdin", &wasm, &source).await;
let ignore = json!({"type": "ignore-stdin"});
ask_type(&host, &mut mailbox, "full-stdin", ignore, "ignoring").await;
let large = json!({"type": "echo", "value": "x".repeat(256 * 1024)});
host.broker
.send("console", Some("full-stdin"), None, None, large);
tokio::time::sleep(Duration::from_millis(100)).await;
let stop = json!({"type": "stop", "actor": "full-stdin"});
let id = host
.broker
.send("console", Some("broker"), None, None, stop);
let stopped = tokio::time::timeout(
Duration::from_secs(6),
receive_reply(&mut mailbox, &id, Duration::from_secs(10)),
)
.await
.expect("stop exceeded the five-second shutdown deadline");
assert_eq!(stopped.body["type"], "stopped");
host.stop().await;
}
#[tokio::test]
async fn shutdown_reply_and_publish_survive_stop_and_upgrade() {
let world = create_world("shutdown-output");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "shutdown-output", &wasm, &source).await;
let subscribe = json!({"type": "subscribe", "actor": "shutdown-output"});
ask_type(&host, &mut mailbox, "broker", subscribe, "subscribed").await;
for operation in ["stop", "upgrade"] {
let arm = json!({"type": "arm-shutdown"});
ask_type(&host, &mut mailbox, "shutdown-output", arm, "armed").await;
let body = match operation {
"stop" => json!({"type": "stop", "actor": "shutdown-output"}),
_ => {
json!({"type": "upgrade", "actor": "shutdown-output", "wasm": wasm.display().to_string(), "source": source.display().to_string()})
}
};
let id = host
.broker
.send("console", Some("broker"), None, None, body);
receive_shutdown_messages(
&mut mailbox,
&id,
if operation == "stop" {
"stopped"
} else {
"upgraded"
},
)
.await;
if operation == "stop" {
let start = json!({"type": "start", "actor": "shutdown-output"});
ask_type(&host, &mut mailbox, "broker", start, "started").await;
}
}
host.stop().await;
}
#[tokio::test]
async fn rapid_remove_recreate_remove_keeps_actor_manageable() {
let world = create_world("rapid-remove");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "rapid", &wasm, &source).await;
let stop = json!({"type": "stop", "actor": "rapid"});
ask_type(&host, &mut mailbox, "broker", stop, "stopped").await;
while SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.subsec_millis()
>= 100
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
let archive_second = epoch_second();
let remove = json!({"type": "remove", "actor": "rapid"});
let first = ask_type(&host, &mut mailbox, "broker", remove.clone(), "removed").await;
assert_eq!(
epoch_second(),
archive_second,
"first remove crossed an epoch-second boundary"
);
// Restore the stopped actor from its archive to avoid a candidate smoke test.
sausage::world::copy_tree(
first.body["archive"]
.as_str()
.map(std::path::Path::new)
.unwrap(),
&world.actor_dir("rapid"),
)
.unwrap();
let scan = json!({"type": "scan"});
ask_type(&host, &mut mailbox, "broker", scan.clone(), "scanned").await;
let second = ask_type(&host, &mut mailbox, "broker", remove.clone(), "removed").await;
assert_eq!(
epoch_second(),
archive_second,
"remove and recreate crossed an epoch-second boundary"
);
sausage::world::copy_tree(
second.body["archive"]
.as_str()
.map(std::path::Path::new)
.unwrap(),
&world.actor_dir("rapid"),
)
.unwrap();
ask_type(&host, &mut mailbox, "broker", scan, "scanned").await;
let start = json!({"type": "start", "actor": "rapid"});
ask_type(&host, &mut mailbox, "broker", start, "started").await;
let echo = ask_type(
&host,
&mut mailbox,
"rapid",
json!({"type": "echo", "value": "managed"}),
"echo",
)
.await;
assert_eq!(echo.body["value"], "managed");
host.stop().await;
}
#[tokio::test]
async fn close_commits_open_file_after_parent_directory_rename() {
let (_, host, mut mailbox) = filesystem_world("rename-open-file").await;
let mkdir = json!({"type": "mkdir", "path": "/public/a"});
ask_type(&host, &mut mailbox, "writer", mkdir, "ok").await;
let open = json!({"type": "open", "path": "/public/a/file", "text": "content"});
ask_type(&host, &mut mailbox, "writer", open, "opened").await;
let rename = json!({"type": "rename", "path": "/public/a", "to": "/public/b"});
ask_type(&host, &mut mailbox, "writer", rename, "ok").await;
let close = json!({"type": "close", "path": "/public/a/file"});
assert_eq!(
ask(&host, &mut mailbox, "writer", close).await.body["closed"],
true
);
let read = json!({"type": "read", "path": "/public/b/file"});
assert_eq!(
ask_type(&host, &mut mailbox, "writer", read, "content")
.await
.body["text"],
"content"
);
host.stop().await;
}
#[tokio::test]
async fn truncate_without_create_commits_existing_file_on_close() {
let (_, host, mut mailbox) = filesystem_world("truncate-without-create").await;
let write = json!({"type": "write", "path": "/public/file", "text": "old"});
ask_type(&host, &mut mailbox, "writer", write, "ok").await;
let open = json!({"type": "open-truncate", "path": "/public/file", "text": "new"});
ask_type(&host, &mut mailbox, "writer", open, "opened").await;
let path = "/actor/writer/public/file";
let read = json!({"type": "read", "path": path});
assert_eq!(
ask_type(&host, &mut mailbox, "reader", read, "content")
.await
.body["text"],
"old"
);
let close = json!({"type": "close", "path": "/public/file"});
assert_eq!(
ask(&host, &mut mailbox, "writer", close).await.body["closed"],
true
);
let read = json!({"type": "read", "path": path});
assert_eq!(
ask_type(&host, &mut mailbox, "reader", read, "content")
.await
.body["text"],
"new"
);
host.stop().await;
}
#[tokio::test]
async fn candidates_receive_version_and_preexisting_spawn_state() {
let world = create_world("candidate-context");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let version_step =
json!({"send": {"type": "version"}, "expect": {"type": "version", "version": "1"}});
let version_source = source_with_test(&world, "version", json!({"steps": [version_step]}));
let state_step = json!({"send": {"type": "read", "path": "/data/state.json"}, "expect": {"type": "content", "text": "seeded"}});
let state_source = source_with_test(&world, "state", json!({"steps": [state_step]}));
let state_dir = world.data_dir("stateful");
std::fs::create_dir_all(&state_dir).unwrap();
std::fs::write(state_dir.join("state.json"), "seeded").unwrap();
let ids = [
spawn_request(&host, "versioned", &wasm, &version_source),
spawn_request(&host, "stateful", &wasm, &state_source),
];
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
let successful = replies
.iter()
.filter(|reply| reply.body["type"] == "spawned")
.count();
assert_eq!(successful, 2);
host.stop().await;
}