Host: fixes from the review, with regression tests
- one live instance per actor: check after the lock; dropping an instance kills it - state snapshots keep target/ and .git/; only source copies skip them - one shutdown deadline; output during shutdown is still routed - deliveries route stdout meanwhile; a stuck stdin kills the instance (no partial lines) and the sender gets undeliverable; control requests are not blocked by a stuck delivery - Host::stop aborts running broker and cargo requests first - compile a version before installing it; roll back if it does not start - unique archive names; a failed archive keeps the actor manageable - commit on close: truncate without create; directory renames - candidates see SAUSAGE_VERSION and the existing state of a new actor - the host publishes PROTOCOL.md at /actor/broker/public/
This commit is contained in:
+328
-1
@@ -1,6 +1,6 @@
|
||||
mod common;
|
||||
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use common::*;
|
||||
use sausage::fs::TMP_PREFIX;
|
||||
@@ -20,6 +20,41 @@ async fn filesystem_world(label: &str) -> (World, Host, mpsc::Receiver<Envelope>
|
||||
(world, host, mailbox)
|
||||
}
|
||||
|
||||
async fn receive_shutdown_messages(
|
||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
||||
id: &str,
|
||||
expected: &str,
|
||||
) {
|
||||
let (mut result, mut reply, mut publish) = (false, false, false);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !result || !reply || !publish {
|
||||
let message = mailbox.recv().await.expect("console mailbox closed");
|
||||
if message.reply_to.as_deref() == Some(id) {
|
||||
assert_eq!(message.body["type"], expected);
|
||||
result = true;
|
||||
} else if message.reply_to.as_deref() == Some("broker/shutdown") {
|
||||
assert_eq!(message.from, "shutdown-output");
|
||||
assert_eq!(message.body["type"], "shutdown-reply");
|
||||
reply = true;
|
||||
} else {
|
||||
assert_eq!(message.from, "shutdown-output");
|
||||
assert_eq!(message.reply_to, None);
|
||||
assert_eq!(message.body["type"], "shutdown-publish");
|
||||
publish = true;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("shutdown output or lifecycle reply did not arrive");
|
||||
}
|
||||
|
||||
fn epoch_second() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routes_replies_publishes_and_reports_undeliverable() {
|
||||
let world = create_world("routing");
|
||||
@@ -639,3 +674,295 @@ async fn rejects_parent_paths() {
|
||||
);
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_spawns_of_one_id_leave_one_live_instance() {
|
||||
let world = create_world("concurrent-spawn");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let source = passing_source(&world, "source");
|
||||
let ids = [
|
||||
spawn_request(&host, "spawn-race", &wasm, &source),
|
||||
spawn_request(&host, "spawn-race", &wasm, &source),
|
||||
];
|
||||
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
|
||||
let spawned = replies
|
||||
.iter()
|
||||
.filter(|r| r.body["type"] == "spawned")
|
||||
.count();
|
||||
let failed = replies.iter().filter(|r| r.body["type"] == "error").count();
|
||||
assert_eq!((spawned, failed), (1, 1));
|
||||
|
||||
let echo = ask_type(
|
||||
&host,
|
||||
&mut mailbox,
|
||||
"spawn-race",
|
||||
json!({"type": "echo", "value": "live"}),
|
||||
"echo",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(echo.body["value"], "live");
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
let list = ask_type(
|
||||
&host,
|
||||
&mut mailbox,
|
||||
"spawn-race",
|
||||
json!({"type": "list", "path": "/data"}),
|
||||
"entries",
|
||||
)
|
||||
.await;
|
||||
let starts = list.body["names"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|n| n.as_str().unwrap().starts_with("start-"))
|
||||
.count();
|
||||
assert_eq!(starts, 1);
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rollback_preserves_target_and_git_files_in_data_and_public() {
|
||||
let world = create_world("snapshot-special-directories");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let source = passing_source(&world, "source");
|
||||
spawn_type(&host, &mut mailbox, "snapshot", &wasm, &source).await;
|
||||
let files = [
|
||||
("/data/target", "/data/target/x", "data-target"),
|
||||
("/data/.git", "/data/.git/x", "data-git"),
|
||||
("/public/target", "/public/target/x", "public-target"),
|
||||
("/public/.git", "/public/.git/x", "public-git"),
|
||||
];
|
||||
for (dir, path, text) in files {
|
||||
let mkdir = json!({"type": "mkdir", "path": dir});
|
||||
ask_type(&host, &mut mailbox, "snapshot", mkdir, "ok").await;
|
||||
let write = json!({"type": "write", "path": path, "text": text});
|
||||
ask_type(&host, &mut mailbox, "snapshot", write, "ok").await;
|
||||
}
|
||||
let upgrade = json!({"type": "upgrade", "actor": "snapshot", "wasm": wasm.display().to_string(), "source": source.display().to_string()});
|
||||
ask_type(&host, &mut mailbox, "broker", upgrade, "upgraded").await;
|
||||
let rollback = json!({"type": "rollback", "actor": "snapshot", "version": 1});
|
||||
ask_type(&host, &mut mailbox, "broker", rollback, "rolled_back").await;
|
||||
for (_, path, text) in files {
|
||||
let read = json!({"type": "read", "path": path});
|
||||
let reply = ask_type(&host, &mut mailbox, "snapshot", read, "content").await;
|
||||
assert_eq!(reply.body["text"], text);
|
||||
}
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_finishes_when_actor_stdin_is_full() {
|
||||
let world = create_world("full-stdin-stop");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let source = passing_source(&world, "source");
|
||||
spawn_type(&host, &mut mailbox, "full-stdin", &wasm, &source).await;
|
||||
let ignore = json!({"type": "ignore-stdin"});
|
||||
ask_type(&host, &mut mailbox, "full-stdin", ignore, "ignoring").await;
|
||||
let large = json!({"type": "echo", "value": "x".repeat(256 * 1024)});
|
||||
host.broker
|
||||
.send("console", Some("full-stdin"), None, None, large);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
let stop = json!({"type": "stop", "actor": "full-stdin"});
|
||||
let id = host
|
||||
.broker
|
||||
.send("console", Some("broker"), None, None, stop);
|
||||
let stopped = tokio::time::timeout(
|
||||
Duration::from_secs(6),
|
||||
receive_reply(&mut mailbox, &id, Duration::from_secs(10)),
|
||||
)
|
||||
.await
|
||||
.expect("stop exceeded the five-second shutdown deadline");
|
||||
assert_eq!(stopped.body["type"], "stopped");
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_reply_and_publish_survive_stop_and_upgrade() {
|
||||
let world = create_world("shutdown-output");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let source = passing_source(&world, "source");
|
||||
spawn_type(&host, &mut mailbox, "shutdown-output", &wasm, &source).await;
|
||||
let subscribe = json!({"type": "subscribe", "actor": "shutdown-output"});
|
||||
ask_type(&host, &mut mailbox, "broker", subscribe, "subscribed").await;
|
||||
|
||||
for operation in ["stop", "upgrade"] {
|
||||
let arm = json!({"type": "arm-shutdown"});
|
||||
ask_type(&host, &mut mailbox, "shutdown-output", arm, "armed").await;
|
||||
let body = match operation {
|
||||
"stop" => json!({"type": "stop", "actor": "shutdown-output"}),
|
||||
_ => {
|
||||
json!({"type": "upgrade", "actor": "shutdown-output", "wasm": wasm.display().to_string(), "source": source.display().to_string()})
|
||||
}
|
||||
};
|
||||
let id = host
|
||||
.broker
|
||||
.send("console", Some("broker"), None, None, body);
|
||||
receive_shutdown_messages(
|
||||
&mut mailbox,
|
||||
&id,
|
||||
if operation == "stop" {
|
||||
"stopped"
|
||||
} else {
|
||||
"upgraded"
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if operation == "stop" {
|
||||
let start = json!({"type": "start", "actor": "shutdown-output"});
|
||||
ask_type(&host, &mut mailbox, "broker", start, "started").await;
|
||||
}
|
||||
}
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rapid_remove_recreate_remove_keeps_actor_manageable() {
|
||||
let world = create_world("rapid-remove");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let source = passing_source(&world, "source");
|
||||
spawn_type(&host, &mut mailbox, "rapid", &wasm, &source).await;
|
||||
let stop = json!({"type": "stop", "actor": "rapid"});
|
||||
ask_type(&host, &mut mailbox, "broker", stop, "stopped").await;
|
||||
|
||||
while SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.subsec_millis()
|
||||
>= 100
|
||||
{
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
let archive_second = epoch_second();
|
||||
let remove = json!({"type": "remove", "actor": "rapid"});
|
||||
let first = ask_type(&host, &mut mailbox, "broker", remove.clone(), "removed").await;
|
||||
assert_eq!(
|
||||
epoch_second(),
|
||||
archive_second,
|
||||
"first remove crossed an epoch-second boundary"
|
||||
);
|
||||
|
||||
// Restore the stopped actor from its archive to avoid a candidate smoke test.
|
||||
sausage::world::copy_tree(
|
||||
first.body["archive"]
|
||||
.as_str()
|
||||
.map(std::path::Path::new)
|
||||
.unwrap(),
|
||||
&world.actor_dir("rapid"),
|
||||
)
|
||||
.unwrap();
|
||||
let scan = json!({"type": "scan"});
|
||||
ask_type(&host, &mut mailbox, "broker", scan.clone(), "scanned").await;
|
||||
let second = ask_type(&host, &mut mailbox, "broker", remove.clone(), "removed").await;
|
||||
assert_eq!(
|
||||
epoch_second(),
|
||||
archive_second,
|
||||
"remove and recreate crossed an epoch-second boundary"
|
||||
);
|
||||
|
||||
sausage::world::copy_tree(
|
||||
second.body["archive"]
|
||||
.as_str()
|
||||
.map(std::path::Path::new)
|
||||
.unwrap(),
|
||||
&world.actor_dir("rapid"),
|
||||
)
|
||||
.unwrap();
|
||||
ask_type(&host, &mut mailbox, "broker", scan, "scanned").await;
|
||||
let start = json!({"type": "start", "actor": "rapid"});
|
||||
ask_type(&host, &mut mailbox, "broker", start, "started").await;
|
||||
let echo = ask_type(
|
||||
&host,
|
||||
&mut mailbox,
|
||||
"rapid",
|
||||
json!({"type": "echo", "value": "managed"}),
|
||||
"echo",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(echo.body["value"], "managed");
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_commits_open_file_after_parent_directory_rename() {
|
||||
let (_, host, mut mailbox) = filesystem_world("rename-open-file").await;
|
||||
let mkdir = json!({"type": "mkdir", "path": "/public/a"});
|
||||
ask_type(&host, &mut mailbox, "writer", mkdir, "ok").await;
|
||||
let open = json!({"type": "open", "path": "/public/a/file", "text": "content"});
|
||||
ask_type(&host, &mut mailbox, "writer", open, "opened").await;
|
||||
let rename = json!({"type": "rename", "path": "/public/a", "to": "/public/b"});
|
||||
ask_type(&host, &mut mailbox, "writer", rename, "ok").await;
|
||||
let close = json!({"type": "close", "path": "/public/a/file"});
|
||||
assert_eq!(
|
||||
ask(&host, &mut mailbox, "writer", close).await.body["closed"],
|
||||
true
|
||||
);
|
||||
let read = json!({"type": "read", "path": "/public/b/file"});
|
||||
assert_eq!(
|
||||
ask_type(&host, &mut mailbox, "writer", read, "content")
|
||||
.await
|
||||
.body["text"],
|
||||
"content"
|
||||
);
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn truncate_without_create_commits_existing_file_on_close() {
|
||||
let (_, host, mut mailbox) = filesystem_world("truncate-without-create").await;
|
||||
let write = json!({"type": "write", "path": "/public/file", "text": "old"});
|
||||
ask_type(&host, &mut mailbox, "writer", write, "ok").await;
|
||||
let open = json!({"type": "open-truncate", "path": "/public/file", "text": "new"});
|
||||
ask_type(&host, &mut mailbox, "writer", open, "opened").await;
|
||||
let path = "/actor/writer/public/file";
|
||||
let read = json!({"type": "read", "path": path});
|
||||
assert_eq!(
|
||||
ask_type(&host, &mut mailbox, "reader", read, "content")
|
||||
.await
|
||||
.body["text"],
|
||||
"old"
|
||||
);
|
||||
let close = json!({"type": "close", "path": "/public/file"});
|
||||
assert_eq!(
|
||||
ask(&host, &mut mailbox, "writer", close).await.body["closed"],
|
||||
true
|
||||
);
|
||||
let read = json!({"type": "read", "path": path});
|
||||
assert_eq!(
|
||||
ask_type(&host, &mut mailbox, "reader", read, "content")
|
||||
.await
|
||||
.body["text"],
|
||||
"new"
|
||||
);
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn candidates_receive_version_and_preexisting_spawn_state() {
|
||||
let world = create_world("candidate-context");
|
||||
let wasm = toolbox_wasm().clone();
|
||||
let (host, mut mailbox) = start(&world).await;
|
||||
let version_step =
|
||||
json!({"send": {"type": "version"}, "expect": {"type": "version", "version": "1"}});
|
||||
let version_source = source_with_test(&world, "version", json!({"steps": [version_step]}));
|
||||
let state_step = json!({"send": {"type": "read", "path": "/data/state.json"}, "expect": {"type": "content", "text": "seeded"}});
|
||||
let state_source = source_with_test(&world, "state", json!({"steps": [state_step]}));
|
||||
let state_dir = world.data_dir("stateful");
|
||||
std::fs::create_dir_all(&state_dir).unwrap();
|
||||
std::fs::write(state_dir.join("state.json"), "seeded").unwrap();
|
||||
let ids = [
|
||||
spawn_request(&host, "versioned", &wasm, &version_source),
|
||||
spawn_request(&host, "stateful", &wasm, &state_source),
|
||||
];
|
||||
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
|
||||
let successful = replies
|
||||
.iter()
|
||||
.filter(|reply| reply.body["type"] == "spawned")
|
||||
.count();
|
||||
assert_eq!(successful, 2);
|
||||
host.stop().await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user