Files
sausage/cli/tests/host.rs
T
Sausage Agent 2ff30ecf70
ci / check (push) Successful in 10s
Actor-served files: /actor/<id>/fs/
File operations under /actor/<id>/fs/ become messages from `fs` to <id>
(fs_read, fs_write, fs_mkdir, fs_remove, fs_rename) with a 10 s timeout.
vfs.rs wraps the wasi:filesystem implementation of wasmtime-wasi.
2026-10-16 01:25:11 +00:00

745 lines
26 KiB
Rust

mod common;
use std::time::Duration;
use common::*;
use sausage::{Envelope, Host, World};
use serde_json::{Value, json};
use tokio::sync::mpsc;
async fn filesystem_world(label: &str) -> (World, Host, mpsc::Receiver<Envelope>) {
let world = create_world(label);
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
for actor in ["writer", "reader"] {
let source = passing_source(&world, actor);
let spawned = spawn_actor(&host, &mut mailbox, actor, &wasm, &source).await;
assert_eq!(spawned.body["type"], "spawned");
}
(world, host, mailbox)
}
async fn receive_shutdown_messages(
mailbox: &mut mpsc::Receiver<Envelope>,
id: &str,
expected: &str,
) {
let (mut result, mut reply, mut publish) = (false, false, false);
tokio::time::timeout(Duration::from_secs(10), async {
while !result || !reply || !publish {
let message = mailbox.recv().await.expect("console mailbox closed");
if message.reply_to.as_deref() == Some(id) {
assert_eq!(message.body["type"], expected);
result = true;
} else if message.reply_to.as_deref() == Some("broker/shutdown") {
assert_eq!(message.from, "shutdown-output");
assert_eq!(message.body["type"], "shutdown-reply");
reply = true;
} else {
assert_eq!(message.from, "shutdown-output");
assert_eq!(message.reply_to, None);
assert_eq!(message.body["type"], "shutdown-publish");
publish = true;
}
}
})
.await
.expect("shutdown output or lifecycle reply did not arrive");
}
#[tokio::test]
async fn routes_replies_publishes_and_reports_undeliverable() {
let world = create_world("routing");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let missing_id = host.broker.send(
"console",
Some("missing"),
None,
None,
json!({"type": "hello"}),
);
let missing = receive_reply(&mut mailbox, &missing_id, Duration::from_secs(10)).await;
assert_eq!(missing.body["type"], "undeliverable");
assert_eq!(missing.body["to"], "missing");
let source = passing_source(&world, "router");
let spawned = spawn_actor(&host, &mut mailbox, "router", &wasm, &source).await;
assert_eq!(spawned.body["type"], "spawned");
let echo = ask(
&host,
&mut mailbox,
"router",
json!({"type": "echo", "value": "hello"}),
)
.await;
assert_eq!(echo.from, "router");
assert_eq!(echo.body, json!({"type": "echo", "value": "hello"}));
let subscribed = request(
&host,
&mut mailbox,
json!({"type": "subscribe", "actor": "router"}),
)
.await;
assert_eq!(subscribed.body["type"], "subscribed");
host.broker.send(
"console",
Some("router"),
None,
None,
json!({"type": "publish", "value": "news"}),
);
let published = receive_message(&mut mailbox, Duration::from_secs(10)).await;
assert_eq!(published.from, "router");
assert_eq!(
published.body,
json!({"type": "published", "value": "news"})
);
assert_eq!(published.reply_to, None);
host.stop().await;
}
#[tokio::test]
async fn broker_does_not_answer_messages_with_reply_to() {
let world = create_world("broker-reply-to");
let (host, mut mailbox) = start(&world).await;
let scanned = request(&host, &mut mailbox, json!({"type": "scan"})).await;
assert_eq!(scanned.body["type"], "scanned");
host.broker.send(
"console",
Some("broker"),
None,
Some("upstream-message"),
json!({"type": "scan"}),
);
assert!(
tokio::time::timeout(Duration::from_secs(1), mailbox.recv())
.await
.is_err(),
"broker replied to a message that already had reply_to"
);
host.stop().await;
}
#[tokio::test]
async fn exit_request_is_answered_and_signals_the_host() {
let world = create_world("broker-exit");
let (host, mut mailbox) = start(&world).await;
let exiting = request(&host, &mut mailbox, json!({"type": "exit"})).await;
assert_eq!(exiting.body["type"], "exiting");
tokio::time::timeout(Duration::from_secs(1), host.broker.exit_requested())
.await
.expect("exit request did not reach the host");
host.stop().await;
}
#[tokio::test]
async fn filesystem_permissions_and_public_sharing() {
let (_, host, mut mailbox) = filesystem_world("filesystem-permissions").await;
for path in [
"/actor/reader/public/denied.txt",
"/actor/writer/public/denied.txt",
] {
let denied = ask(
&host,
&mut mailbox,
"writer",
json!({"type": "write", "path": path, "text": "denied"}),
)
.await;
assert_eq!(denied.body["type"], "error", "write to {path} succeeded");
}
let written = ask(
&host,
&mut mailbox,
"writer",
json!({"type": "write", "path": "/public/shared.txt", "text": "public"}),
)
.await;
assert_eq!(written.body["type"], "ok");
let shared = ask(
&host,
&mut mailbox,
"reader",
json!({"type": "read", "path": "/actor/writer/public/shared.txt"}),
)
.await;
assert_eq!(shared.body, json!({"type": "content", "text": "public"}));
host.stop().await;
}
#[tokio::test]
async fn actors_can_read_but_not_write_other_actor_data() {
let (_, host, mut mailbox) = filesystem_world("actor-data-sharing").await;
ask_type(
&host,
&mut mailbox,
"writer",
json!({"type": "write", "path": "/data/x", "text": "shared"}),
"ok",
)
.await;
let shared = ask_type(
&host,
&mut mailbox,
"reader",
json!({"type": "read", "path": "/actor/writer/data/x"}),
"content",
)
.await;
assert_eq!(shared.body["text"], "shared");
ask_type(
&host,
&mut mailbox,
"reader",
json!({"type": "write", "path": "/actor/writer/data/x", "text": "changed"}),
"error",
)
.await;
let unchanged = ask_type(
&host,
&mut mailbox,
"writer",
json!({"type": "read", "path": "/data/x"}),
"content",
)
.await;
assert_eq!(unchanged.body["text"], "shared");
host.stop().await;
}
#[tokio::test]
async fn actors_serve_files_under_fs() {
// "writer" serves its fs/ from memory (toolbox fs_read / fs_write / fs_remove).
let (_, host, mut mailbox) = filesystem_world("actor-fs").await;
let file = "/actor/writer/fs/notes/a.txt";
let write = json!({"type": "write", "path": file, "text": "hello"});
ask_type(&host, &mut mailbox, "reader", write, "ok").await;
// The write reached the owner as one fs_write message.
let direct = json!({"type": "fs_read", "path": "/notes/a.txt"});
let served = ask_type(&host, &mut mailbox, "writer", direct, "fs_file").await;
assert_eq!(served.body["text"], "hello");
let read = json!({"type": "read", "path": file});
assert_eq!(ask_type(&host, &mut mailbox, "reader", read, "content").await.body["text"], "hello");
let list = json!({"type": "list", "path": "/actor/writer/fs/notes"});
assert_eq!(ask_type(&host, &mut mailbox, "reader", list, "entries").await.body["names"], json!(["a.txt"]));
let root = json!({"type": "list", "path": "/actor/writer/fs"});
assert_eq!(ask_type(&host, &mut mailbox, "reader", root, "entries").await.body["names"], json!(["notes"]));
// fs_error and undeliverable become errors of the file operation.
for path in ["/actor/writer/fs/missing.txt", "/actor/nobody/fs/x"] {
ask_type(&host, &mut mailbox, "reader", json!({"type": "read", "path": path}), "error").await;
}
// An actor that does not answer (here: it is busy reading itself) fails after the timeout.
let id = host.broker.send("console", Some("reader"), None, None, json!({"type": "read", "path": "/actor/reader/fs/x"}));
let timed_out = receive_reply(&mut mailbox, &id, Duration::from_secs(20)).await;
assert_eq!(timed_out.body["type"], "error");
host.stop().await;
}
#[tokio::test]
async fn files_are_plain_posix_files() {
let (_, host, mut mailbox) = filesystem_world("posix").await;
// An open file is visible by path, to the writer and to every reader.
let open = json!({"type": "open", "path": "/public/new.txt", "text": "new"});
ask_type(&host, &mut mailbox, "writer", open, "opened").await;
let own = json!({"type": "read", "path": "/public/new.txt"});
assert_eq!(ask_type(&host, &mut mailbox, "writer", own, "content").await.body["text"], "new");
let other = json!({"type": "read", "path": "/actor/writer/public/new.txt"});
assert_eq!(ask_type(&host, &mut mailbox, "reader", other, "content").await.body["text"], "new");
// It can be renamed while open.
let rename = json!({"type": "rename", "path": "/public/new.txt", "to": "/public/moved.txt"});
ask_type(&host, &mut mailbox, "writer", rename, "ok").await;
ask_type(&host, &mut mailbox, "writer", json!({"type": "close", "path": "/public/new.txt"}), "closed").await;
let moved = json!({"type": "read", "path": "/actor/writer/public/moved.txt"});
assert_eq!(ask_type(&host, &mut mailbox, "reader", moved, "content").await.body["text"], "new");
host.stop().await;
}
#[tokio::test]
async fn filesystem_rejects_paths_that_escape_data() {
let (_, host, mut mailbox) = filesystem_world("filesystem-path-escape").await;
let escaped = ask(
&host,
&mut mailbox,
"writer",
json!({"type": "write", "path": "/data/../../..", "text": "outside"}),
)
.await;
assert_eq!(escaped.body["type"], "error");
host.stop().await;
}
#[tokio::test]
async fn upgrades_test_candidates_keep_old_versions_and_queue_messages_in_order() {
let world = create_world("upgrade");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let passing = passing_source(&world, "passing");
let failing = source_with_test(
&world,
"failing",
json!({
"steps": [{
"send": {"type": "echo", "value": "candidate"},
"expect": {"type": "wrong"}
}]
}),
);
assert_eq!(
spawn_actor(&host, &mut mailbox, "queue", &wasm, &passing)
.await
.body["type"],
"spawned"
);
let failed = upgrade_actor(&host, &mut mailbox, "queue", &wasm, &failing).await;
assert_eq!(failed.body["type"], "error");
let old_version = ask(&host, &mut mailbox, "queue", json!({"type": "version"})).await;
assert_eq!(old_version.body["version"], "1");
// A version that fails its tests never enters versions/.
assert!(!world.actor_dir("queue").join("versions/2").exists());
let upgrade_id = host.broker.send(
"console",
Some("broker"),
None,
None,
json!({"type": "upgrade", "actor": "queue", "wasm": wasm.display().to_string(), "source": passing.display().to_string()}),
);
tokio::time::timeout(Duration::from_secs(10), async {
while !world.data_dir("queue").join("shutdown").is_file() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("old actor did not enter shutdown during upgrade");
let mut message_ids = Vec::new();
for n in 1..=3 {
message_ids.push(host.broker.send(
"console",
Some("queue"),
None,
None,
json!({"type": "ordered", "n": n}),
));
}
let mut upgrade_result = None;
let mut actor_replies = Vec::new();
tokio::time::timeout(Duration::from_secs(15), async {
while upgrade_result.is_none() || actor_replies.len() < message_ids.len() {
let envelope = mailbox.recv().await.expect("console mailbox closed");
if envelope.reply_to.as_deref() == Some(upgrade_id.as_str()) {
upgrade_result = Some(envelope);
} else if message_ids
.iter()
.any(|id| envelope.reply_to.as_deref() == Some(id.as_str()))
{
actor_replies.push(envelope);
} else {
panic!("unexpected message during queued upgrade: {envelope:?}");
}
}
})
.await
.expect("upgrade did not deliver queued messages");
assert_eq!(upgrade_result.unwrap().body["type"], "upgraded");
let actual_ids: Vec<String> = actor_replies
.iter()
.map(|message| message.reply_to.clone().unwrap())
.collect();
assert_eq!(actual_ids, message_ids);
for (index, envelope) in actor_replies.iter().enumerate() {
assert_eq!(envelope.from, "queue");
assert_eq!(envelope.body["type"], "ordered");
assert_eq!(envelope.body["n"], json!(index + 1));
assert_eq!(envelope.body["version"], "2");
}
host.stop().await;
}
#[tokio::test]
async fn rolls_back_a_crash_loop_to_the_previous_version_without_restoring_data() {
let world = create_world("crash-loop");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
assert_eq!(
spawn_actor(&host, &mut mailbox, "crashloop", &wasm, &source)
.await
.body["type"],
"spawned"
);
assert_eq!(
ask(
&host,
&mut mailbox,
"crashloop",
json!({"type": "write", "path": "/data/state", "text": "stable"})
)
.await
.body["type"],
"ok"
);
let upgrade_id = host.broker.send(
"console",
Some("broker"),
None,
None,
json!({"type": "upgrade", "actor": "crashloop", "wasm": wasm.display().to_string(), "source": source.display().to_string()}),
);
let mut upgraded: Option<Value> = None;
let mut rolled_back: Option<Value> = None;
tokio::time::timeout(Duration::from_secs(35), async {
while upgraded.is_none() || rolled_back.is_none() {
let envelope = mailbox.recv().await.expect("console mailbox closed");
let kind = envelope.body["type"].as_str().map(str::to_string);
match kind.as_deref() {
Some("upgraded") if envelope.reply_to.as_deref() == Some(upgrade_id.as_str()) => {
upgraded = Some(envelope.body);
}
Some("rolled_back") => rolled_back = Some(envelope.body),
other => panic!("unexpected automatic rollback message: {other:?}, {envelope:?}"),
}
}
})
.await
.expect("crash loop did not trigger automatic rollback");
assert_eq!(upgraded.unwrap()["version"], 2);
let rollback = rolled_back.unwrap();
assert_eq!(rollback["actor"], "crashloop");
assert_eq!(rollback["from"], 2);
assert_eq!(rollback["to"], 1);
assert_eq!(rollback["reason"], "crash loop after upgrade");
let version = ask(&host, &mut mailbox, "crashloop", json!({"type": "version"})).await;
assert_eq!(version.body["version"], "1");
let current = ask(
&host,
&mut mailbox,
"crashloop",
json!({"type": "read", "path": "/data/state"}),
)
.await;
assert_eq!(current.body["text"], "bad-v2");
host.stop().await;
}
#[tokio::test]
async fn explicit_rollback_keeps_current_data() {
let world = create_world("explicit-rollback");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
assert_eq!(
spawn_actor(&host, &mut mailbox, "recover", &wasm, &source)
.await
.body["type"],
"spawned"
);
ask(
&host,
&mut mailbox,
"recover",
json!({"type": "write", "path": "/data/state", "text": "version-one"}),
)
.await;
let upgraded = upgrade_actor(&host, &mut mailbox, "recover", &wasm, &source).await;
assert_eq!(upgraded.body["type"], "upgraded");
ask(
&host,
&mut mailbox,
"recover",
json!({"type": "write", "path": "/data/state", "text": "version-two"}),
)
.await;
let rolled_back = request(
&host,
&mut mailbox,
json!({"type": "rollback", "actor": "recover", "version": 1}),
)
.await;
assert_eq!(rolled_back.body["type"], "rolled_back");
assert_eq!(rolled_back.body["from"], 2);
assert_eq!(rolled_back.body["to"], 1);
let version = ask(&host, &mut mailbox, "recover", json!({"type": "version"})).await;
assert_eq!(version.body["version"], "1");
let current = ask(
&host,
&mut mailbox,
"recover",
json!({"type": "read", "path": "/data/state"}),
)
.await;
assert_eq!(current.body["text"], "version-two");
host.stop().await;
}
#[tokio::test]
async fn cargo_actor_builds_valid_crates_and_reports_compiler_errors() {
let world = create_world("cargo-actor");
let (host, mut mailbox) = start(&world).await;
let valid_source = cargo_source("cargo-valid");
let valid_id = host.broker.send(
"console",
Some("cargo"),
None,
None,
json!({"type": "build", "source": valid_source.display().to_string()}),
);
let built = receive_reply(&mut mailbox, &valid_id, Duration::from_secs(120)).await;
assert_eq!(built.body["type"], "built");
let wasm = built.body["wasm"].as_str().unwrap();
assert!(wasm.ends_with(".wasm"));
assert!(host.world.resolve("console", wasm).unwrap().is_file());
let invalid_source = cargo_source("cargo-fail");
let invalid_id = host.broker.send(
"console",
Some("cargo"),
None,
None,
json!({"type": "build", "source": invalid_source.display().to_string()}),
);
let failed = receive_reply(&mut mailbox, &invalid_id, Duration::from_secs(120)).await;
assert_eq!(failed.body["type"], "build_failed");
assert!(
failed.body["errors"]
.as_str()
.unwrap()
.contains("missing_function")
);
host.stop().await;
}
#[tokio::test]
async fn rejects_invalid_actor_ids() {
let world = create_world("invalid-id");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let invalid = request(
&host,
&mut mailbox,
json!({"type": "spawn", "actor": "bad/actor", "wasm": wasm.display().to_string()}),
)
.await;
assert_eq!(invalid.body["type"], "error");
assert!(
invalid.body["reason"]
.as_str()
.unwrap()
.contains("invalid actor ID")
);
host.stop().await;
}
#[tokio::test]
async fn rejects_parent_paths() {
let world = create_world("parent-path");
let (host, mut mailbox) = start(&world).await;
let parent_path = request(
&host,
&mut mailbox,
json!({"type": "spawn", "actor": "safe", "wasm": "/public/../escape.wasm"}),
)
.await;
assert_eq!(parent_path.body["type"], "error");
assert!(
parent_path.body["reason"]
.as_str()
.unwrap()
.contains("must not contain '..'")
);
host.stop().await;
}
#[tokio::test]
async fn concurrent_spawns_of_one_id_leave_one_live_instance() {
let world = create_world("concurrent-spawn");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
let ids = [
spawn_request(&host, "spawn-race", &wasm, &source),
spawn_request(&host, "spawn-race", &wasm, &source),
];
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
let spawned = replies
.iter()
.filter(|r| r.body["type"] == "spawned")
.count();
let failed = replies.iter().filter(|r| r.body["type"] == "error").count();
assert_eq!((spawned, failed), (1, 1));
let echo = ask_type(
&host,
&mut mailbox,
"spawn-race",
json!({"type": "echo", "value": "live"}),
"echo",
)
.await;
assert_eq!(echo.body["value"], "live");
tokio::time::sleep(Duration::from_millis(100)).await;
let list = ask_type(
&host,
&mut mailbox,
"spawn-race",
json!({"type": "list", "path": "/data"}),
"entries",
)
.await;
let starts = list.body["names"]
.as_array()
.unwrap()
.iter()
.filter(|n| n.as_str().unwrap().starts_with("start-"))
.count();
assert_eq!(starts, 1);
host.stop().await;
}
#[tokio::test]
async fn stop_finishes_when_actor_stdin_is_full() {
let world = create_world("full-stdin-stop");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "full-stdin", &wasm, &source).await;
let ignore = json!({"type": "ignore-stdin"});
ask_type(&host, &mut mailbox, "full-stdin", ignore, "ignoring").await;
let large = json!({"type": "echo", "value": "x".repeat(256 * 1024)});
host.broker
.send("console", Some("full-stdin"), None, None, large);
tokio::time::sleep(Duration::from_millis(100)).await;
let stop = json!({"type": "stop", "actor": "full-stdin"});
let id = host
.broker
.send("console", Some("broker"), None, None, stop);
let stopped = tokio::time::timeout(
Duration::from_secs(6),
receive_reply(&mut mailbox, &id, Duration::from_secs(10)),
)
.await
.expect("stop exceeded the five-second shutdown deadline");
assert_eq!(stopped.body["type"], "stopped");
host.stop().await;
}
#[tokio::test]
async fn shutdown_reply_and_publish_survive_stop_and_upgrade() {
let world = create_world("shutdown-output");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let source = passing_source(&world, "source");
spawn_type(&host, &mut mailbox, "shutdown-output", &wasm, &source).await;
let subscribe = json!({"type": "subscribe", "actor": "shutdown-output"});
ask_type(&host, &mut mailbox, "broker", subscribe, "subscribed").await;
for operation in ["stop", "upgrade"] {
let arm = json!({"type": "arm-shutdown"});
ask_type(&host, &mut mailbox, "shutdown-output", arm, "armed").await;
let body = match operation {
"stop" => json!({"type": "stop", "actor": "shutdown-output"}),
_ => {
json!({"type": "upgrade", "actor": "shutdown-output", "wasm": wasm.display().to_string(), "source": source.display().to_string()})
}
};
let id = host
.broker
.send("console", Some("broker"), None, None, body);
receive_shutdown_messages(
&mut mailbox,
&id,
if operation == "stop" {
"stopped"
} else {
"upgraded"
},
)
.await;
if operation == "stop" {
let start = json!({"type": "start", "actor": "shutdown-output"});
ask_type(&host, &mut mailbox, "broker", start, "started").await;
}
}
host.stop().await;
}
#[tokio::test]
async fn candidates_receive_version_and_preexisting_spawn_state() {
let world = create_world("candidate-context");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let version_step =
json!({"send": {"type": "version"}, "expect": {"type": "version", "version": "1"}});
let version_source = source_with_test(&world, "version", json!({"steps": [version_step]}));
let state_step = json!({"send": {"type": "read", "path": "/data/state.json"}, "expect": {"type": "content", "text": "seeded"}});
let state_source = source_with_test(&world, "state", json!({"steps": [state_step]}));
let state_dir = world.data_dir("stateful");
std::fs::create_dir_all(&state_dir).unwrap();
std::fs::write(state_dir.join("state.json"), "seeded").unwrap();
let ids = [
spawn_request(&host, "versioned", &wasm, &version_source),
spawn_request(&host, "stateful", &wasm, &state_source),
];
let replies = receive_replies(&mut mailbox, &ids, Duration::from_secs(20)).await;
let successful = replies
.iter()
.filter(|reply| reply.body["type"] == "spawned")
.count();
assert_eq!(successful, 2);
host.stop().await;
}
#[tokio::test]
async fn running_actors_have_sockets_and_candidates_do_not() {
// A TCP server on the host that answers each line with "echo: <line>".
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap().to_string();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
tokio::spawn(async move {
use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
let (read, mut write) = stream.into_split();
let mut lines = tokio::io::BufReader::new(read).lines();
while let Ok(Some(line)) = lines.next_line().await {
let _ = write.write_all(format!("echo: {line}\n").as_bytes()).await;
}
});
}
});
let world = create_world("sockets");
let wasm = toolbox_wasm().clone();
let (host, mut mailbox) = start(&world).await;
let tcp = json!({"type": "tcp", "address": address, "text": "hi"});
// The candidate test expects the connection to fail.
let offline = source_with_test(&world, "offline", json!({"steps": [{"send": tcp, "expect": {"type": "error"}}]}));
assert_eq!(spawn_actor(&host, &mut mailbox, "net", &wasm, &offline).await.body["type"], "spawned");
let answer = ask(&host, &mut mailbox, "net", tcp).await;
assert_eq!(answer.body, json!({"type": "tcp", "line": "echo: hi"}));
host.stop().await;
}