Sausage: host, broker, commit-on-close filesystem, cargo actor, console, agent
Imported from the spike branch, with: - a library crate (Host API) for the CLI and the tests - JSON test scenarios for candidates - commit on close only for new or truncated files - a Nix flake: nix run -- init, nix run -- run
This commit is contained in:
@@ -0,0 +1,856 @@
|
||||
//! The broker: message routing, mailboxes, subscriptions, and the lifecycle of
|
||||
//! WASM actors (spawn, upgrade, rollback, stop, start, remove).
|
||||
//!
|
||||
//! The broker accepts only `.wasm` files. It does not know any source language.
|
||||
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use wasmtime::component::Component;
|
||||
|
||||
use crate::candidate;
|
||||
use crate::fs::Mount;
|
||||
use crate::runtime::{Exit, Instance, InstanceConfig, Runtime};
|
||||
use crate::world::{self, ActorRecord, NATIVE, Registry, VersionMeta, World};
|
||||
|
||||
pub const MAILBOX_SIZE: usize = 1000;
|
||||
pub const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(5);
|
||||
/// A new version that crashes this often within `CRASH_WINDOW` after a swap
|
||||
/// is rolled back automatically.
|
||||
const CRASH_LIMIT: usize = 3;
|
||||
const CRASH_WINDOW: Duration = Duration::from_secs(30);
|
||||
const STDIN_WRITE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Envelope {
|
||||
pub id: String,
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
pub reply_to: Option<String>,
|
||||
pub body: Value,
|
||||
}
|
||||
|
||||
impl Envelope {
|
||||
/// The NDJSON line that the recipient reads on stdin.
|
||||
pub fn to_line(&self) -> String {
|
||||
let mut v = json!({"id": self.id, "from": self.from, "body": self.body});
|
||||
if let Some(r) = &self.reply_to {
|
||||
v["reply_to"] = json!(r);
|
||||
}
|
||||
v.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn shutdown_line() -> String {
|
||||
json!({"id": "broker/shutdown", "from": "broker", "body": {"type": "shutdown"}}).to_string()
|
||||
}
|
||||
|
||||
enum Control {
|
||||
Upgrade { version: u64, requester: String, reply: oneshot::Sender<Result<Value>> },
|
||||
Rollback { target: Option<u64>, reply: oneshot::Sender<Result<Value>> },
|
||||
Stop { reply: oneshot::Sender<Result<Value>> },
|
||||
Start { reply: oneshot::Sender<Result<Value>> },
|
||||
Remove { reply: oneshot::Sender<Result<Value>> },
|
||||
Shutdown { reply: oneshot::Sender<Result<Value>> },
|
||||
}
|
||||
|
||||
pub struct Broker {
|
||||
pub world: World,
|
||||
pub runtime: Arc<Runtime>,
|
||||
mailboxes: Mutex<HashMap<String, mpsc::Sender<Envelope>>>,
|
||||
controls: Mutex<HashMap<String, mpsc::Sender<Control>>>,
|
||||
registry: Mutex<Registry>,
|
||||
op_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
|
||||
seq: AtomicU64,
|
||||
}
|
||||
|
||||
impl Broker {
|
||||
pub fn new(world: World, runtime: Arc<Runtime>) -> Result<Arc<Self>> {
|
||||
let registry = world.load_registry()?;
|
||||
Ok(Arc::new(Self {
|
||||
world,
|
||||
runtime,
|
||||
mailboxes: Mutex::new(HashMap::new()),
|
||||
controls: Mutex::new(HashMap::new()),
|
||||
registry: Mutex::new(registry),
|
||||
op_locks: Mutex::new(HashMap::new()),
|
||||
seq: AtomicU64::new(1),
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn registry(&self) -> Registry {
|
||||
self.registry.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn update_registry(&self, f: impl FnOnce(&mut Registry)) -> Result<()> {
|
||||
let mut reg = self.registry.lock().unwrap();
|
||||
f(&mut reg);
|
||||
self.world.save_registry(®)
|
||||
}
|
||||
|
||||
/// Create the mailbox of an actor. Messages wait here while the actor is
|
||||
/// down, during a swap, or while it restarts.
|
||||
pub fn open_mailbox(&self, id: &str) -> mpsc::Receiver<Envelope> {
|
||||
let (tx, rx) = mpsc::channel(MAILBOX_SIZE);
|
||||
self.mailboxes.lock().unwrap().insert(id.to_string(), tx);
|
||||
rx
|
||||
}
|
||||
|
||||
fn next_id(&self) -> String {
|
||||
format!("m-{}", self.seq.fetch_add(1, Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Send a message from a native actor. `to: None` publishes to the
|
||||
/// subscribers of `from`. Returns the message ID.
|
||||
pub fn send(&self, from: &str, to: Option<&str>, id: Option<&str>, reply_to: Option<&str>, body: Value) -> String {
|
||||
let id = match id {
|
||||
Some(own) => format!("{from}/{own}"),
|
||||
None => self.next_id(),
|
||||
};
|
||||
let recipients: Vec<String> = match to {
|
||||
Some(to) => vec![to.to_string()],
|
||||
None => self
|
||||
.registry
|
||||
.lock()
|
||||
.unwrap()
|
||||
.subscriptions
|
||||
.get(from)
|
||||
.map(|s| s.iter().cloned().collect())
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
for to in recipients {
|
||||
self.deliver(Envelope {
|
||||
id: id.clone(),
|
||||
from: from.to_string(),
|
||||
to,
|
||||
reply_to: reply_to.map(str::to_string),
|
||||
body: body.clone(),
|
||||
});
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Route one stdout line of a WASM actor.
|
||||
pub fn route_line(&self, from: &str, line: &str) {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
return;
|
||||
}
|
||||
let parsed: Option<Value> = serde_json::from_str(line).ok();
|
||||
let Some(Value::Object(obj)) = parsed else {
|
||||
self.log(from, &format!("stdout line is not a JSON object, ignored: {}", truncate(line, 500)));
|
||||
return;
|
||||
};
|
||||
let Some(body) = obj.get("body").cloned() else {
|
||||
self.log(from, &format!("message without \"body\", ignored: {}", truncate(line, 500)));
|
||||
return;
|
||||
};
|
||||
let to = obj.get("to").and_then(Value::as_str);
|
||||
let own_id = obj.get("id").and_then(|v| match v {
|
||||
Value::String(s) => Some(s.clone()),
|
||||
Value::Number(n) => Some(n.to_string()),
|
||||
_ => None,
|
||||
});
|
||||
let reply_to = obj.get("reply_to").and_then(Value::as_str);
|
||||
self.send(from, to, own_id.as_deref(), reply_to, body);
|
||||
}
|
||||
|
||||
fn deliver(&self, env: Envelope) {
|
||||
let tx = self.mailboxes.lock().unwrap().get(&env.to).cloned();
|
||||
let error = match tx {
|
||||
None => Some(format!("no actor with ID {}", env.to)),
|
||||
Some(tx) => match tx.try_send(env.clone()) {
|
||||
Ok(()) => None,
|
||||
Err(mpsc::error::TrySendError::Full(_)) => Some(format!("mailbox of {} is full", env.to)),
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Some(format!("actor {} is gone", env.to)),
|
||||
},
|
||||
};
|
||||
if let Some(reason) = error
|
||||
&& env.from != "broker"
|
||||
&& env.from != "test"
|
||||
{
|
||||
self.send(
|
||||
"broker",
|
||||
Some(&env.from),
|
||||
None,
|
||||
Some(&env.id),
|
||||
json!({"type": "undeliverable", "reason": reason, "to": env.to}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a host note to the stderr log of an actor.
|
||||
pub fn log(&self, id: &str, text: &str) {
|
||||
use std::io::Write;
|
||||
let path = self.world.actor_dir(id).join("stderr.log");
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) {
|
||||
let _ = writeln!(f, "[sausage {}] {text}", world::now());
|
||||
}
|
||||
}
|
||||
|
||||
fn op_lock(&self, id: &str) -> Arc<tokio::sync::Mutex<()>> {
|
||||
self.op_locks.lock().unwrap().entry(id.to_string()).or_default().clone()
|
||||
}
|
||||
|
||||
/// Mounts of a running actor.
|
||||
pub fn mounts(&self, id: &str) -> Vec<Mount> {
|
||||
vec![
|
||||
Mount { guest: "/actor".into(), host: self.world.actors_root(), writable: false },
|
||||
Mount { guest: "/public".into(), host: self.world.public_dir(id), writable: true },
|
||||
Mount { guest: "/data".into(), host: self.world.data_dir(id), writable: true },
|
||||
]
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Lifecycle
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
/// Start the supervisors of all registered WASM actors.
|
||||
pub fn start_all(self: &Arc<Self>) {
|
||||
let reg = self.registry();
|
||||
for (id, rec) in reg.actors {
|
||||
self.start_supervisor(&id, rec.version, rec.running);
|
||||
}
|
||||
}
|
||||
|
||||
fn start_supervisor(self: &Arc<Self>, id: &str, version: u64, running: bool) {
|
||||
let mailbox = self.open_mailbox(id);
|
||||
let (ctl_tx, ctl_rx) = mpsc::channel(16);
|
||||
self.controls.lock().unwrap().insert(id.to_string(), ctl_tx);
|
||||
let sv = Supervisor::new(self.clone(), id.to_string(), version, running, mailbox, ctl_rx);
|
||||
tokio::spawn(sv.run());
|
||||
}
|
||||
|
||||
async fn control(&self, id: &str, make: impl FnOnce(oneshot::Sender<Result<Value>>) -> Control) -> Result<Value> {
|
||||
let tx = self.controls.lock().unwrap().get(id).cloned().ok_or_else(|| anyhow!("no WASM actor with ID {id}"))?;
|
||||
let (reply, rx) = oneshot::channel();
|
||||
tx.send(make(reply)).await.map_err(|_| anyhow!("supervisor of {id} is gone"))?;
|
||||
rx.await.map_err(|_| anyhow!("supervisor of {id} did not answer"))?
|
||||
}
|
||||
|
||||
/// Stop every actor gracefully (host shutdown). The registry keeps their
|
||||
/// `running` flags, so the next `sausage run` starts them again.
|
||||
pub async fn shutdown_all(&self) {
|
||||
let ids: Vec<String> = self.controls.lock().unwrap().keys().cloned().collect();
|
||||
let mut waits = Vec::new();
|
||||
for id in ids {
|
||||
waits.push(async move {
|
||||
let _ = self.control(&id, |reply| Control::Shutdown { reply }).await;
|
||||
});
|
||||
}
|
||||
futures_join_all(waits).await;
|
||||
}
|
||||
|
||||
/// Store a new version: copy the `.wasm` file and the source folder into
|
||||
/// the version store. Returns the version directory and the component.
|
||||
fn store_version(&self, id: &str, version: u64, wasm: &Path, source: Option<&Path>, meta: &VersionMeta) -> Result<(PathBuf, Component)> {
|
||||
let dir = self.world.version_dir(id, version);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
std::fs::copy(wasm, dir.join("actor.wasm")).with_context(|| format!("copy {}", wasm.display()))?;
|
||||
match source {
|
||||
Some(src) => world::copy_tree(src, &dir.join("source"))?,
|
||||
None => std::fs::create_dir_all(dir.join("source"))?,
|
||||
}
|
||||
world::write_atomic(&dir.join("meta.json"), serde_json::to_string_pretty(meta)?.as_bytes())?;
|
||||
let component = self.runtime.load(&dir.join("actor.wasm"))?;
|
||||
Ok((dir, component))
|
||||
}
|
||||
|
||||
/// Create a new actor from a `.wasm` file. With `start`, also start its
|
||||
/// supervisor (in a running host). Without `start` (during `init`), only
|
||||
/// register it.
|
||||
pub async fn create_actor(
|
||||
self: &Arc<Self>,
|
||||
id: &str,
|
||||
wasm: &Path,
|
||||
source: Option<&Path>,
|
||||
meta: VersionMeta,
|
||||
start: bool,
|
||||
) -> Result<Value> {
|
||||
if !world::valid_id(id) {
|
||||
bail!("invalid actor ID {id:?}: use a-z, 0-9, '-' and '_'");
|
||||
}
|
||||
if NATIVE.contains(&id) || self.registry().actors.contains_key(id) {
|
||||
bail!("an actor with ID {id} already exists");
|
||||
}
|
||||
let lock = self.op_lock(id);
|
||||
let _guard = lock.lock().await;
|
||||
let (dir, component) = self.store_version(id, 1, wasm, source, &meta)?;
|
||||
let report = candidate::test(self, id, &component, &dir.join("source"), None, None).await;
|
||||
let report = match report {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
let _ = std::fs::remove_dir_all(self.world.versions_dir(id));
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
self.world.ensure_actor_dirs(id)?;
|
||||
world::replace_tree(&dir.join("source"), &self.world.source_dir(id))?;
|
||||
self.update_registry(|r| {
|
||||
r.actors.insert(id.to_string(), ActorRecord { version: 1, next_version: 2, running: true });
|
||||
})?;
|
||||
write_status(&self.world, id, &json!({"id": id, "version": 1, "state": "created"}));
|
||||
if start {
|
||||
self.start_supervisor(id, 1, true);
|
||||
}
|
||||
Ok(json!({"type": "spawned", "actor": id, "version": 1, "tests": report}))
|
||||
}
|
||||
|
||||
pub async fn upgrade_actor(self: &Arc<Self>, id: &str, wasm: &Path, source: Option<&Path>, mut meta: VersionMeta) -> Result<Value> {
|
||||
let lock = self.op_lock(id);
|
||||
let _guard = lock.lock().await;
|
||||
let rec = self.registry().actors.get(id).cloned().ok_or_else(|| anyhow!("no WASM actor with ID {id}"))?;
|
||||
let version = rec.next_version;
|
||||
self.update_registry(|r| {
|
||||
if let Some(a) = r.actors.get_mut(id) {
|
||||
a.next_version = version + 1;
|
||||
}
|
||||
})?;
|
||||
meta.version = version;
|
||||
let (dir, component) = self.store_version(id, version, wasm, source, &meta)?;
|
||||
let report = candidate::test(
|
||||
self,
|
||||
id,
|
||||
&component,
|
||||
&dir.join("source"),
|
||||
Some(&self.world.data_dir(id)),
|
||||
Some(&self.world.public_dir(id)),
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = report {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
return Err(e);
|
||||
}
|
||||
let mut result = self
|
||||
.control(id, |reply| Control::Upgrade { version, requester: meta.requested_by.clone(), reply })
|
||||
.await?;
|
||||
result["tests"] = report.unwrap();
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Handle one message to `broker`.
|
||||
async fn handle(self: Arc<Self>, env: Envelope) {
|
||||
let result = self.clone().command(&env).await;
|
||||
let body = match result {
|
||||
Ok(v) => v,
|
||||
Err(e) => json!({"type": "error", "request": env.body.get("type"), "reason": format!("{e:#}")}),
|
||||
};
|
||||
self.send("broker", Some(&env.from), None, Some(&env.id), body);
|
||||
}
|
||||
|
||||
async fn command(self: Arc<Self>, env: &Envelope) -> Result<Value> {
|
||||
let b = &env.body;
|
||||
let ty = b.get("type").and_then(Value::as_str).unwrap_or("");
|
||||
let actor = || -> Result<String> {
|
||||
b.get("actor").and_then(Value::as_str).map(str::to_string).ok_or_else(|| anyhow!("missing \"actor\""))
|
||||
};
|
||||
let meta = |version: u64| -> Result<(PathBuf, Option<PathBuf>, VersionMeta)> {
|
||||
let wasm_s = b.get("wasm").and_then(Value::as_str).ok_or_else(|| anyhow!("missing \"wasm\""))?;
|
||||
let wasm = self.world.resolve(&env.from, wasm_s)?;
|
||||
if !wasm.is_file() {
|
||||
bail!("{wasm_s} is not a file");
|
||||
}
|
||||
let source_s = b.get("source").and_then(Value::as_str);
|
||||
let source = source_s.map(|s| self.world.resolve(&env.from, s)).transpose()?;
|
||||
if let Some(s) = &source
|
||||
&& !s.is_dir()
|
||||
{
|
||||
bail!("{} is not a folder", source_s.unwrap());
|
||||
}
|
||||
let m = VersionMeta {
|
||||
version,
|
||||
wasm: wasm_s.to_string(),
|
||||
source: source_s.map(str::to_string),
|
||||
built_by: b.get("built_by").and_then(Value::as_str).map(str::to_string),
|
||||
requested_by: env.from.clone(),
|
||||
created_at: world::now(),
|
||||
};
|
||||
Ok((wasm, source, m))
|
||||
};
|
||||
match ty {
|
||||
"spawn" => {
|
||||
let id = actor()?;
|
||||
let (wasm, source, m) = meta(1)?;
|
||||
self.create_actor(&id, &wasm, source.as_deref(), m, true).await
|
||||
}
|
||||
"upgrade" => {
|
||||
let id = actor()?;
|
||||
let (wasm, source, m) = meta(0)?;
|
||||
self.upgrade_actor(&id, &wasm, source.as_deref(), m).await
|
||||
}
|
||||
"rollback" => {
|
||||
let id = actor()?;
|
||||
let target = b.get("version").and_then(Value::as_u64);
|
||||
let lock = self.op_lock(&id);
|
||||
let _g = lock.lock().await;
|
||||
self.control(&id, |reply| Control::Rollback { target, reply }).await
|
||||
}
|
||||
"stop" => self.control(&actor()?, |reply| Control::Stop { reply }).await,
|
||||
"start" => self.control(&actor()?, |reply| Control::Start { reply }).await,
|
||||
"remove" => {
|
||||
let id = actor()?;
|
||||
let lock = self.op_lock(&id);
|
||||
let _g = lock.lock().await;
|
||||
let r = self.control(&id, |reply| Control::Remove { reply }).await?;
|
||||
self.mailboxes.lock().unwrap().remove(&id);
|
||||
self.controls.lock().unwrap().remove(&id);
|
||||
self.update_registry(|reg| {
|
||||
reg.actors.remove(&id);
|
||||
reg.subscriptions.remove(&id);
|
||||
for subs in reg.subscriptions.values_mut() {
|
||||
subs.remove(&id);
|
||||
}
|
||||
})?;
|
||||
Ok(r)
|
||||
}
|
||||
"subscribe" | "unsubscribe" => {
|
||||
let publisher = actor()?;
|
||||
let sub = env.from.clone();
|
||||
let on = ty == "subscribe";
|
||||
self.update_registry(|reg| {
|
||||
let set = reg.subscriptions.entry(publisher.clone()).or_insert_with(BTreeSet::new);
|
||||
if on {
|
||||
set.insert(sub.clone());
|
||||
} else {
|
||||
set.remove(&sub);
|
||||
}
|
||||
})?;
|
||||
Ok(json!({"type": format!("{ty}d"), "actor": publisher}))
|
||||
}
|
||||
"list" => {
|
||||
let reg = self.registry();
|
||||
let mut actors: Vec<Value> = NATIVE.iter().map(|id| json!({"id": id, "kind": "native"})).collect();
|
||||
for (id, rec) in reg.actors {
|
||||
let status = std::fs::read_to_string(self.world.actor_dir(&id).join("status.json"))
|
||||
.ok()
|
||||
.and_then(|s| serde_json::from_str::<Value>(&s).ok());
|
||||
actors.push(json!({
|
||||
"id": id, "kind": "wasm", "version": rec.version, "running": rec.running,
|
||||
"state": status.as_ref().and_then(|s| s.get("state")).cloned(),
|
||||
}));
|
||||
}
|
||||
Ok(json!({"type": "actors", "actors": actors, "subscriptions": reg.subscriptions}))
|
||||
}
|
||||
"shutdown" => Ok(json!({"type": "ignored"})),
|
||||
other => bail!("unknown broker request {other:?}. Known: spawn, upgrade, rollback, stop, start, remove, subscribe, unsubscribe, list"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The native `broker` actor.
|
||||
pub async fn run_actor(self: Arc<Self>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||
while let Some(env) = mailbox.recv().await {
|
||||
tokio::spawn(self.clone().handle(env));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn futures_join_all<F: Future<Output = ()>>(futs: Vec<F>) {
|
||||
let mut set = Vec::new();
|
||||
for f in futs {
|
||||
set.push(Box::pin(f));
|
||||
}
|
||||
for f in set {
|
||||
f.await;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
return s.to_string();
|
||||
}
|
||||
let mut end = max;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
format!("{}… ({} bytes)", &s[..end], s.len())
|
||||
}
|
||||
|
||||
fn write_status(world: &World, id: &str, status: &Value) {
|
||||
let mut s = status.clone();
|
||||
s["updated_at"] = json!(world::now());
|
||||
let _ = world::write_atomic(
|
||||
&world.actor_dir(id).join("status.json"),
|
||||
serde_json::to_string_pretty(&s).unwrap_or_default().as_bytes(),
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Supervisor: owns one WASM actor and its instance.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
struct Supervisor {
|
||||
broker: Arc<Broker>,
|
||||
id: String,
|
||||
version: u64,
|
||||
desired_running: bool,
|
||||
mailbox: mpsc::Receiver<Envelope>,
|
||||
control: mpsc::Receiver<Control>,
|
||||
component: Option<Component>,
|
||||
instance: Option<Instance>,
|
||||
state: &'static str,
|
||||
started_at: Instant,
|
||||
restarts: u32,
|
||||
restart_at: Option<Instant>,
|
||||
last_exit: Option<String>,
|
||||
last_delivered: Option<Envelope>,
|
||||
/// The message that the actor got just before it crashed. The broker
|
||||
/// delivers it again after the next upgrade.
|
||||
parked: Option<Envelope>,
|
||||
redeliver: Option<Envelope>,
|
||||
previous: Option<u64>,
|
||||
swapped_at: Option<Instant>,
|
||||
crashes_since_swap: usize,
|
||||
upgrade_requester: Option<String>,
|
||||
}
|
||||
|
||||
async fn recv_line(instance: &mut Option<Instance>) -> Option<String> {
|
||||
match instance {
|
||||
Some(i) => i.stdout.recv().await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn sleep_until_opt(t: Option<Instant>) {
|
||||
match t {
|
||||
Some(t) => tokio::time::sleep_until(t.into()).await,
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
}
|
||||
|
||||
impl Supervisor {
|
||||
fn new(
|
||||
broker: Arc<Broker>,
|
||||
id: String,
|
||||
version: u64,
|
||||
running: bool,
|
||||
mailbox: mpsc::Receiver<Envelope>,
|
||||
control: mpsc::Receiver<Control>,
|
||||
) -> Self {
|
||||
Self {
|
||||
broker,
|
||||
id,
|
||||
version,
|
||||
desired_running: running,
|
||||
mailbox,
|
||||
control,
|
||||
component: None,
|
||||
instance: None,
|
||||
state: if running { "starting" } else { "stopped" },
|
||||
started_at: Instant::now(),
|
||||
restarts: 0,
|
||||
restart_at: None,
|
||||
last_exit: None,
|
||||
last_delivered: None,
|
||||
parked: None,
|
||||
redeliver: None,
|
||||
previous: None,
|
||||
swapped_at: None,
|
||||
crashes_since_swap: 0,
|
||||
upgrade_requester: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn world(&self) -> &World {
|
||||
&self.broker.world
|
||||
}
|
||||
|
||||
fn status(&self) {
|
||||
write_status(
|
||||
self.world(),
|
||||
&self.id,
|
||||
&json!({
|
||||
"id": self.id,
|
||||
"version": self.version,
|
||||
"state": self.state,
|
||||
"restarts": self.restarts,
|
||||
"last_exit": self.last_exit,
|
||||
"parked_message": self.parked.as_ref().map(|m| json!({"id": m.id, "from": m.from, "body": m.body})),
|
||||
"previous_version": self.previous,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async fn run(mut self) {
|
||||
self.status();
|
||||
loop {
|
||||
if self.desired_running && self.instance.is_none() && self.restart_at.is_none_or(|t| Instant::now() >= t) {
|
||||
self.restart_at = None;
|
||||
self.start_instance().await;
|
||||
}
|
||||
let has_instance = self.instance.is_some();
|
||||
let waiting = !has_instance && self.desired_running;
|
||||
tokio::select! {
|
||||
ctl = self.control.recv() => {
|
||||
let Some(ctl) = ctl else { break };
|
||||
if self.handle_control(ctl).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
msg = self.mailbox.recv(), if has_instance => {
|
||||
let Some(msg) = msg else { break };
|
||||
self.deliver(msg).await;
|
||||
}
|
||||
line = recv_line(&mut self.instance) => {
|
||||
match line {
|
||||
Some(line) => self.broker.route_line(&self.id, &line),
|
||||
None => {
|
||||
let exit = self.instance.as_mut().unwrap().wait().await;
|
||||
self.instance = None;
|
||||
self.on_exit(exit).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = sleep_until_opt(self.restart_at), if waiting => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn deliver(&mut self, msg: Envelope) {
|
||||
let Some(inst) = self.instance.as_mut() else { return };
|
||||
let line = msg.to_line();
|
||||
match tokio::time::timeout(STDIN_WRITE_TIMEOUT, inst.send_line(&line)).await {
|
||||
Ok(Ok(())) => self.last_delivered = Some(msg),
|
||||
Ok(Err(_)) => {} // The instance is ending. The exit path handles it.
|
||||
Err(_) => {
|
||||
let note = format!("actor did not read stdin for {STDIN_WRITE_TIMEOUT:?}; message {} dropped", msg.id);
|
||||
self.broker.log(&self.id, ¬e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_instance(&mut self) {
|
||||
if self.component.is_none() {
|
||||
let path = self.world().version_dir(&self.id, self.version).join("actor.wasm");
|
||||
let t = Instant::now();
|
||||
match self.broker.runtime.load(&path) {
|
||||
Ok(c) => {
|
||||
self.broker.log(&self.id, &format!("compiled version {} in {} ms", self.version, t.elapsed().as_millis()));
|
||||
self.component = Some(c);
|
||||
}
|
||||
Err(e) => {
|
||||
self.broker.log(&self.id, &format!("cannot load version {}: {e:#}", self.version));
|
||||
self.state = "broken";
|
||||
self.desired_running = false;
|
||||
self.status();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
let cfg = InstanceConfig {
|
||||
mounts: self.broker.mounts(&self.id),
|
||||
allow_http: true,
|
||||
stderr_log: self.world().actor_dir(&self.id).join("stderr.log"),
|
||||
env: vec![
|
||||
("SAUSAGE_ACTOR".into(), self.id.clone()),
|
||||
("SAUSAGE_VERSION".into(), self.version.to_string()),
|
||||
],
|
||||
};
|
||||
match self.broker.runtime.start(self.component.as_ref().unwrap(), cfg) {
|
||||
Ok(inst) => {
|
||||
self.instance = Some(inst);
|
||||
self.state = "running";
|
||||
self.started_at = Instant::now();
|
||||
self.status();
|
||||
if let Some(msg) = self.redeliver.take() {
|
||||
self.broker.log(&self.id, &format!("delivering parked message {} again", msg.id));
|
||||
self.deliver(msg).await;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
self.broker.log(&self.id, &format!("cannot start: {e:#}"));
|
||||
self.state = "broken";
|
||||
self.desired_running = false;
|
||||
self.status();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_instance(&mut self) -> Option<Exit> {
|
||||
let inst = self.instance.take()?;
|
||||
let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
|
||||
self.broker.log(&self.id, &format!("stopped: {exit}"));
|
||||
self.last_exit = Some(exit.to_string());
|
||||
Some(exit)
|
||||
}
|
||||
|
||||
async fn on_exit(&mut self, exit: Exit) {
|
||||
self.broker.log(&self.id, &format!("instance ended unexpectedly: {exit}"));
|
||||
self.last_exit = Some(exit.to_string());
|
||||
if !exit.is_clean()
|
||||
&& let Some(m) = self.last_delivered.take()
|
||||
{
|
||||
self.parked = Some(m);
|
||||
}
|
||||
if self.started_at.elapsed() > Duration::from_secs(60) {
|
||||
self.restarts = 0;
|
||||
}
|
||||
if let (Some(prev), Some(at)) = (self.previous, self.swapped_at)
|
||||
&& at.elapsed() < CRASH_WINDOW
|
||||
{
|
||||
self.crashes_since_swap += 1;
|
||||
if self.crashes_since_swap >= CRASH_LIMIT {
|
||||
let from = self.version;
|
||||
self.broker.log(&self.id, &format!("version {from} crashed {CRASH_LIMIT} times after the upgrade; rolling back to {prev}"));
|
||||
let result = self.rollback(Some(prev)).await;
|
||||
let body = match result {
|
||||
Ok(_) => json!({"type": "rolled_back", "actor": self.id, "from": from, "to": prev, "reason": "crash loop after upgrade", "last_exit": exit.to_string()}),
|
||||
Err(e) => json!({"type": "error", "actor": self.id, "reason": format!("automatic rollback failed: {e:#}")}),
|
||||
};
|
||||
if let Some(r) = self.upgrade_requester.clone() {
|
||||
self.broker.send("broker", Some(&r), None, None, body);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
let delay = Duration::from_millis(500 * 2u64.pow(self.restarts.min(6))).min(Duration::from_secs(30));
|
||||
self.restarts += 1;
|
||||
self.restart_at = Some(Instant::now() + delay);
|
||||
self.state = "restarting";
|
||||
self.status();
|
||||
}
|
||||
|
||||
fn snapshot_current(&self) -> Result<()> {
|
||||
let snap = self.world().version_dir(&self.id, self.version).join("snapshot");
|
||||
world::replace_tree(&self.world().data_dir(&self.id), &snap.join("data"))?;
|
||||
world::replace_tree(&self.world().public_dir(&self.id), &snap.join("public"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn install(&mut self, version: u64) -> Result<()> {
|
||||
let src = self.world().version_dir(&self.id, version).join("source");
|
||||
world::replace_tree(&src, &self.world().source_dir(&self.id))?;
|
||||
self.version = version;
|
||||
self.component = None;
|
||||
let id = self.id.clone();
|
||||
self.broker.update_registry(|r| {
|
||||
if let Some(a) = r.actors.get_mut(&id) {
|
||||
a.version = version;
|
||||
a.running = true;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn upgrade(&mut self, version: u64, requester: String) -> Result<Value> {
|
||||
self.stop_instance().await;
|
||||
self.snapshot_current()?;
|
||||
let previous = self.version;
|
||||
self.install(version)?;
|
||||
self.previous = Some(previous);
|
||||
self.swapped_at = Some(Instant::now());
|
||||
self.crashes_since_swap = 0;
|
||||
self.restarts = 0;
|
||||
self.restart_at = None;
|
||||
self.upgrade_requester = Some(requester);
|
||||
self.redeliver = self.parked.take();
|
||||
self.desired_running = true;
|
||||
self.start_instance().await;
|
||||
Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous}))
|
||||
}
|
||||
|
||||
async fn rollback(&mut self, target: Option<u64>) -> Result<Value> {
|
||||
let target = match target {
|
||||
Some(t) => t,
|
||||
None => (1..self.version)
|
||||
.rev()
|
||||
.find(|v| self.world().version_dir(&self.id, *v).join("actor.wasm").is_file())
|
||||
.ok_or_else(|| anyhow!("{} has no version before {}", self.id, self.version))?,
|
||||
};
|
||||
let dir = self.world().version_dir(&self.id, target);
|
||||
if !dir.join("actor.wasm").is_file() {
|
||||
bail!("{} has no version {target}", self.id);
|
||||
}
|
||||
self.stop_instance().await;
|
||||
self.snapshot_current()?;
|
||||
let from = self.version;
|
||||
let snap = dir.join("snapshot");
|
||||
if snap.is_dir() {
|
||||
world::replace_tree(&snap.join("data"), &self.world().data_dir(&self.id))?;
|
||||
world::replace_tree(&snap.join("public"), &self.world().public_dir(&self.id))?;
|
||||
}
|
||||
self.install(target)?;
|
||||
self.previous = None;
|
||||
self.swapped_at = None;
|
||||
self.restarts = 0;
|
||||
self.restart_at = None;
|
||||
self.desired_running = true;
|
||||
self.start_instance().await;
|
||||
Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": target, "restored_state": snap.is_dir()}))
|
||||
}
|
||||
|
||||
/// Returns `true` when the supervisor must end.
|
||||
async fn handle_control(&mut self, ctl: Control) -> bool {
|
||||
match ctl {
|
||||
Control::Upgrade { version, requester, reply } => {
|
||||
let r = self.upgrade(version, requester).await;
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Control::Rollback { target, reply } => {
|
||||
let r = self.rollback(target).await;
|
||||
let _ = reply.send(r);
|
||||
}
|
||||
Control::Stop { reply } => {
|
||||
self.desired_running = false;
|
||||
self.restart_at = None;
|
||||
self.stop_instance().await;
|
||||
self.state = "stopped";
|
||||
let id = self.id.clone();
|
||||
let r = self.broker.update_registry(|r| {
|
||||
if let Some(a) = r.actors.get_mut(&id) {
|
||||
a.running = false;
|
||||
}
|
||||
});
|
||||
let _ = reply.send(r.map(|_| json!({"type": "stopped", "actor": self.id})));
|
||||
}
|
||||
Control::Start { reply } => {
|
||||
self.desired_running = true;
|
||||
self.restart_at = None;
|
||||
let id = self.id.clone();
|
||||
let r = self.broker.update_registry(|r| {
|
||||
if let Some(a) = r.actors.get_mut(&id) {
|
||||
a.running = true;
|
||||
}
|
||||
});
|
||||
let _ = reply.send(r.map(|_| json!({"type": "started", "actor": self.id})));
|
||||
}
|
||||
Control::Remove { reply } => {
|
||||
self.desired_running = false;
|
||||
self.stop_instance().await;
|
||||
let r = self.archive();
|
||||
let _ = reply.send(r);
|
||||
return true;
|
||||
}
|
||||
Control::Shutdown { reply } => {
|
||||
self.stop_instance().await;
|
||||
self.state = "host stopped";
|
||||
self.status();
|
||||
let _ = reply.send(Ok(json!({"type": "shut_down"})));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
self.status();
|
||||
false
|
||||
}
|
||||
|
||||
/// Move every folder of the actor into `archive/`. Nothing is deleted.
|
||||
fn archive(&self) -> Result<Value> {
|
||||
let dest = self.world().root.join("archive").join(format!("{}-{}", self.id, world::now()));
|
||||
std::fs::create_dir_all(&dest)?;
|
||||
for (from, name) in [
|
||||
(self.world().actor_dir(&self.id), "actor"),
|
||||
(self.world().root.join("private").join(&self.id), "private"),
|
||||
(self.world().versions_dir(&self.id), "versions"),
|
||||
] {
|
||||
if from.exists() {
|
||||
std::fs::rename(&from, dest.join(name))?;
|
||||
}
|
||||
}
|
||||
Ok(json!({"type": "removed", "actor": self.id, "archive": dest.display().to_string()}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
//! Tests for a candidate version, before the broker installs it.
|
||||
//!
|
||||
//! A test is a JSON file in `source/tests/*.json`, one scenario per file:
|
||||
//!
|
||||
//! ```json
|
||||
//! {
|
||||
//! "description": "add, then list",
|
||||
//! "steps": [
|
||||
//! {"send": {"type": "add", "text": "milk"}, "expect": {"type": "added"}},
|
||||
//! {"send": {"type": "list"}, "from": "someone", "expect": [{"type": "notes"}], "to": "someone"}
|
||||
//! ]
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! Each step sends `send` (optional, from `from` or "test"), then reads one
|
||||
//! message for each entry of `expect` (an object or a list). An expected body
|
||||
//! matches if it is a subset of the actual body. With `to`, each message must
|
||||
//! also go to that recipient.
|
||||
//!
|
||||
//! Each file runs in a fresh instance on a copy of the actor's `/data` and
|
||||
//! `/public`. The candidate has no HTTP access. Without test files, the
|
||||
//! candidate must start and stay up for `SMOKE_TIME`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, anyhow, bail};
|
||||
use serde_json::{Value, json};
|
||||
use wasmtime::component::Component;
|
||||
|
||||
use crate::broker::{Broker, SHUTDOWN_DEADLINE, shutdown_line, truncate};
|
||||
use crate::fs::Mount;
|
||||
use crate::runtime::{Instance, InstanceConfig};
|
||||
use crate::world;
|
||||
|
||||
const SMOKE_TIME: Duration = Duration::from_secs(2);
|
||||
const EXPECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// `expected` matches `actual` if every key of an expected object exists in
|
||||
/// the actual object with a matching value. Other values must be equal.
|
||||
pub fn subset_match(expected: &Value, actual: &Value) -> bool {
|
||||
match (expected, actual) {
|
||||
(Value::Object(e), Value::Object(a)) => e.iter().all(|(k, v)| a.get(k).is_some_and(|av| subset_match(v, av))),
|
||||
(Value::Array(e), Value::Array(a)) => e.len() == a.len() && e.iter().zip(a).all(|(x, y)| subset_match(x, y)),
|
||||
_ => expected == actual,
|
||||
}
|
||||
}
|
||||
|
||||
struct Scratch {
|
||||
dir: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl Drop for Scratch {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn stderr_tail(path: &Path) -> String {
|
||||
let text = std::fs::read_to_string(path).unwrap_or_default();
|
||||
let start = text.len().saturating_sub(2000);
|
||||
let mut s = start;
|
||||
while !text.is_char_boundary(s) {
|
||||
s += 1;
|
||||
}
|
||||
text[s..].to_string()
|
||||
}
|
||||
|
||||
pub async fn test(
|
||||
broker: &Broker,
|
||||
id: &str,
|
||||
component: &Component,
|
||||
source: &Path,
|
||||
data: Option<&Path>,
|
||||
public: Option<&Path>,
|
||||
) -> Result<Value> {
|
||||
let mut files: Vec<_> = std::fs::read_dir(source.join("tests"))
|
||||
.map(|rd| {
|
||||
rd.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().is_some_and(|x| x == "json"))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
files.sort();
|
||||
|
||||
if files.is_empty() {
|
||||
let (mut inst, scratch) = start(broker, id, component, data, public)?;
|
||||
tokio::time::sleep(SMOKE_TIME).await;
|
||||
if inst.is_finished() {
|
||||
let exit = inst.wait().await;
|
||||
bail!(
|
||||
"candidate ended during the smoke test ({exit}). stderr:\n{}",
|
||||
stderr_tail(&scratch.dir.join("stderr.log"))
|
||||
);
|
||||
}
|
||||
inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
|
||||
return Ok(json!({"smoke_test": "passed", "files": 0}));
|
||||
}
|
||||
|
||||
let mut passed = Vec::new();
|
||||
for file in &files {
|
||||
let name = file.file_name().unwrap().to_string_lossy().to_string();
|
||||
run_file(broker, id, component, data, public, file)
|
||||
.await
|
||||
.map_err(|e| anyhow!("test {name} failed: {e:#}"))?;
|
||||
passed.push(name);
|
||||
}
|
||||
Ok(json!({"passed": passed}))
|
||||
}
|
||||
|
||||
fn start(broker: &Broker, id: &str, component: &Component, data: Option<&Path>, public: Option<&Path>) -> Result<(Instance, Scratch)> {
|
||||
let dir = broker.world.scratch_dir().join(format!("{id}-{}-{}", world::now(), rand_suffix()));
|
||||
let scratch = Scratch { dir: dir.clone() };
|
||||
std::fs::create_dir_all(dir.join("data"))?;
|
||||
std::fs::create_dir_all(dir.join("public"))?;
|
||||
if let Some(d) = data.filter(|d| d.is_dir()) {
|
||||
world::copy_tree(d, &dir.join("data"))?;
|
||||
}
|
||||
if let Some(p) = public.filter(|p| p.is_dir()) {
|
||||
world::copy_tree(p, &dir.join("public"))?;
|
||||
}
|
||||
let cfg = InstanceConfig {
|
||||
mounts: vec![
|
||||
Mount { guest: "/actor".into(), host: broker.world.actors_root(), writable: false },
|
||||
Mount { guest: "/public".into(), host: dir.join("public"), writable: true },
|
||||
Mount { guest: "/data".into(), host: dir.join("data"), writable: true },
|
||||
],
|
||||
allow_http: false,
|
||||
stderr_log: dir.join("stderr.log"),
|
||||
env: vec![("SAUSAGE_ACTOR".into(), id.to_string()), ("SAUSAGE_CANDIDATE".into(), "1".into())],
|
||||
};
|
||||
Ok((broker.runtime.start(component, cfg)?, scratch))
|
||||
}
|
||||
|
||||
fn rand_suffix() -> u32 {
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
let mut h = std::collections::hash_map::RandomState::new().build_hasher();
|
||||
h.write_u64(std::process::id() as u64);
|
||||
h.finish() as u32
|
||||
}
|
||||
|
||||
async fn run_file(
|
||||
broker: &Broker,
|
||||
id: &str,
|
||||
component: &Component,
|
||||
data: Option<&Path>,
|
||||
public: Option<&Path>,
|
||||
file: &Path,
|
||||
) -> Result<()> {
|
||||
let test: Value = serde_json::from_str(&std::fs::read_to_string(file)?).map_err(|e| anyhow!("invalid JSON: {e}"))?;
|
||||
let steps = test["steps"].as_array().ok_or_else(|| anyhow!("missing \"steps\" list"))?;
|
||||
let (mut inst, scratch) = start(broker, id, component, data, public)?;
|
||||
let stderr = scratch.dir.join("stderr.log");
|
||||
for (n, step) in steps.iter().enumerate() {
|
||||
let n = n + 1;
|
||||
if let Some(body) = step.get("send") {
|
||||
let from = step["from"].as_str().unwrap_or("test");
|
||||
let env = json!({"id": format!("test/{n}"), "from": from, "body": body});
|
||||
inst.send_line(&env.to_string())
|
||||
.await
|
||||
.map_err(|e| anyhow!("step {n}: candidate does not read stdin: {e}"))?;
|
||||
}
|
||||
let expected = match step.get("expect") {
|
||||
None => vec![],
|
||||
Some(Value::Array(list)) => list.clone(),
|
||||
Some(one) => vec![one.clone()],
|
||||
};
|
||||
for expected in expected {
|
||||
let line = match tokio::time::timeout(EXPECT_TIMEOUT, inst.stdout.recv()).await {
|
||||
Err(_) => bail!("step {n}: no message within {EXPECT_TIMEOUT:?}; expected {expected}"),
|
||||
Ok(None) => {
|
||||
let exit = inst.wait().await;
|
||||
bail!("step {n}: candidate ended ({exit}); expected {expected}. stderr:\n{}", stderr_tail(&stderr))
|
||||
}
|
||||
Ok(Some(l)) => l,
|
||||
};
|
||||
let msg: Value = serde_json::from_str(&line)
|
||||
.map_err(|_| anyhow!("step {n}: candidate wrote a non-JSON line: {}", truncate(&line, 300)))?;
|
||||
if !subset_match(&expected, &msg["body"]) {
|
||||
bail!("step {n}: expected body {expected}, got {}", truncate(&msg["body"].to_string(), 1000));
|
||||
}
|
||||
if let Some(to) = step.get("to")
|
||||
&& msg.get("to") != Some(to)
|
||||
{
|
||||
bail!("step {n}: expected recipient {to}, got {}", msg.get("to").unwrap_or(&Value::Null));
|
||||
}
|
||||
}
|
||||
}
|
||||
inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! The native `cargo` actor: builds Rust source folders into `.wasm` files.
|
||||
//!
|
||||
//! Request: `{"type": "build", "source": "/actor/<id>/public/<folder>"}`
|
||||
//! Replies: `{"type": "built", "wasm": "/actor/cargo/public/builds/<n>/<name>.wasm", ...}`
|
||||
//! `{"type": "build_failed", "errors": "..."}`
|
||||
//!
|
||||
//! The broker knows nothing about Rust. This actor is one way to get a
|
||||
//! `.wasm` file. Other actors can add other languages.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
use crate::broker::{Broker, Envelope, truncate};
|
||||
use crate::world::{self, World};
|
||||
|
||||
pub const ID: &str = "cargo";
|
||||
pub const TARGET: &str = "wasm32-wasip2";
|
||||
const BUILD_TIMEOUT: Duration = Duration::from_secs(900);
|
||||
|
||||
pub struct BuildOutput {
|
||||
/// Host path of the published `.wasm` file.
|
||||
pub wasm: PathBuf,
|
||||
pub seconds: f64,
|
||||
pub warnings: String,
|
||||
}
|
||||
|
||||
pub struct Cargo {
|
||||
world: World,
|
||||
/// One build at a time: all builds share one target folder.
|
||||
lock: Mutex<()>,
|
||||
}
|
||||
|
||||
impl Cargo {
|
||||
pub fn new(world: World) -> Self {
|
||||
Self { world, lock: Mutex::new(()) }
|
||||
}
|
||||
|
||||
/// Build the crate in `source` (a host path) and publish the `.wasm` file
|
||||
/// under `/actor/cargo/public/builds/<n>/`.
|
||||
pub async fn build(&self, source: &Path) -> Result<BuildOutput> {
|
||||
if !source.join("Cargo.toml").is_file() {
|
||||
bail!("{} has no Cargo.toml", source.display());
|
||||
}
|
||||
let _guard = self.lock.lock().await;
|
||||
let started = Instant::now();
|
||||
let data = self.world.data_dir(ID);
|
||||
let key = format!("{:016x}", fxhash(source.to_string_lossy().as_bytes()));
|
||||
let build_dir = data.join("work").join(&key);
|
||||
world::replace_tree(source, &build_dir)?;
|
||||
// The crate must not join a parent workspace by accident.
|
||||
let manifest = build_dir.join("Cargo.toml");
|
||||
let text = std::fs::read_to_string(&manifest)?;
|
||||
if !text.contains("[workspace]") {
|
||||
std::fs::write(&manifest, format!("{text}\n[workspace]\n"))?;
|
||||
}
|
||||
|
||||
let mut cmd = Command::new("cargo");
|
||||
cmd.arg("build")
|
||||
.arg("--release")
|
||||
.arg("--target")
|
||||
.arg(TARGET)
|
||||
.arg("--message-format=json-render-diagnostics")
|
||||
.arg("--target-dir")
|
||||
.arg(data.join("target"))
|
||||
.current_dir(&build_dir)
|
||||
.kill_on_drop(true);
|
||||
let out = tokio::time::timeout(BUILD_TIMEOUT, cmd.output())
|
||||
.await
|
||||
.map_err(|_| anyhow!("cargo build took longer than {BUILD_TIMEOUT:?}"))?
|
||||
.context("cannot run cargo (is it on PATH, with the wasm32-wasip2 target?)")?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
|
||||
if !out.status.success() {
|
||||
bail!("{}", truncate(&strip_progress(&stderr), 20_000));
|
||||
}
|
||||
let mut artifact = None;
|
||||
for line in String::from_utf8_lossy(&out.stdout).lines() {
|
||||
if let Ok(v) = serde_json::from_str::<Value>(line)
|
||||
&& v["reason"] == "compiler-artifact"
|
||||
&& let Some(exe) = v["executable"].as_str()
|
||||
{
|
||||
artifact = Some(PathBuf::from(exe));
|
||||
}
|
||||
}
|
||||
let artifact = artifact.ok_or_else(|| anyhow!("cargo built no binary. Does the crate have a src/main.rs?"))?;
|
||||
|
||||
let n = next_build_number(&self.world.public_dir(ID).join("builds"))?;
|
||||
let dest_dir = self.world.public_dir(ID).join("builds").join(n.to_string());
|
||||
let name = artifact.file_name().context("artifact has no file name")?;
|
||||
let dest = dest_dir.join(name);
|
||||
world::write_atomic(&dest, &std::fs::read(&artifact)?)?;
|
||||
Ok(BuildOutput { wasm: dest, seconds: started.elapsed().as_secs_f64(), warnings: strip_progress(&stderr) })
|
||||
}
|
||||
|
||||
pub async fn run_actor(self: Arc<Self>, broker: Arc<Broker>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||
while let Some(env) = mailbox.recv().await {
|
||||
let this = self.clone();
|
||||
let broker = broker.clone();
|
||||
tokio::spawn(async move {
|
||||
let body = this.handle(&broker, &env).await.unwrap_or_else(|e| {
|
||||
json!({"type": "build_failed", "source": env.body.get("source"), "errors": format!("{e:#}")})
|
||||
});
|
||||
broker.send(ID, Some(&env.from), None, Some(&env.id), body);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle(&self, broker: &Broker, env: &Envelope) -> Result<Value> {
|
||||
match env.body.get("type").and_then(Value::as_str) {
|
||||
Some("build") => {
|
||||
let source_s = env.body.get("source").and_then(Value::as_str).ok_or_else(|| anyhow!("missing \"source\""))?;
|
||||
let source = broker.world.resolve(&env.from, source_s)?;
|
||||
let out = self.build(&source).await?;
|
||||
let wasm = self.world.global_path(&out.wasm).unwrap_or_else(|| out.wasm.display().to_string());
|
||||
Ok(json!({
|
||||
"type": "built",
|
||||
"wasm": wasm,
|
||||
"source": source_s,
|
||||
"built_by": ID,
|
||||
"seconds": (out.seconds * 10.0).round() / 10.0,
|
||||
"warnings": truncate(&out.warnings, 4000),
|
||||
}))
|
||||
}
|
||||
Some("shutdown") => Ok(json!({"type": "ignored"})),
|
||||
other => bail!("unknown request {other:?}. Known: build"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn next_build_number(dir: &Path) -> Result<u64> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let max = std::fs::read_dir(dir)?
|
||||
.filter_map(|e| e.ok()?.file_name().to_str()?.parse::<u64>().ok())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
Ok(max + 1)
|
||||
}
|
||||
|
||||
/// Remove cargo's "Compiling ..." progress lines. Keep diagnostics.
|
||||
fn strip_progress(stderr: &str) -> String {
|
||||
stderr
|
||||
.lines()
|
||||
.filter(|l| {
|
||||
let t = l.trim_start();
|
||||
!(t.starts_with("Compiling ")
|
||||
|| t.starts_with("Downloaded ")
|
||||
|| t.starts_with("Downloading ")
|
||||
|| t.starts_with("Updating ")
|
||||
|| t.starts_with("Locking ")
|
||||
|| t.starts_with("Adding ")
|
||||
|| t.starts_with("Blocking "))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn fxhash(bytes: &[u8]) -> u64 {
|
||||
let mut h: u64 = 0xcbf29ce484222325;
|
||||
for b in bytes {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x100000001b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
//! The native `console` actor: your terminal.
|
||||
//!
|
||||
//! - `@<actor> <json>` sends a JSON body. `@<actor> <text>` sends
|
||||
//! `{"type": "text", "text": ...}`.
|
||||
//! - A line without `@` goes to `agent` as text.
|
||||
//! - `/deploy <actor> <folder>` builds a Rust folder with `cargo`, then spawns
|
||||
//! or upgrades the actor.
|
||||
//! - `/list`, `/sleep <seconds>`, `/help`, `/quit`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, anyhow, bail};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
use crate::broker::{Broker, Envelope};
|
||||
|
||||
pub const ID: &str = "console";
|
||||
|
||||
const HELP: &str = "\
|
||||
@<actor> <json> send a JSON body to an actor
|
||||
@<actor> <text> send {\"type\":\"text\",\"text\":...}
|
||||
<text> send text to the agent
|
||||
/deploy <actor> <dir> build a Rust folder with cargo, then spawn or upgrade
|
||||
/list list actors
|
||||
/sleep <seconds> wait (for scripts)
|
||||
/await <actor> <secs> wait for the next message from an actor (for scripts)
|
||||
/quit stop all actors and exit";
|
||||
|
||||
pub struct Console {
|
||||
broker: Arc<Broker>,
|
||||
waiters: Mutex<HashMap<String, oneshot::Sender<Envelope>>>,
|
||||
seq: AtomicU64,
|
||||
/// Number of messages received per sender, for `/await`.
|
||||
seen: Mutex<HashMap<String, u64>>,
|
||||
arrived: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
fn show(env: &Envelope) {
|
||||
let reply = if env.reply_to.is_some() { " ↩" } else { "" };
|
||||
let b = &env.body;
|
||||
if b["type"] == "text"
|
||||
&& let Some(t) = b["text"].as_str()
|
||||
{
|
||||
println!("[{}{reply}] {t}", env.from);
|
||||
return;
|
||||
}
|
||||
let mut compact = b.clone();
|
||||
let mut extra = Vec::new();
|
||||
if let Some(obj) = compact.as_object_mut() {
|
||||
for key in ["errors", "warnings", "reason"] {
|
||||
if let Some(Value::String(s)) = obj.get(key)
|
||||
&& s.contains('\n')
|
||||
{
|
||||
extra.push((key, s.clone()));
|
||||
obj.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("[{}{reply}] {compact}", env.from);
|
||||
for (k, s) in extra {
|
||||
println!(" {k}:\n{}", s.lines().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n"));
|
||||
}
|
||||
}
|
||||
|
||||
impl Console {
|
||||
pub fn new(broker: Arc<Broker>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
broker,
|
||||
waiters: Mutex::new(HashMap::new()),
|
||||
seq: AtomicU64::new(1),
|
||||
seen: Mutex::new(HashMap::new()),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run_mailbox(self: Arc<Self>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||
while let Some(env) = mailbox.recv().await {
|
||||
show(&env);
|
||||
*self.seen.lock().unwrap().entry(env.from.clone()).or_default() += 1;
|
||||
self.arrived.notify_waiters();
|
||||
if let Some(r) = &env.reply_to
|
||||
&& let Some(tx) = self.waiters.lock().unwrap().remove(r)
|
||||
{
|
||||
let _ = tx.send(env);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn send(&self, to: &str, body: Value) -> String {
|
||||
let own = format!("c{}", self.seq.fetch_add(1, Ordering::Relaxed));
|
||||
self.broker.send(ID, Some(to), Some(&own), None, body)
|
||||
}
|
||||
|
||||
async fn request(&self, to: &str, body: Value, timeout: Duration) -> Result<Envelope> {
|
||||
let own = format!("c{}", self.seq.fetch_add(1, Ordering::Relaxed));
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.waiters.lock().unwrap().insert(format!("{ID}/{own}"), tx);
|
||||
self.broker.send(ID, Some(to), Some(&own), None, body);
|
||||
tokio::time::timeout(timeout, rx)
|
||||
.await
|
||||
.map_err(|_| anyhow!("no reply from {to} within {timeout:?}"))?
|
||||
.map_err(|_| anyhow!("reply channel closed"))
|
||||
}
|
||||
|
||||
async fn deploy(&self, actor: &str, dir: &str) -> Result<()> {
|
||||
let path = std::path::absolute(dir)?;
|
||||
let source = path.display().to_string();
|
||||
println!("· building {source} …");
|
||||
let built = self.request(crate::cargo_actor::ID, json!({"type": "build", "source": source}), Duration::from_secs(900)).await?;
|
||||
if built.body["type"] != "built" {
|
||||
bail!("build failed");
|
||||
}
|
||||
let exists = self.broker.registry().actors.contains_key(actor);
|
||||
let ty = if exists { "upgrade" } else { "spawn" };
|
||||
println!("· {ty} {actor} …");
|
||||
let body = json!({
|
||||
"type": ty,
|
||||
"actor": actor,
|
||||
"wasm": built.body["wasm"],
|
||||
"source": source,
|
||||
"built_by": built.body["built_by"],
|
||||
});
|
||||
let r = self.request("broker", body, Duration::from_secs(300)).await?;
|
||||
if r.body["type"] == "error" {
|
||||
bail!("{ty} failed");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read commands until EOF or `/quit`.
|
||||
pub async fn run_input(self: Arc<Self>) {
|
||||
println!("Sausage console. Type /help for commands.");
|
||||
let mut lines = BufReader::new(tokio::io::stdin()).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = self.command(line).await {
|
||||
if e.to_string() == "quit" {
|
||||
break;
|
||||
}
|
||||
println!("! {e:#}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn command(&self, line: &str) -> Result<()> {
|
||||
if let Some(rest) = line.strip_prefix('@') {
|
||||
let (to, text) = rest.split_once(char::is_whitespace).unwrap_or((rest, ""));
|
||||
let text = text.trim();
|
||||
let body = match serde_json::from_str::<Value>(text) {
|
||||
Ok(v @ Value::Object(_)) => v,
|
||||
_ => json!({"type": "text", "text": text}),
|
||||
};
|
||||
self.send(to, body);
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(cmd) = line.strip_prefix('/') {
|
||||
let mut parts = cmd.split_whitespace();
|
||||
match parts.next() {
|
||||
Some("help") => println!("{HELP}"),
|
||||
Some("quit") => bail!("quit"),
|
||||
Some("list") => {
|
||||
self.send("broker", json!({"type": "list"}));
|
||||
}
|
||||
Some("sleep") => {
|
||||
let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(1.0);
|
||||
tokio::time::sleep(Duration::from_secs_f64(secs)).await;
|
||||
}
|
||||
Some("await") => {
|
||||
let from = parts.next().unwrap_or("agent").to_string();
|
||||
let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(60.0);
|
||||
let count = |c: &Self| c.seen.lock().unwrap().get(&from).copied().unwrap_or(0);
|
||||
let before = count(self);
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs_f64(secs);
|
||||
loop {
|
||||
let notified = self.arrived.notified();
|
||||
tokio::pin!(notified);
|
||||
notified.as_mut().enable();
|
||||
if count(self) != before {
|
||||
break;
|
||||
}
|
||||
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||
bail!("no message from {from} within {secs} s");
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("deploy") => {
|
||||
let (Some(actor), Some(dir)) = (parts.next(), parts.next()) else {
|
||||
bail!("usage: /deploy <actor> <dir>");
|
||||
};
|
||||
self.deploy(actor, dir).await?;
|
||||
}
|
||||
_ => bail!("unknown command. Type /help."),
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
self.send("agent", json!({"type": "text", "text": line}));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
//! Filesystem layer with commit-on-close semantics.
|
||||
//!
|
||||
//! The layer wraps the wasmtime-wasi `wasi:filesystem` implementation. In a
|
||||
//! writable mount, an open that creates or truncates a file goes to a hidden
|
||||
//! temporary file next to the target. When the guest drops the descriptor
|
||||
//! (close), the layer renames the temporary file into place. Until then, no
|
||||
//! other reader sees the change. If the instance traps or is killed, the
|
||||
//! changes are lost.
|
||||
//!
|
||||
//! An open of an existing file without truncate (append, in-place edit)
|
||||
//! writes in place, so that appending to a log does not copy the log.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use wasmtime::component::{HasData, Resource};
|
||||
use wasmtime_wasi::filesystem::{Descriptor, WasiFilesystemCtxView};
|
||||
use wasmtime_wasi::p2::bindings::filesystem::types::{
|
||||
self, ErrorCode, HostDescriptor, HostDirectoryEntryStream,
|
||||
};
|
||||
use wasmtime_wasi::p2::bindings::filesystem::preopens;
|
||||
use wasmtime_wasi::p2::{DynInputStream, DynOutputStream, FsError, FsResult};
|
||||
|
||||
/// Name prefix of uncommitted files. The layer hides these names from guests.
|
||||
pub const TMP_PREFIX: &str = ".sausage-tmp-";
|
||||
|
||||
static TMP_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Mount {
|
||||
pub guest: String,
|
||||
pub host: PathBuf,
|
||||
pub writable: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct DirInfo {
|
||||
host: PathBuf,
|
||||
root: PathBuf,
|
||||
commit: bool,
|
||||
}
|
||||
|
||||
struct Pending {
|
||||
tmp: PathBuf,
|
||||
target: PathBuf,
|
||||
}
|
||||
|
||||
/// Per-instance state of the layer.
|
||||
pub struct FsExt {
|
||||
mounts: Vec<Mount>,
|
||||
dirs: HashMap<u32, DirInfo>,
|
||||
pending: HashMap<u32, Pending>,
|
||||
}
|
||||
|
||||
impl FsExt {
|
||||
pub fn new(mounts: Vec<Mount>) -> Self {
|
||||
Self { mounts, dirs: HashMap::new(), pending: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Commit every open write. Use this when the guest exits normally,
|
||||
/// because process exit closes all files.
|
||||
pub fn commit_all(&mut self) {
|
||||
for (_, p) in self.pending.drain() {
|
||||
commit(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FsExt {
|
||||
/// Discard writes that were not committed (trap, kill).
|
||||
fn drop(&mut self) {
|
||||
for (_, p) in self.pending.drain() {
|
||||
let _ = std::fs::remove_file(&p.tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn commit(p: Pending) {
|
||||
if let Err(e) = std::fs::rename(&p.tmp, &p.target) {
|
||||
eprintln!("sausage: commit of {} failed: {e}", p.target.display());
|
||||
let _ = std::fs::remove_file(&p.tmp);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_tmp_name(path: &str) -> bool {
|
||||
path.rsplit('/').next().is_some_and(|n| n.starts_with(TMP_PREFIX))
|
||||
}
|
||||
|
||||
/// Join `rel` to the directory `dir` lexically. Return `None` if the result
|
||||
/// leaves the mount root.
|
||||
fn resolve(dir: &DirInfo, rel: &str) -> Option<PathBuf> {
|
||||
let mut out = dir.host.clone();
|
||||
for c in Path::new(rel).components() {
|
||||
match c {
|
||||
Component::Normal(n) => out.push(n),
|
||||
Component::CurDir => {}
|
||||
Component::ParentDir => {
|
||||
if !out.pop() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Component::RootDir | Component::Prefix(_) => return None,
|
||||
}
|
||||
}
|
||||
out.starts_with(&dir.root).then_some(out)
|
||||
}
|
||||
|
||||
pub struct ActorFs<'a> {
|
||||
pub inner: WasiFilesystemCtxView<'a>,
|
||||
pub ext: &'a mut FsExt,
|
||||
}
|
||||
|
||||
pub struct ActorFsData;
|
||||
|
||||
impl HasData for ActorFsData {
|
||||
type Data<'a> = ActorFs<'a>;
|
||||
}
|
||||
|
||||
impl preopens::Host for ActorFs<'_> {
|
||||
fn get_directories(&mut self) -> wasmtime::Result<Vec<(Resource<Descriptor>, String)>> {
|
||||
let dirs = preopens::Host::get_directories(&mut self.inner)?;
|
||||
for (r, name) in &dirs {
|
||||
if let Some(m) = self.ext.mounts.iter().find(|m| &m.guest == name) {
|
||||
self.ext.dirs.insert(
|
||||
r.rep(),
|
||||
DirInfo { host: m.host.clone(), root: m.host.clone(), commit: m.writable },
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(dirs)
|
||||
}
|
||||
}
|
||||
|
||||
impl types::Host for ActorFs<'_> {
|
||||
fn convert_error_code(&mut self, err: FsError) -> wasmtime::Result<ErrorCode> {
|
||||
types::Host::convert_error_code(&mut self.inner, err)
|
||||
}
|
||||
|
||||
fn filesystem_error_code(
|
||||
&mut self,
|
||||
err: Resource<wasmtime::Error>,
|
||||
) -> wasmtime::Result<Option<ErrorCode>> {
|
||||
types::Host::filesystem_error_code(&mut self.inner, err)
|
||||
}
|
||||
}
|
||||
|
||||
impl HostDescriptor for ActorFs<'_> {
|
||||
async fn advise(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
offset: types::Filesize,
|
||||
len: types::Filesize,
|
||||
advice: types::Advice,
|
||||
) -> FsResult<()> {
|
||||
HostDescriptor::advise(&mut self.inner, fd, offset, len, advice).await
|
||||
}
|
||||
|
||||
async fn sync_data(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
||||
HostDescriptor::sync_data(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn get_flags(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorFlags> {
|
||||
HostDescriptor::get_flags(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn get_type(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorType> {
|
||||
HostDescriptor::get_type(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn set_size(&mut self, fd: Resource<types::Descriptor>, size: types::Filesize) -> FsResult<()> {
|
||||
HostDescriptor::set_size(&mut self.inner, fd, size).await
|
||||
}
|
||||
|
||||
async fn set_times(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
atim: types::NewTimestamp,
|
||||
mtim: types::NewTimestamp,
|
||||
) -> FsResult<()> {
|
||||
HostDescriptor::set_times(&mut self.inner, fd, atim, mtim).await
|
||||
}
|
||||
|
||||
async fn read(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
len: types::Filesize,
|
||||
offset: types::Filesize,
|
||||
) -> FsResult<(Vec<u8>, bool)> {
|
||||
HostDescriptor::read(&mut self.inner, fd, len, offset).await
|
||||
}
|
||||
|
||||
async fn write(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
buf: Vec<u8>,
|
||||
offset: types::Filesize,
|
||||
) -> FsResult<types::Filesize> {
|
||||
HostDescriptor::write(&mut self.inner, fd, buf, offset).await
|
||||
}
|
||||
|
||||
async fn read_directory(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
) -> FsResult<Resource<types::DirectoryEntryStream>> {
|
||||
HostDescriptor::read_directory(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn sync(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
||||
HostDescriptor::sync(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn create_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||
HostDescriptor::create_directory_at(&mut self.inner, fd, path).await
|
||||
}
|
||||
|
||||
async fn stat(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorStat> {
|
||||
HostDescriptor::stat(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn stat_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
path_flags: types::PathFlags,
|
||||
path: String,
|
||||
) -> FsResult<types::DescriptorStat> {
|
||||
if is_tmp_name(&path) {
|
||||
return Err(ErrorCode::NoEntry.into());
|
||||
}
|
||||
HostDescriptor::stat_at(&mut self.inner, fd, path_flags, path).await
|
||||
}
|
||||
|
||||
async fn set_times_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
path_flags: types::PathFlags,
|
||||
path: String,
|
||||
atim: types::NewTimestamp,
|
||||
mtim: types::NewTimestamp,
|
||||
) -> FsResult<()> {
|
||||
HostDescriptor::set_times_at(&mut self.inner, fd, path_flags, path, atim, mtim).await
|
||||
}
|
||||
|
||||
async fn link_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
old_path_flags: types::PathFlags,
|
||||
old_path: String,
|
||||
new_descriptor: Resource<types::Descriptor>,
|
||||
new_path: String,
|
||||
) -> FsResult<()> {
|
||||
HostDescriptor::link_at(&mut self.inner, fd, old_path_flags, old_path, new_descriptor, new_path).await
|
||||
}
|
||||
|
||||
async fn open_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
path_flags: types::PathFlags,
|
||||
path: String,
|
||||
oflags: types::OpenFlags,
|
||||
flags: types::DescriptorFlags,
|
||||
) -> FsResult<Resource<types::Descriptor>> {
|
||||
if is_tmp_name(&path) {
|
||||
return Err(ErrorCode::NoEntry.into());
|
||||
}
|
||||
let parent = self.ext.dirs.get(&fd.rep()).cloned();
|
||||
|
||||
// Commit-on-close: a new or truncated file goes to a hidden temporary file.
|
||||
if let Some(dir) = parent.as_ref().filter(|d| d.commit)
|
||||
&& flags.contains(types::DescriptorFlags::WRITE)
|
||||
&& !oflags.contains(types::OpenFlags::DIRECTORY)
|
||||
&& let Some(target) = resolve(dir, &path)
|
||||
{
|
||||
let meta = std::fs::metadata(&target).ok();
|
||||
let new_file = meta.is_none() && oflags.contains(types::OpenFlags::CREATE);
|
||||
let truncated = meta.as_ref().is_some_and(|m| m.is_file())
|
||||
&& oflags.contains(types::OpenFlags::TRUNCATE)
|
||||
&& !oflags.contains(types::OpenFlags::EXCLUSIVE);
|
||||
if new_file || truncated {
|
||||
let n = TMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let tmp_name = format!("{TMP_PREFIX}{}-{n}", std::process::id());
|
||||
let tmp_rel = match path.rsplit_once('/') {
|
||||
Some((d, _)) => format!("{d}/{tmp_name}"),
|
||||
None => tmp_name.clone(),
|
||||
};
|
||||
let tmp = target.with_file_name(&tmp_name);
|
||||
let r = HostDescriptor::open_at(&mut self.inner, fd, path_flags, tmp_rel, oflags, flags).await?;
|
||||
self.ext.pending.insert(r.rep(), Pending { tmp, target });
|
||||
return Ok(r);
|
||||
}
|
||||
}
|
||||
|
||||
let r = HostDescriptor::open_at(&mut self.inner, fd, path_flags, path.clone(), oflags, flags).await?;
|
||||
if let Some(dir) = parent
|
||||
&& matches!(self.inner.table.get(&r)?, Descriptor::Dir(_))
|
||||
&& let Some(host) = resolve(&dir, &path)
|
||||
{
|
||||
self.ext.dirs.insert(r.rep(), DirInfo { host, root: dir.root, commit: dir.commit });
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
fn drop(&mut self, fd: Resource<types::Descriptor>) -> wasmtime::Result<()> {
|
||||
let rep = fd.rep();
|
||||
HostDescriptor::drop(&mut self.inner, fd)?;
|
||||
self.ext.dirs.remove(&rep);
|
||||
if let Some(p) = self.ext.pending.remove(&rep) {
|
||||
commit(p);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn readlink_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<String> {
|
||||
HostDescriptor::readlink_at(&mut self.inner, fd, path).await
|
||||
}
|
||||
|
||||
async fn remove_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||
HostDescriptor::remove_directory_at(&mut self.inner, fd, path).await
|
||||
}
|
||||
|
||||
async fn rename_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
old_path: String,
|
||||
new_fd: Resource<types::Descriptor>,
|
||||
new_path: String,
|
||||
) -> FsResult<()> {
|
||||
if is_tmp_name(&old_path) || is_tmp_name(&new_path) {
|
||||
return Err(ErrorCode::NoEntry.into());
|
||||
}
|
||||
HostDescriptor::rename_at(&mut self.inner, fd, old_path, new_fd, new_path).await
|
||||
}
|
||||
|
||||
async fn symlink_at(
|
||||
&mut self,
|
||||
_fd: Resource<types::Descriptor>,
|
||||
_src_path: String,
|
||||
_dest_path: String,
|
||||
) -> FsResult<()> {
|
||||
// Symlinks would break the lexical path tracking of commit-on-close.
|
||||
Err(ErrorCode::NotPermitted.into())
|
||||
}
|
||||
|
||||
async fn unlink_file_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||
if is_tmp_name(&path) {
|
||||
return Err(ErrorCode::NoEntry.into());
|
||||
}
|
||||
HostDescriptor::unlink_file_at(&mut self.inner, fd, path).await
|
||||
}
|
||||
|
||||
fn read_via_stream(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
offset: types::Filesize,
|
||||
) -> FsResult<Resource<DynInputStream>> {
|
||||
HostDescriptor::read_via_stream(&mut self.inner, fd, offset)
|
||||
}
|
||||
|
||||
fn write_via_stream(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
offset: types::Filesize,
|
||||
) -> FsResult<Resource<DynOutputStream>> {
|
||||
HostDescriptor::write_via_stream(&mut self.inner, fd, offset)
|
||||
}
|
||||
|
||||
fn append_via_stream(&mut self, fd: Resource<types::Descriptor>) -> FsResult<Resource<DynOutputStream>> {
|
||||
HostDescriptor::append_via_stream(&mut self.inner, fd)
|
||||
}
|
||||
|
||||
async fn is_same_object(
|
||||
&mut self,
|
||||
a: Resource<types::Descriptor>,
|
||||
b: Resource<types::Descriptor>,
|
||||
) -> wasmtime::Result<bool> {
|
||||
HostDescriptor::is_same_object(&mut self.inner, a, b).await
|
||||
}
|
||||
|
||||
async fn metadata_hash(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::MetadataHashValue> {
|
||||
HostDescriptor::metadata_hash(&mut self.inner, fd).await
|
||||
}
|
||||
|
||||
async fn metadata_hash_at(
|
||||
&mut self,
|
||||
fd: Resource<types::Descriptor>,
|
||||
path_flags: types::PathFlags,
|
||||
path: String,
|
||||
) -> FsResult<types::MetadataHashValue> {
|
||||
HostDescriptor::metadata_hash_at(&mut self.inner, fd, path_flags, path).await
|
||||
}
|
||||
}
|
||||
|
||||
impl HostDirectoryEntryStream for ActorFs<'_> {
|
||||
async fn read_directory_entry(
|
||||
&mut self,
|
||||
stream: Resource<types::DirectoryEntryStream>,
|
||||
) -> FsResult<Option<types::DirectoryEntry>> {
|
||||
let rep = stream.rep();
|
||||
loop {
|
||||
let entry = HostDirectoryEntryStream::read_directory_entry(
|
||||
&mut self.inner,
|
||||
Resource::new_borrow(rep),
|
||||
)
|
||||
.await?;
|
||||
match entry {
|
||||
Some(e) if e.name.starts_with(TMP_PREFIX) => continue,
|
||||
other => return Ok(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn drop(&mut self, stream: Resource<types::DirectoryEntryStream>) -> wasmtime::Result<()> {
|
||||
HostDirectoryEntryStream::drop(&mut self.inner, stream)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! Sausage: grow a system of WASI actors by talking to it.
|
||||
//!
|
||||
//! The host runs the broker, the native actors (`broker`, `cargo`, and the
|
||||
//! console), and one supervisor per WASM actor. See `PROTOCOL.md`.
|
||||
|
||||
pub mod broker;
|
||||
pub mod candidate;
|
||||
pub mod cargo_actor;
|
||||
pub mod console;
|
||||
pub mod fs;
|
||||
pub mod runtime;
|
||||
pub mod world;
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub use broker::{Broker, Envelope};
|
||||
pub use world::World;
|
||||
|
||||
/// A running world.
|
||||
pub struct Host {
|
||||
pub world: World,
|
||||
pub broker: Arc<Broker>,
|
||||
_lock: std::fs::File,
|
||||
}
|
||||
|
||||
impl Host {
|
||||
/// Start the broker, `cargo`, and every registered actor of the world at
|
||||
/// `root`. Only one host can run a world at a time.
|
||||
pub async fn start(root: &Path) -> Result<Host> {
|
||||
let world = World::new(std::path::absolute(root)?);
|
||||
if !world.exists() {
|
||||
bail!("{} is not a Sausage world. Create one with: sausage init", root.display());
|
||||
}
|
||||
let lock = std::fs::File::create(world.root.join(".lock"))?;
|
||||
if lock.try_lock().is_err() {
|
||||
bail!("another sausage process runs {}", root.display());
|
||||
}
|
||||
let broker = Broker::new(world.clone(), runtime::Runtime::new()?)?;
|
||||
|
||||
let rx = broker.open_mailbox("broker");
|
||||
tokio::spawn(broker.clone().run_actor(rx));
|
||||
let rx = broker.open_mailbox(cargo_actor::ID);
|
||||
tokio::spawn(Arc::new(cargo_actor::Cargo::new(world.clone())).run_actor(broker.clone(), rx));
|
||||
|
||||
broker.start_all();
|
||||
Ok(Host { world, broker, _lock: lock })
|
||||
}
|
||||
|
||||
/// Register a native actor with the ID `id` and return its mailbox.
|
||||
pub fn mailbox(&self, id: &str) -> mpsc::Receiver<Envelope> {
|
||||
self.broker.open_mailbox(id)
|
||||
}
|
||||
|
||||
/// Stop every actor gracefully.
|
||||
pub async fn stop(self) {
|
||||
self.broker.shutdown_all().await;
|
||||
}
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::{Parser, Subcommand};
|
||||
use include_dir::{Dir, include_dir};
|
||||
use serde_json::json;
|
||||
|
||||
use sausage::world::{self, VersionMeta, World};
|
||||
use sausage::{Broker, Host, cargo_actor, console, runtime};
|
||||
|
||||
/// The source of the agent actor. `init` builds it like any other actor.
|
||||
static AGENT_SOURCE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../agent");
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(version, about)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Create a new world and build the agent actor.
|
||||
///
|
||||
/// Reads SAUSAGE_API, SAUSAGE_BASE_URL, SAUSAGE_MODEL and
|
||||
/// SAUSAGE_API_TOKEN, and asks for every value that is missing.
|
||||
Init {
|
||||
/// Folder of the new world.
|
||||
#[arg(default_value = "world")]
|
||||
dir: PathBuf,
|
||||
},
|
||||
/// Start the broker, all actors, and the console.
|
||||
Run {
|
||||
#[arg(default_value = "world")]
|
||||
dir: PathBuf,
|
||||
},
|
||||
}
|
||||
|
||||
/// The value of `var`, or the answer to `question`.
|
||||
fn setting(var: &str, question: &str, default: Option<&str>) -> Result<String> {
|
||||
if let Ok(v) = std::env::var(var)
|
||||
&& !v.is_empty()
|
||||
{
|
||||
return Ok(v);
|
||||
}
|
||||
match default {
|
||||
Some(d) => print!("{question} [{d}]: "),
|
||||
None => print!("{question}: "),
|
||||
}
|
||||
std::io::stdout().flush()?;
|
||||
let mut line = String::new();
|
||||
std::io::stdin().read_line(&mut line)?;
|
||||
match (line.trim(), default) {
|
||||
("", Some(d)) => Ok(d.to_string()),
|
||||
("", None) => bail!("{question} is required (or set {var})"),
|
||||
(v, _) => Ok(v.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn init(dir: PathBuf) -> Result<()> {
|
||||
let api = setting("SAUSAGE_API", "API shape (openai-responses or anthropic-messages)", Some("openai-responses"))?;
|
||||
if api != "openai-responses" && api != "anthropic-messages" {
|
||||
bail!("unknown API shape {api:?}");
|
||||
}
|
||||
let base_url = setting("SAUSAGE_BASE_URL", "API base URL", None)?;
|
||||
let model = setting("SAUSAGE_MODEL", "Model", None)?;
|
||||
let token = setting("SAUSAGE_API_TOKEN", "API token", None)?;
|
||||
|
||||
let world = World::create(&dir)?;
|
||||
let mut reg = world.load_registry()?;
|
||||
reg.subscriptions.entry("agent".into()).or_default().insert("console".into());
|
||||
world.save_registry(®)?;
|
||||
|
||||
// The agent's private configuration. Only the agent can read /data.
|
||||
world.ensure_actor_dirs("agent")?;
|
||||
let config = json!({"api": api, "base_url": base_url.trim_end_matches('/'), "model": model, "token": token});
|
||||
let config_path = world.data_dir("agent").join("config.json");
|
||||
std::fs::write(&config_path, serde_json::to_string_pretty(&config)?)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(&config_path, std::fs::Permissions::from_mode(0o600))?;
|
||||
}
|
||||
|
||||
let source = world.root.join("broker").join("seed").join("agent");
|
||||
std::fs::create_dir_all(&source)?;
|
||||
AGENT_SOURCE.extract(&source).context("extract agent source")?;
|
||||
println!("Building the agent actor (the first build takes a few minutes) …");
|
||||
let out = cargo_actor::Cargo::new(world.clone()).build(&source).await.context("build agent")?;
|
||||
println!("Built in {:.1} s.", out.seconds);
|
||||
|
||||
let broker = Broker::new(world.clone(), runtime::Runtime::new()?)?;
|
||||
let meta = VersionMeta {
|
||||
version: 1,
|
||||
wasm: world.global_path(&out.wasm).unwrap_or_default(),
|
||||
source: Some(source.display().to_string()),
|
||||
built_by: Some(cargo_actor::ID.into()),
|
||||
requested_by: "init".into(),
|
||||
created_at: world::now(),
|
||||
};
|
||||
broker.create_actor("agent", &out.wasm, Some(&source), meta, false).await?;
|
||||
println!("World ready. Start it with: sausage run {}", dir.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(dir: PathBuf) -> Result<()> {
|
||||
let host = Host::start(&dir).await?;
|
||||
let rx = host.mailbox(console::ID);
|
||||
let console = console::Console::new(host.broker.clone());
|
||||
tokio::spawn(Arc::clone(&console).run_mailbox(rx));
|
||||
console.run_input().await;
|
||||
println!("Stopping actors …");
|
||||
host.stop().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
match Cli::parse().command {
|
||||
Cmd::Init { dir } => init(dir).await,
|
||||
Cmd::Run { dir } => run(dir).await,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
//! Wasmtime engine and actor instances.
|
||||
//!
|
||||
//! An instance is one run of a `wasi:cli/command` component. The host writes
|
||||
//! NDJSON lines to its stdin and reads NDJSON lines from its stdout.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, DuplexStream};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use wasmtime::component::{Component, Linker, ResourceTable};
|
||||
use wasmtime::{Config, Engine, Store};
|
||||
use wasmtime_wasi::cli::{AsyncStdinStream, AsyncStdoutStream};
|
||||
use wasmtime_wasi::p2::bindings::Command;
|
||||
use wasmtime_wasi::p2::bindings::filesystem::{preopens, types};
|
||||
use wasmtime_wasi::{FsPerms, I32Exit, WasiCtx, WasiCtxView, WasiView};
|
||||
use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, WasiHttpView};
|
||||
|
||||
use crate::fs::{ActorFs, ActorFsData, FsExt, Mount};
|
||||
|
||||
pub struct WasiPart {
|
||||
ctx: WasiCtx,
|
||||
table: ResourceTable,
|
||||
}
|
||||
|
||||
impl WasiView for WasiPart {
|
||||
fn ctx(&mut self) -> WasiCtxView<'_> {
|
||||
WasiCtxView { ctx: &mut self.ctx, table: &mut self.table }
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HttpHooks {
|
||||
allow: bool,
|
||||
}
|
||||
|
||||
impl WasiHttpHooks for HttpHooks {
|
||||
fn send_request(
|
||||
&mut self,
|
||||
request: http::Request<wasmtime_wasi_http::WasiBody>,
|
||||
options: Option<wasmtime_wasi_http::RequestOptions>,
|
||||
fut: Box<dyn Future<Output = Result<(), wasmtime_wasi_http::Error>> + Send>,
|
||||
) -> Box<
|
||||
dyn Future<
|
||||
Output = wasmtime_wasi_http::Result<(
|
||||
http::Response<wasmtime_wasi_http::WasiBody>,
|
||||
Box<dyn Future<Output = Result<(), wasmtime_wasi_http::Error>> + Send>,
|
||||
)>,
|
||||
> + Send,
|
||||
> {
|
||||
_ = fut;
|
||||
let allow = self.allow;
|
||||
Box::new(async move {
|
||||
use http_body_util::BodyExt;
|
||||
if !allow {
|
||||
return Err(wasmtime_wasi_http::Error::HttpRequestDenied);
|
||||
}
|
||||
let (res, io) = wasmtime_wasi_http::default_send_request(request, options).await?;
|
||||
Ok((
|
||||
res.map(BodyExt::boxed_unsync),
|
||||
Box::new(io) as Box<dyn Future<Output = _> + Send>,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ActorState {
|
||||
wasi: WasiPart,
|
||||
fsx: FsExt,
|
||||
http: WasiHttpCtx,
|
||||
hooks: HttpHooks,
|
||||
}
|
||||
|
||||
impl WasiView for ActorState {
|
||||
fn ctx(&mut self) -> WasiCtxView<'_> {
|
||||
self.wasi.ctx()
|
||||
}
|
||||
}
|
||||
|
||||
impl WasiHttpView for ActorState {
|
||||
fn http(&mut self) -> WasiHttpCtxView<'_> {
|
||||
WasiHttpCtxView { ctx: &mut self.http, table: &mut self.wasi.table, hooks: &mut self.hooks }
|
||||
}
|
||||
}
|
||||
|
||||
fn actor_fs(s: &mut ActorState) -> ActorFs<'_> {
|
||||
ActorFs {
|
||||
inner: wasmtime_wasi::filesystem::WasiFilesystemView::filesystem(&mut s.wasi),
|
||||
ext: &mut s.fsx,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Runtime {
|
||||
engine: Engine,
|
||||
linker: Linker<ActorState>,
|
||||
}
|
||||
|
||||
/// How an instance ended.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Exit {
|
||||
/// `main` returned or the guest called `exit(0)`.
|
||||
Clean,
|
||||
/// The guest exited with a nonzero code.
|
||||
Code(i32),
|
||||
/// The guest trapped (panic, invalid memory access, ...).
|
||||
Trap(String),
|
||||
}
|
||||
|
||||
impl Exit {
|
||||
pub fn is_clean(&self) -> bool {
|
||||
matches!(self, Exit::Clean)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Exit {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Exit::Clean => write!(f, "exit 0"),
|
||||
Exit::Code(c) => write!(f, "exit {c}"),
|
||||
Exit::Trap(t) => write!(f, "trap: {t}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct InstanceConfig {
|
||||
pub mounts: Vec<Mount>,
|
||||
pub allow_http: bool,
|
||||
pub stderr_log: PathBuf,
|
||||
pub env: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
pub struct Instance {
|
||||
stdin: DuplexStream,
|
||||
/// Lines that the guest writes to stdout. Closed when the guest ends.
|
||||
pub stdout: mpsc::Receiver<String>,
|
||||
task: JoinHandle<Exit>,
|
||||
}
|
||||
|
||||
impl Instance {
|
||||
/// Write one line to the guest's stdin.
|
||||
pub async fn send_line(&mut self, line: &str) -> std::io::Result<()> {
|
||||
self.stdin.write_all(line.as_bytes()).await?;
|
||||
self.stdin.write_all(b"\n").await?;
|
||||
self.stdin.flush().await
|
||||
}
|
||||
|
||||
/// Wait until the guest ends.
|
||||
pub async fn wait(&mut self) -> Exit {
|
||||
match (&mut self.task).await {
|
||||
Ok(exit) => exit,
|
||||
Err(e) if e.is_cancelled() => Exit::Trap("killed".into()),
|
||||
Err(e) => Exit::Trap(format!("host task failed: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_finished(&self) -> bool {
|
||||
self.task.is_finished()
|
||||
}
|
||||
|
||||
/// Send `shutdown`, close stdin, and wait. Kill the guest after `deadline`.
|
||||
pub async fn stop(mut self, shutdown_line: &str, deadline: Duration) -> Exit {
|
||||
let _ = self.send_line(shutdown_line).await;
|
||||
let _ = self.stdin.shutdown().await;
|
||||
drop(self.stdin);
|
||||
match tokio::time::timeout(deadline, &mut self.task).await {
|
||||
Ok(Ok(exit)) => exit,
|
||||
Ok(Err(e)) => Exit::Trap(format!("host task failed: {e}")),
|
||||
Err(_) => {
|
||||
self.task.abort();
|
||||
let _ = (&mut self.task).await;
|
||||
Exit::Trap("killed after shutdown deadline".into())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn log_note(path: &Path, text: &str) {
|
||||
use std::io::Write;
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) {
|
||||
let _ = writeln!(f, "[sausage] {text}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The most useful line of a trap: the `wasm trap: ...` reason.
|
||||
fn short_trap(full: &str) -> String {
|
||||
let line = full
|
||||
.lines()
|
||||
.find_map(|l| l.find("wasm trap:").map(|i| l[i..].to_string()))
|
||||
.unwrap_or_else(|| full.lines().next().unwrap_or("trap").to_string());
|
||||
let mut line = line.trim().to_string();
|
||||
line.truncate(300);
|
||||
line
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
pub fn new() -> Result<Arc<Self>> {
|
||||
let mut config = Config::new();
|
||||
config.epoch_interruption(true);
|
||||
let engine = Engine::new(&config)?;
|
||||
|
||||
let mut linker: Linker<ActorState> = Linker::new(&engine);
|
||||
wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
|
||||
wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?;
|
||||
// Replace the filesystem with the commit-on-close layer.
|
||||
linker.allow_shadowing(true);
|
||||
types::add_to_linker::<ActorState, ActorFsData>(&mut linker, actor_fs)?;
|
||||
preopens::add_to_linker::<ActorState, ActorFsData>(&mut linker, actor_fs)?;
|
||||
linker.allow_shadowing(false);
|
||||
|
||||
// Time slices: every 10 ms each running guest yields to the scheduler.
|
||||
let ticker = engine.clone();
|
||||
std::thread::spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
ticker.increment_epoch();
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Arc::new(Self { engine, linker }))
|
||||
}
|
||||
|
||||
pub fn load(&self, wasm: &Path) -> Result<Component> {
|
||||
Component::from_file(&self.engine, wasm)
|
||||
.map_err(anyhow::Error::from)
|
||||
.with_context(|| format!("load {}", wasm.display()))
|
||||
}
|
||||
|
||||
pub fn start(self: &Arc<Self>, component: &Component, cfg: InstanceConfig) -> Result<Instance> {
|
||||
let (host_in, guest_in) = tokio::io::duplex(64 * 1024);
|
||||
let (guest_out, host_out) = tokio::io::duplex(64 * 1024);
|
||||
|
||||
let stderr = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&cfg.stderr_log)
|
||||
.with_context(|| format!("open {}", cfg.stderr_log.display()))?;
|
||||
|
||||
let mut b = WasiCtx::builder();
|
||||
b.stdin(AsyncStdinStream::new(guest_in));
|
||||
b.stdout(AsyncStdoutStream::new(8192, guest_out));
|
||||
b.stderr(AsyncStdoutStream::new(8192, tokio::fs::File::from_std(stderr)));
|
||||
for (k, v) in &cfg.env {
|
||||
b.env(k, v);
|
||||
}
|
||||
for m in &cfg.mounts {
|
||||
let perms = if m.writable { FsPerms::ReadWrite } else { FsPerms::ReadOnly };
|
||||
b.preopened_dir(&m.host, &m.guest, perms)
|
||||
.map_err(anyhow::Error::from)
|
||||
.with_context(|| format!("mount {} at {}", m.host.display(), m.guest))?;
|
||||
}
|
||||
let state = ActorState {
|
||||
wasi: WasiPart { ctx: b.build(), table: ResourceTable::new() },
|
||||
fsx: FsExt::new(cfg.mounts),
|
||||
http: WasiHttpCtx::new(),
|
||||
hooks: HttpHooks { allow: cfg.allow_http },
|
||||
};
|
||||
|
||||
let (line_tx, line_rx) = mpsc::channel(1024);
|
||||
tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(host_out).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
if line_tx.send(line).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let rt = self.clone();
|
||||
let component = component.clone();
|
||||
let stderr_log = cfg.stderr_log.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut store = Store::new(&rt.engine, state);
|
||||
store.epoch_deadline_async_yield_and_update(1);
|
||||
let result = async {
|
||||
let cmd = Command::instantiate_async(&mut store, &component, &rt.linker).await?;
|
||||
cmd.wasi_cli_run().call_run(&mut store).await
|
||||
}
|
||||
.await;
|
||||
let exit = match result {
|
||||
Ok(Ok(())) => Exit::Clean,
|
||||
Ok(Err(())) => Exit::Code(1),
|
||||
Err(e) => match e.downcast_ref::<I32Exit>() {
|
||||
Some(I32Exit(0)) => Exit::Clean,
|
||||
Some(I32Exit(c)) => Exit::Code(*c),
|
||||
None => {
|
||||
// Keep the full backtrace in the log, and a short reason in the status.
|
||||
let full = format!("{e:?}");
|
||||
log_note(&stderr_log, &format!("trap details:\n{full}"));
|
||||
Exit::Trap(short_trap(&full))
|
||||
}
|
||||
},
|
||||
};
|
||||
// Process exit closes every file, so a clean exit commits.
|
||||
if exit.is_clean() {
|
||||
store.data_mut().fsx.commit_all();
|
||||
}
|
||||
exit
|
||||
});
|
||||
|
||||
Ok(Instance { stdin: host_in, stdout: line_rx, task })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! On-disk layout of a Sausage workspace ("world").
|
||||
//!
|
||||
//! ```text
|
||||
//! <world>/
|
||||
//! sausage.json registry: actors, versions, subscriptions
|
||||
//! actor/<id>/public/ written by the actor, read-only for others
|
||||
//! actor/<id>/source/ source of the running version (broker)
|
||||
//! actor/<id>/status.json state of the actor (broker)
|
||||
//! actor/<id>/stderr.log stderr of the actor (broker)
|
||||
//! private/<id>/data/ private state of the actor
|
||||
//! broker/versions/<id>/<n>/ actor.wasm, source/, meta.json, snapshot/
|
||||
//! archive/ removed actors
|
||||
//! ```
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::fs::TMP_PREFIX;
|
||||
|
||||
/// IDs of the actors that the host implements natively.
|
||||
pub const NATIVE: &[&str] = &["broker", "console", "cargo"];
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct World {
|
||||
pub root: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize, Clone)]
|
||||
pub struct Registry {
|
||||
pub actors: BTreeMap<String, ActorRecord>,
|
||||
/// Publisher ID → subscriber IDs.
|
||||
pub subscriptions: BTreeMap<String, BTreeSet<String>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct ActorRecord {
|
||||
pub version: u64,
|
||||
pub next_version: u64,
|
||||
pub running: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
||||
pub struct VersionMeta {
|
||||
pub version: u64,
|
||||
/// Path of the `.wasm` file, as the requester gave it.
|
||||
pub wasm: String,
|
||||
/// Path of the source folder, as the requester gave it.
|
||||
pub source: Option<String>,
|
||||
/// The actor that built the `.wasm` file.
|
||||
pub built_by: Option<String>,
|
||||
/// The actor that requested the spawn or upgrade.
|
||||
pub requested_by: String,
|
||||
pub created_at: u64,
|
||||
}
|
||||
|
||||
pub fn valid_id(id: &str) -> bool {
|
||||
!id.is_empty()
|
||||
&& id.len() <= 64
|
||||
&& id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_')
|
||||
}
|
||||
|
||||
pub fn now() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
impl World {
|
||||
pub fn new(root: impl Into<PathBuf>) -> Self {
|
||||
Self { root: root.into() }
|
||||
}
|
||||
|
||||
pub fn registry_path(&self) -> PathBuf {
|
||||
self.root.join("sausage.json")
|
||||
}
|
||||
pub fn actor_dir(&self, id: &str) -> PathBuf {
|
||||
self.root.join("actor").join(id)
|
||||
}
|
||||
pub fn actors_root(&self) -> PathBuf {
|
||||
self.root.join("actor")
|
||||
}
|
||||
pub fn public_dir(&self, id: &str) -> PathBuf {
|
||||
self.actor_dir(id).join("public")
|
||||
}
|
||||
pub fn source_dir(&self, id: &str) -> PathBuf {
|
||||
self.actor_dir(id).join("source")
|
||||
}
|
||||
pub fn data_dir(&self, id: &str) -> PathBuf {
|
||||
self.root.join("private").join(id).join("data")
|
||||
}
|
||||
pub fn versions_dir(&self, id: &str) -> PathBuf {
|
||||
self.root.join("broker").join("versions").join(id)
|
||||
}
|
||||
pub fn version_dir(&self, id: &str, v: u64) -> PathBuf {
|
||||
self.versions_dir(id).join(v.to_string())
|
||||
}
|
||||
pub fn scratch_dir(&self) -> PathBuf {
|
||||
self.root.join("broker").join("scratch")
|
||||
}
|
||||
|
||||
pub fn exists(&self) -> bool {
|
||||
self.registry_path().is_file()
|
||||
}
|
||||
|
||||
/// Create an empty world with the folders of the native actors.
|
||||
pub fn create(root: &Path) -> Result<World> {
|
||||
let world = World::new(std::path::absolute(root)?);
|
||||
if world.exists() {
|
||||
bail!("{} is already a Sausage world", root.display());
|
||||
}
|
||||
std::fs::create_dir_all(&world.root)?;
|
||||
world.save_registry(&Registry::default())?;
|
||||
for id in NATIVE {
|
||||
world.ensure_actor_dirs(id)?;
|
||||
let status = serde_json::json!({"id": id, "kind": "native", "state": "built-in"});
|
||||
write_atomic(&world.actor_dir(id).join("status.json"), serde_json::to_string_pretty(&status)?.as_bytes())?;
|
||||
}
|
||||
Ok(world)
|
||||
}
|
||||
|
||||
/// Create the directories of one actor.
|
||||
pub fn ensure_actor_dirs(&self, id: &str) -> Result<()> {
|
||||
std::fs::create_dir_all(self.public_dir(id))?;
|
||||
std::fs::create_dir_all(self.data_dir(id))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn load_registry(&self) -> Result<Registry> {
|
||||
let text = std::fs::read_to_string(self.registry_path())
|
||||
.with_context(|| format!("no Sausage world at {}", self.root.display()))?;
|
||||
Ok(serde_json::from_str(&text)?)
|
||||
}
|
||||
|
||||
pub fn save_registry(&self, reg: &Registry) -> Result<()> {
|
||||
write_atomic(&self.registry_path(), serde_json::to_string_pretty(reg)?.as_bytes())
|
||||
}
|
||||
|
||||
/// Map a path in an actor's namespace to a host path.
|
||||
///
|
||||
/// - `/actor/<id>/...` is the global filesystem.
|
||||
/// - `/public/...` and `/data/...` belong to `sender`.
|
||||
/// - Native senders (the console) can also give absolute host paths.
|
||||
pub fn resolve(&self, sender: &str, path: &str) -> Result<PathBuf> {
|
||||
let clean = |base: PathBuf, rest: &str| -> Result<PathBuf> {
|
||||
let mut out = base.clone();
|
||||
for c in Path::new(rest).components() {
|
||||
match c {
|
||||
std::path::Component::Normal(n) => out.push(n),
|
||||
std::path::Component::CurDir | std::path::Component::RootDir => {}
|
||||
_ => bail!("path must not contain '..': {path}"),
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
};
|
||||
if let Some(rest) = path.strip_prefix("/actor/") {
|
||||
return clean(self.actors_root(), rest);
|
||||
}
|
||||
if let Some(rest) = path.strip_prefix("/public/").or(path.strip_prefix("/public").filter(|r| r.is_empty())) {
|
||||
return clean(self.public_dir(sender), rest);
|
||||
}
|
||||
if let Some(rest) = path.strip_prefix("/data/").or(path.strip_prefix("/data").filter(|r| r.is_empty())) {
|
||||
return clean(self.data_dir(sender), rest);
|
||||
}
|
||||
if NATIVE.contains(&sender) && Path::new(path).is_absolute() {
|
||||
return Ok(PathBuf::from(path));
|
||||
}
|
||||
bail!("cannot resolve path {path} for {sender}: use /actor/<id>/..., /public/... or /data/...")
|
||||
}
|
||||
|
||||
/// The path of a host file in the global filesystem, if it has one.
|
||||
pub fn global_path(&self, host: &Path) -> Option<String> {
|
||||
let rel = host.strip_prefix(self.actors_root()).ok()?;
|
||||
Some(format!("/actor/{}", rel.display()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a file so that readers never see a partial version.
|
||||
pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
|
||||
let dir = path.parent().context("path has no parent")?;
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let tmp = dir.join(format!(
|
||||
"{TMP_PREFIX}host-{}-{}",
|
||||
std::process::id(),
|
||||
path.file_name().and_then(|n| n.to_str()).unwrap_or("file")
|
||||
));
|
||||
std::fs::write(&tmp, bytes)?;
|
||||
std::fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Copy a directory tree. Skips `target/` folders and uncommitted files.
|
||||
pub fn copy_tree(from: &Path, to: &Path) -> Result<()> {
|
||||
std::fs::create_dir_all(to)?;
|
||||
for entry in std::fs::read_dir(from).with_context(|| format!("read {}", from.display()))? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with(TMP_PREFIX) {
|
||||
continue;
|
||||
}
|
||||
let ty = entry.file_type()?;
|
||||
let dst = to.join(&name);
|
||||
if ty.is_dir() {
|
||||
if name_str == "target" || name_str == ".git" {
|
||||
continue;
|
||||
}
|
||||
copy_tree(&entry.path(), &dst)?;
|
||||
} else if ty.is_file() {
|
||||
std::fs::copy(entry.path(), &dst)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the directory `dst` with a copy of `src`, as close to atomic as a
|
||||
/// directory can be: copy to a sibling, then swap with two renames.
|
||||
pub fn replace_tree(src: &Path, dst: &Path) -> Result<()> {
|
||||
let parent = dst.parent().context("path has no parent")?;
|
||||
let name = dst.file_name().and_then(|n| n.to_str()).unwrap_or("dir");
|
||||
let fresh = parent.join(format!("{TMP_PREFIX}new-{name}"));
|
||||
let old = parent.join(format!("{TMP_PREFIX}old-{name}"));
|
||||
let _ = std::fs::remove_dir_all(&fresh);
|
||||
let _ = std::fs::remove_dir_all(&old);
|
||||
if src.is_dir() {
|
||||
copy_tree(src, &fresh)?;
|
||||
} else {
|
||||
std::fs::create_dir_all(&fresh)?;
|
||||
}
|
||||
if dst.exists() {
|
||||
std::fs::rename(dst, &old)?;
|
||||
}
|
||||
std::fs::rename(&fresh, dst)?;
|
||||
let _ = std::fs::remove_dir_all(&old);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user