surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Host: fixes from the review, with regression tests

Browse Source 
- one live instance per actor: check after the lock; dropping an instance kills it
- state snapshots keep target/ and .git/; only source copies skip them
- one shutdown deadline; output during shutdown is still routed
- deliveries route stdout meanwhile; a stuck stdin kills the instance
  (no partial lines) and the sender gets undeliverable; control requests
  are not blocked by a stuck delivery
- Host::stop aborts running broker and cargo requests first
- compile a version before installing it; roll back if it does not start
- unique archive names; a failed archive keeps the actor manageable
- commit on close: truncate without create; directory renames
- candidates see SAUSAGE_VERSION and the existing state of a new actor
- the host publishes PROTOCOL.md at /actor/broker/public/
This commit is contained in:
surma
2026-10-15 12:03:26 +01:00
parent 69cf80a114
commit 9d575b6a62
11 changed files with 811 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/broker.rs
+129 -70
View File
@@ -241,9 +241,17 @@ impl Broker {
let dir = self.world.version_dir(id, meta.version);
std::fs::create_dir_all(&dir)?;
std::fs::copy(wasm, dir.join("actor.wasm")).with_context(|| format!("copy {}", wasm.display()))?;
world::replace_tree(source.unwrap_or(&dir.join("no-source")), &dir.join("source"))?;
world::replace_source(source.unwrap_or(&dir.join("no-source")), &dir.join("source"))?;
world::write_atomic(&dir.join("meta.json"), serde_json::to_string_pretty(meta)?.as_bytes())?;
self.runtime.load(&dir.join("actor.wasm"))
self.load_version(id, meta.version)
}
/// Compile `versions/<version>/actor.wasm`.
pub fn load_version(&self, id: &str, version: u64) -> Result<Component> {
let started = Instant::now();
let component = self.runtime.load(&self.world.version_dir(id, version).join("actor.wasm"))?;
self.log(id, &format!("compiled version {version} in {} ms", started.elapsed().as_millis()));
Ok(component)
}
/// Record in `versions/<n>/meta.json` why the broker refused a version.
@@ -282,11 +290,11 @@ impl Broker {
if !world::valid_id(id) {
bail!("invalid actor ID {id:?}: use a-z, 0-9, '-' and '_'");
}
let lock = self.op_lock(id);
let _guard = lock.lock().await;
if NATIVE.contains(&id) || self.is_supervised(id) || self.world.wasm_path(id).exists() {
bail!("an actor with ID {id} already exists");
}
let lock = self.op_lock(id);
let _guard = lock.lock().await;
let version = self.world.next_version(id);
let meta = VersionMeta { version, ..meta };
self.create_version(id, version, &meta, wasm, source, start).await
@@ -295,7 +303,8 @@ impl Broker {
async fn create_version(self: &Arc<Self>, id: &str, version: u64, meta: &VersionMeta, wasm: &Path, source: Option<&Path>, start: bool) -> Result<Value> {
let tested = async {
let component = self.store_version(id, meta, wasm, source)?;
candidate::test(self, id, &component, &self.world.version_dir(id, version).join("source"), None, None).await
let source = self.world.version_dir(id, version).join("source");
candidate::test(self, id, version, &component, &source, &self.world.data_dir(id), &self.world.public_dir(id)).await
}
.await;
let report = match tested {
@@ -328,7 +337,7 @@ impl Broker {
let tested = async {
let component = self.store_version(id, meta, wasm, source)?;
let source = self.world.version_dir(id, version).join("source");
candidate::test(self, id, &component, &source, Some(&self.world.data_dir(id)), Some(&self.world.public_dir(id))).await
candidate::test(self, id, version, &component, &source, &self.world.data_dir(id), &self.world.public_dir(id)).await
}
.await;
let report = match tested {
@@ -405,10 +414,13 @@ impl Broker {
let source = source.is_dir().then_some(source.as_path());
let result = match installed {
None => self.create_version(id, version, &vmeta, &wasm, source, true).await,
Some(current) => {
Some(_) => {
let r = self.upgrade_version(id, &vmeta, &wasm, source).await;
if r.is_err() {
// Put the installed version back in place.
// Put the installed version back in place. Read it again: an
// automatic rollback can have changed it meanwhile.
if r.is_err()
&& let Some(current) = self.meta(id).and_then(|m| m.version)
{
let _ = self.restore_files(id, current);
}
r
@@ -527,10 +539,18 @@ impl Broker {
}
}
/// The native `broker` actor.
/// The native `broker` actor. Requests run concurrently. Aborting this
/// task aborts the requests that still run.
pub async fn run_actor(self: Arc<Self>, mut mailbox: mpsc::Receiver<Envelope>) {
while let Some(env) = mailbox.recv().await {
tokio::spawn(self.clone().handle(env));
let mut requests = tokio::task::JoinSet::new();
loop {
tokio::select! {
env = mailbox.recv() => match env {
Some(env) => { requests.spawn(self.clone().handle(env)); }
None => break,
},
Some(_) = requests.join_next() => {}
}
}
}
}
@@ -586,6 +606,8 @@ struct Supervisor {
swapped_at: Option<Instant>,
crashes_since_swap: usize,
upgrade_requester: Option<String>,
/// A control request that arrived during a blocked delivery.
next_control: Option<Control>,
}
async fn recv_line(instance: &mut Option<Instance>) -> Option<String> {
@@ -632,6 +654,7 @@ impl Supervisor {
swapped_at: None,
crashes_since_swap: 0,
upgrade_requester: None,
next_control: None,
}
}
@@ -658,9 +681,15 @@ impl Supervisor {
async fn run(mut self) {
self.status();
loop {
if let Some(ctl) = self.next_control.take() {
if self.handle_control(ctl).await {
break;
}
continue;
}
if self.desired_running && self.instance.is_none() && self.restart_at.is_none_or(|t| Instant::now() >= t) {
self.restart_at = None;
self.start_instance().await;
let _ = self.start_instance().await;
}
let has_instance = self.instance.is_some();
let waiting = !has_instance && self.desired_running;
@@ -690,36 +719,65 @@ impl Supervisor {
}
}
/// Write a message to the instance's stdin, and route its output
/// meanwhile. An instance that does not read its stdin for
/// `STDIN_WRITE_TIMEOUT`, or while a control request waits, is killed and
/// restarted: a partly written line would corrupt the next message.
async fn deliver(&mut self, msg: Envelope) {
let Some(inst) = self.instance.as_mut() else { return };
let line = msg.to_line();
match tokio::time::timeout(STDIN_WRITE_TIMEOUT, inst.send_line(&line)).await {
Ok(Ok(())) => self.last_delivered = Some(msg),
Ok(Err(_)) => {} // The instance is ending. The exit path handles it.
Err(_) => {
let note = format!("actor did not read stdin for {STDIN_WRITE_TIMEOUT:?}; message {} dropped", msg.id);
self.broker.log(&self.id, &note);
let (broker, id, line) = (self.broker.clone(), self.id.clone(), msg.to_line());
let write = tokio::time::timeout(STDIN_WRITE_TIMEOUT, inst.send_line(&line, |l| broker.route_line(&id, &l)));
let failure = tokio::select! {
written = write => match written {
Ok(Ok(())) => {
self.last_delivered = Some(msg);
return;
}
// The instance is ending. The main loop sees its closed stdout.
Ok(Err(_)) => None,
Err(_) => Some(format!("the actor did not read stdin for {STDIN_WRITE_TIMEOUT:?}")),
},
Some(ctl) = self.control.recv() => {
self.next_control = Some(ctl);
Some("the actor did not read stdin when the broker needed it".to_string())
}
};
if msg.from != "broker" {
let reason = failure.as_deref().unwrap_or("the actor stopped");
let body = json!({"type": "undeliverable", "reason": reason, "to": self.id});
self.broker.send("broker", Some(&msg.from), None, Some(&msg.id), body);
}
if let (Some(failure), Some(inst)) = (failure, &self.instance) {
// The main loop sees the end of the instance and restarts it.
self.broker.log(&self.id, &format!("killed: {failure}"));
inst.kill();
}
}
async fn start_instance(&mut self) {
/// Start an instance of the current version. On failure, the actor is
/// "broken" until the next start, upgrade, or rollback.
async fn start_instance(&mut self) -> Result<()> {
let started = self.try_start();
if let Err(e) = &started {
self.broker.log(&self.id, &format!("cannot start version {}: {e:#}", self.version));
self.state = "broken";
self.desired_running = false;
self.status();
return started;
}
self.state = "running";
self.started_at = Instant::now();
self.status();
if let Some(msg) = self.redeliver.take() {
self.broker.log(&self.id, &format!("delivering parked message {} again", msg.id));
self.deliver(msg).await;
}
Ok(())
}
fn try_start(&mut self) -> Result<()> {
if self.component.is_none() {
let path = self.world().version_dir(&self.id, self.version).join("actor.wasm");
let t = Instant::now();
match self.broker.runtime.load(&path) {
Ok(c) => {
self.broker.log(&self.id, &format!("compiled version {} in {} ms", self.version, t.elapsed().as_millis()));
self.component = Some(c);
}
Err(e) => {
self.broker.log(&self.id, &format!("cannot load version {}: {e:#}", self.version));
self.state = "broken";
self.desired_running = false;
self.status();
return;
}
}
self.component = Some(self.broker.load_version(&self.id, self.version)?);
}
let cfg = InstanceConfig {
mounts: self.broker.mounts(&self.id),
@@ -730,29 +788,15 @@ impl Supervisor {
("SAUSAGE_VERSION".into(), self.version.to_string()),
],
};
match self.broker.runtime.start(self.component.as_ref().unwrap(), cfg) {
Ok(inst) => {
self.instance = Some(inst);
self.state = "running";
self.started_at = Instant::now();
self.status();
if let Some(msg) = self.redeliver.take() {
self.broker.log(&self.id, &format!("delivering parked message {} again", msg.id));
self.deliver(msg).await;
}
}
Err(e) => {
self.broker.log(&self.id, &format!("cannot start: {e:#}"));
self.state = "broken";
self.desired_running = false;
self.status();
}
}
self.instance = Some(self.broker.runtime.start(self.component.as_ref().unwrap(), cfg)?);
Ok(())
}
/// Stop the instance gracefully. Its last messages still get routed.
async fn stop_instance(&mut self) -> Option<Exit> {
let inst = self.instance.take()?;
let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
let (broker, id) = (self.broker.clone(), self.id.clone());
let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |line| broker.route_line(&id, &line)).await;
self.broker.log(&self.id, &format!("stopped: {exit}"));
self.last_exit = Some(exit.to_string());
Some(exit)
@@ -801,37 +845,48 @@ impl Supervisor {
Ok(())
}
/// Make `version` the current files: `actor.wasm`, `source/`, `actor.json`.
/// Make `version` current: compile it first, then write `actor.wasm`,
/// `source/`, and `actor.json`. If compiling fails, nothing changes.
fn install(&mut self, version: u64) -> Result<()> {
let component = self.broker.load_version(&self.id, version)?;
let dir = self.world().version_dir(&self.id, version);
world::write_atomic(&self.world().wasm_path(&self.id), &std::fs::read(dir.join("actor.wasm"))?)?;
world::replace_tree(&dir.join("source"), &self.world().source_dir(&self.id))?;
self.version = version;
self.component = None;
self.broker.update_meta(&self.id, |m| {
m.version = Some(version);
m.running = true;
})
})?;
self.version = version;
self.component = Some(component);
Ok(())
}
fn set_running(&self, running: bool) -> Result<()> {
self.broker.update_meta(&self.id, |m| m.running = running)
}
/// Clear the restart and crash-loop state before a swap.
fn reset_restarts(&mut self) {
self.restarts = 0;
self.restart_at = None;
self.crashes_since_swap = 0;
self.desired_running = true;
}
async fn upgrade(&mut self, version: u64, requester: String) -> Result<Value> {
self.stop_instance().await;
self.snapshot_current()?;
let previous = self.version;
self.install(version)?;
self.reset_restarts();
self.previous = Some(previous);
self.swapped_at = Some(Instant::now());
self.crashes_since_swap = 0;
self.restarts = 0;
self.restart_at = None;
self.upgrade_requester = Some(requester);
self.redeliver = self.parked.take();
self.desired_running = true;
self.start_instance().await;
if let Err(e) = self.start_instance().await {
self.rollback(Some(previous)).await?;
bail!("version {version} did not start ({e:#}); rolled back to {previous}");
}
Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous}))
}
@@ -847,6 +902,7 @@ impl Supervisor {
if !dir.join("actor.wasm").is_file() {
bail!("{} has no version {target}", self.id);
}
self.broker.load_version(&self.id, target)?;
self.stop_instance().await;
self.snapshot_current()?;
let from = self.version;
@@ -856,12 +912,10 @@ impl Supervisor {
world::replace_tree(&snap.join("public"), &self.world().public_dir(&self.id))?;
}
self.install(target)?;
self.reset_restarts();
self.previous = None;
self.swapped_at = None;
self.restarts = 0;
self.restart_at = None;
self.desired_running = true;
self.start_instance().await;
self.start_instance().await?;
Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": target, "restored_state": snap.is_dir()}))
}
@@ -894,8 +948,12 @@ impl Supervisor {
self.desired_running = false;
self.stop_instance().await;
let r = self.archive();
let removed = r.is_ok();
let _ = reply.send(r);
return true;
if removed {
return true;
}
self.state = "stopped";
}
Control::Shutdown { reply } => {
self.stop_instance().await;
@@ -913,7 +971,8 @@ impl Supervisor {
fn archive(&self) -> Result<Value> {
let archive = self.world().root.join("archive");
std::fs::create_dir_all(&archive)?;
let dest = archive.join(format!("{}-{}", self.id, world::now()));
let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos();
let dest = archive.join(format!("{}-{nanos}", self.id));
std::fs::rename(self.world().actor_dir(&self.id), &dest)?;
Ok(json!({"type": "removed", "actor": self.id, "archive": dest.display().to_string()}))
}