core: sausage-vfs per process (/, /proc, /self, /run), activation, graph, watcher, upgrades

- /proc is a core layer: core/ (PROTOCOL.md, graph.json, graph.dot,
  locks.json, plugins/<name>/discover.json and client.rs) and one folder
  per process (info.json, log, and its /self files).
- Locks are released only after the task ended, the store dropped, and
  in-flight file operations returned.
- core.activate and core.revert with copies, discovery, compatibility
  checks of imports and consumers, groups, force. An admission gate
  makes the snapshot of old processes complete.
- Graph of discover documents by hash, refreshed by a file watcher.
- Upgrades: grace periods (x-sausage-grace), stop, service restarts,
  rpc.retired.
- SDK plugins in the real core (cli/tests/sdk.rs).
This commit is contained in:
2026-10-23 01:05:19 +01:00
parent 151550242b
commit edc2d1a63b
21 changed files with 2306 additions and 101 deletions
+204 -9
View File
@@ -1,15 +1,210 @@
//! Activation and revert of plugin versions (PROTOCOL.md section 10.3).
use crate::{Host, RpcResult, error};
use serde_json::Value;
//! Serialized candidate-copy activation and revert under PROTOCOL.md section 10.3.
use crate::{
Host, RpcResult, error,
graph::{Interface, io_error, problems},
};
use serde_json::{Value, json};
use std::{collections::BTreeMap, path::PathBuf};
struct Candidate {
plugin: String,
copy: PathBuf,
interface: Interface,
}
impl Host {
/// `core.activate` and `core.revert`.
pub(crate) async fn activation(
/// Copies first, discovers that copy, checks the combined graph, then installs it.
pub(crate) async fn activation(&self, _caller: &str, method: &str, params: Value) -> RpcResult {
let _activation = self.0.activations.lock().await;
let mut candidates = Vec::new();
let result = self
.activate_candidates(method, &params, &mut candidates)
.await;
for candidate in candidates {
let _ = tokio::fs::remove_file(candidate.copy).await;
}
result
}
/// Prepares every member before graph checks or changes to installed files.
async fn activate_candidates(
&self,
_caller: &str,
_method: &str,
_params: Value,
method: &str,
params: &Value,
candidates: &mut Vec<Candidate>,
) -> RpcResult {
Err(error(-32601, "method not found"))
let requests = params
.get("plugins")
.and_then(Value::as_array)
.cloned()
.unwrap_or_else(|| vec![params.clone()]);
if requests.is_empty() {
return Err(error(-32602, "plugins must not be empty"));
}
for request in requests {
let plugin = request["plugin"]
.as_str()
.ok_or_else(|| error(-32602, "plugin must be a string"))?;
if plugin.is_empty()
|| !plugin.bytes().all(|byte| {
byte.is_ascii_lowercase()
|| byte.is_ascii_digit()
|| byte == b'_'
|| byte == b'-'
})
|| candidates
.iter()
.any(|candidate| candidate.plugin == plugin)
{
return Err(error(-32602, "invalid or duplicate plugin"));
}
let directory = self.0.world.plugin_dir(plugin);
tokio::fs::create_dir_all(&directory)
.await
.map_err(io_error)?;
let source = if method == "revert" {
directory.join("plugin.prev.wasm")
} else {
let source = PathBuf::from(
request["wasm"]
.as_str()
.ok_or_else(|| error(-32602, "wasm must be a string"))?,
);
if source.starts_with(&self.0.world.root) {
source
} else {
self.0
.world
.root
.join(source.strip_prefix("/").unwrap_or(&source))
}
};
let copy = directory.join(format!("activate-{}.wasm", uuid::Uuid::now_v7()));
// Read once. Discovery and installation both use the same private bytes.
let bytes = tokio::fs::read(source).await.map_err(io_error)?;
tokio::fs::write(&copy, &bytes).await.map_err(io_error)?;
self.0
.journal
.write("core", plugin, "core", "activation copied candidate");
let discovery = self.discover_copy(plugin, &copy, &bytes).await;
let interface = match discovery {
Ok(interface) => interface,
Err(error) => {
let _ = tokio::fs::remove_file(copy).await;
return Err(error);
}
};
self.0
.journal
.write("core", plugin, "core", "activation discovered candidate");
candidates.push(Candidate {
plugin: plugin.into(),
copy,
interface,
});
}
self.refresh_graph().await?;
let previous = self.0.graph.lock().unwrap().plugins.clone();
if previous.values().any(|entry| entry.error.is_some()) {
return Err(error(
-32003,
"activation waits for successful discovery of every plugin",
));
}
let mut proposed = previous.clone();
let names: Vec<_> = candidates
.iter()
.map(|candidate| candidate.plugin.clone())
.collect();
for candidate in candidates.iter() {
proposed.insert(candidate.plugin.clone(), candidate.interface.clone());
}
if params["force"] != true {
let failures = problems(&proposed, Some(&names));
if !failures.is_empty() {
self.0.journal.write(
"core",
"core",
"core",
&format!("activation rejected {}", json!(failures)),
);
let details = failures
.iter()
.map(|problem| {
format!(
"- {} imports {}: {}",
problem["consumer"],
problem["provider"],
problem["display"]
.as_str()
.unwrap_or_else(|| problem["message"]
.as_str()
.unwrap_or("incompatible import"))
)
})
.collect::<Vec<_>>()
.join("\n");
return Err(
json!({"code":-32003,"message":format!("{} was not activated: consumers break.\n{details}\nUpdate consumers first, or activate both together.",names.join(", ")),"data":failures}),
);
}
}
self.install_candidates(candidates, previous, proposed)
.await
}
/// Installs staged files and snapshots all old admissions under the shared write gate.
async fn install_candidates(
&self,
candidates: &[Candidate],
previous: BTreeMap<String, Interface>,
proposed: BTreeMap<String, Interface>,
) -> RpcResult {
let admission = self.0.admission.write().await;
// Every candidate already has a temporary name before any installed file changes.
for candidate in candidates {
let directory = self.0.world.plugin_dir(&candidate.plugin);
let current = directory.join("plugin.wasm");
if tokio::fs::try_exists(&current).await.map_err(io_error)? {
tokio::fs::rename(&current, directory.join("plugin.prev.wasm"))
.await
.map_err(io_error)?;
}
tokio::fs::rename(&candidate.copy, &current)
.await
.map_err(io_error)?;
}
let mut upgrades = Vec::new();
let mut documents = serde_json::Map::new();
for candidate in candidates {
documents.insert(
candidate.plugin.clone(),
candidate.interface.document.clone(),
);
if let Some(old) = previous
.get(&candidate.plugin)
.filter(|old| old.hash != candidate.interface.hash)
{
let grace = old.document["x-sausage-grace"]
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.unwrap_or(60.0);
upgrades.push(self.snapshot_upgrade(
&candidate.plugin,
old.hash.clone(),
&candidate.interface.hash,
grace,
));
}
self.0.journal.write(
"core",
&candidate.plugin,
"core",
&format!("activation installed {}", candidate.interface.hash),
);
}
self.publish_graph(proposed);
drop(admission);
for upgrade in upgrades {
self.begin_upgrade(upgrade);
}
Ok(Value::Object(documents))
}
}
+13 -1
View File
@@ -31,7 +31,19 @@ impl Host {
Ok(Value::Null)
}
"kill" => {
self.kill(required_str(&p, "pid")?);
let pid = required_str(&p, "pid")?;
let ended = self
.0
.state
.lock()
.unwrap()
.processes
.get(pid)
.map(|p| p.ended.subscribe());
self.kill(pid);
if let Some(mut ended) = ended {
let _ = ended.wait_for(|ended| *ended).await;
}
Ok(json!({}))
}
"stop" => {
+11 -10
View File
@@ -1,7 +1,7 @@
//! Execution of native futures and WASI processes, including transport drainage.
use crate::{
Host, NativeContext, NativePlugin,
process::{ProcessGuard, ProcessInfo},
process::ProcessInfo,
runtime::{Instance, InstanceConfig},
};
use serde_json::{Value, json};
@@ -17,11 +17,7 @@ impl Host {
native: Option<NativePlugin>,
incoming: mpsc::UnboundedReceiver<Value>,
has_id: bool,
) {
let _guard = ProcessGuard {
host: self.clone(),
pid: pid.clone(),
};
) -> String {
if let Some(native) = native {
let context = NativeContext {
host: self.clone(),
@@ -29,13 +25,12 @@ impl Host {
};
let result = native(context, info.method, info.params).await;
self.answer(&pid, result);
self.finish(&pid, "exit 0");
return;
return "exit 0".into();
}
let result = self
.run_guest(&pid, &info, &component.unwrap(), incoming, has_id)
.await;
self.finish(&pid, &result.unwrap_or_else(|error| error.to_string()));
result.unwrap_or_else(|error| error.to_string())
}
/// Drain stdout before deciding whether the guest returned an answer.
@@ -47,8 +42,14 @@ impl Host {
mut incoming: mpsc::UnboundedReceiver<Value>,
has_id: bool,
) -> anyhow::Result<String> {
let (memory, tracker, store_ended) = {
let state = self.0.state.lock().unwrap();
let p = state.processes.get(pid).unwrap();
(p.memory.clone(), p.files.clone(), p.store_ended.clone())
};
let config = InstanceConfig {
world: self.0.world.root.clone(),
filesystem: self.mounts(Some(pid.into()), memory, tracker).await?,
store_ended,
network: true,
env: vec![
("SAUSAGE_PID".into(), pid.into()),
+301
View File
@@ -0,0 +1,301 @@
//! Per-process mount tables, reserved self files, and the live process namespace.
use crate::Host;
use sausage_vfs::{
Entry, File, Generated, Layer, Memory, Mounts, Open, Process, Result, Stat, async_trait,
};
use std::{io, sync::Arc};
fn denied<T>() -> Result<T> {
Err(io::ErrorKind::ReadOnlyFilesystem.into())
}
fn missing<T>() -> Result<T> {
Err(io::ErrorKind::NotFound.into())
}
fn directory() -> Stat {
Stat {
directory: true,
symlink: false,
size: 0,
accessed: None,
modified: None,
}
}
fn entry(name: String, directory: bool) -> Entry {
Entry {
name,
directory,
symlink: false,
}
}
fn reserved(path: &str) -> bool {
matches!(
path.trim_start_matches('/').split('/').next(),
Some("info.json" | "log" | "fs")
)
}
impl Host {
/// Create a console filesystem view without adding a process. Its `/self` is private and ephemeral.
pub async fn filesystem(&self) -> anyhow::Result<Mounts> {
self.mounts(None, Memory::default(), Process::default())
.await
}
pub(crate) async fn mounts(
&self,
pid: Option<String>,
memory: Memory,
tracker: Process,
) -> anyhow::Result<Mounts> {
let root = self.0.world.root.clone();
let host = tracker
.clone()
.blocking(move || sausage_vfs::Host::new(root, tracker))
.await?;
let mut mounts = Mounts::default();
mounts.mount("/", Arc::new(host), false);
mounts.mount("/proc", Arc::new(Proc(self.clone())), true);
mounts.mount(
"/self",
Arc::new(SelfFolder {
host: self.clone(),
pid,
memory,
}),
false,
);
mounts.mount("/run", Arc::new(self.0.run.clone()), false);
Ok(mounts)
}
fn core_files(&self) -> Generated {
let mut files = Generated::default();
let protocol = include_bytes!("../../PROTOCOL.md");
files.insert("/PROTOCOL.md", move || Ok(protocol.to_vec()));
let graph = self.graph_json();
files.insert("/graph.json", move || Ok(graph.clone()));
let dot = self.graph_dot();
files.insert("/graph.dot", move || Ok(dot.clone()));
let locks = {
let state = self.0.state.lock().unwrap();
let locks: std::collections::BTreeMap<_, _> = state
.locks
.iter()
.map(|(path, lock)| {
let waiting: Vec<_> = lock.queue.iter().map(|(pid, _)| pid).collect();
let info = serde_json::json!({
"owner": lock.owner,
"waiting": waiting,
});
(path.clone(), info)
})
.collect();
serde_json::to_vec(&locks).unwrap()
};
files.insert("/locks.json", move || Ok(locks.clone()));
for name in self.discovered_plugins() {
if let Some(doc) = self.discover_document(&name) {
let bytes = serde_json::to_vec(&doc).unwrap();
files.insert(&format!("/plugins/{name}/discover.json"), move || {
Ok(bytes.clone())
});
files.insert(&format!("/plugins/{name}/client.rs"), move || {
sausage_interface::client_rs(&name, &doc)
.map(String::into_bytes)
.map_err(io::Error::other)
});
}
}
files
}
}
struct Proc(Host);
impl Proc {
fn folder(&self, path: &str) -> Result<(SelfFolder, String)> {
let (pid, rest) = path
.trim_start_matches('/')
.split_once('/')
.unwrap_or((path.trim_start_matches('/'), ""));
let memory = self
.0
.0
.state
.lock()
.unwrap()
.processes
.get(pid)
.map(|p| p.memory.clone())
.ok_or(io::ErrorKind::NotFound)?;
Ok((
SelfFolder {
host: self.0.clone(),
pid: Some(pid.into()),
memory,
},
format!("/{rest}"),
))
}
}
#[async_trait]
impl Layer for Proc {
async fn stat(&self, path: &str, follow: bool) -> Result<Stat> {
if matches!(path.trim_end_matches('/'), "" | "/core" | "/core/plugins") {
return Ok(directory());
}
if let Some(path) = path.strip_prefix("/core/") {
return self.0.core_files().stat(&format!("/{path}"), follow).await;
}
let (folder, path) = self.folder(path)?;
folder.stat(&path, follow).await
}
async fn list(&self, path: &str) -> Result<Vec<Entry>> {
if path == "/" {
let mut entries = vec![entry("core".into(), true)];
entries.extend(self.0.processes().into_keys().map(|pid| entry(pid, true)));
entries.sort_by(|a, b| a.name.cmp(&b.name));
return Ok(entries);
}
if path.trim_end_matches('/') == "/core/plugins" {
return Ok(self
.0
.discovered_plugins()
.into_iter()
.filter(|name| self.0.discover_document(name).is_some())
.map(|name| entry(name, true))
.collect());
}
if path.trim_end_matches('/') == "/core" {
let mut entries = self.0.core_files().list("/").await?;
if !entries.iter().any(|e| e.name == "plugins") {
entries.push(entry("plugins".into(), true));
}
return Ok(entries);
}
if let Some(path) = path.strip_prefix("/core/") {
return self.0.core_files().list(&format!("/{path}")).await;
}
let (folder, path) = self.folder(path)?;
folder.list(&path).await
}
async fn open(&self, path: &str, options: Open) -> Result<Arc<dyn File>> {
if options.write || options.create || options.truncate {
return denied();
}
if matches!(path.trim_end_matches('/'), "" | "/core" | "/core/plugins") {
return Memory::default().open("/", options).await;
}
if let Some(path) = path.strip_prefix("/core/") {
return self.0.core_files().open(&format!("/{path}"), options).await;
}
let (folder, path) = self.folder(path)?;
folder.open(&path, options).await
}
async fn mkdir(&self, _: &str) -> Result<()> {
denied()
}
async fn remove_file(&self, _: &str) -> Result<()> {
denied()
}
async fn remove_dir(&self, _: &str) -> Result<()> {
denied()
}
async fn rename(&self, _: &str, _: &str) -> Result<()> {
denied()
}
}
struct SelfFolder {
host: Host,
pid: Option<String>,
memory: Memory,
}
impl SelfFolder {
async fn generated(&self, path: &str) -> Result<Generated> {
let pid = self.pid.as_ref().ok_or(io::ErrorKind::NotFound)?;
let bytes = match path {
"/info.json" => {
let info = self
.host
.0
.state
.lock()
.unwrap()
.processes
.get(pid)
.map(|p| p.info.clone())
.ok_or(io::ErrorKind::NotFound)?;
serde_json::to_vec(&info).unwrap()
}
"/log" => {
let lines = self
.host
.0
.journal
.query(serde_json::json!({"pid":pid,"limit":i64::MAX}))
.await;
let mut bytes = Vec::new();
for line in lines.as_array().unwrap() {
serde_json::to_writer(&mut bytes, line).unwrap();
bytes.push(b'\n');
}
bytes
}
_ => return missing(),
};
let mut files = Generated::default();
files.insert(path, move || Ok(bytes.clone()));
Ok(files)
}
}
#[async_trait]
impl Layer for SelfFolder {
async fn stat(&self, path: &str, follow: bool) -> Result<Stat> {
if reserved(path) {
self.generated(path).await?.stat(path, follow).await
} else {
self.memory.stat(path, follow).await
}
}
async fn list(&self, path: &str) -> Result<Vec<Entry>> {
let mut entries = self.memory.list(path).await?;
if path == "/" && self.pid.is_some() {
entries.extend([entry("info.json".into(), false), entry("log".into(), false)]);
}
Ok(entries)
}
async fn open(&self, path: &str, options: Open) -> Result<Arc<dyn File>> {
if reserved(path) {
if options.write || options.create || options.truncate {
return denied();
}
self.generated(path).await?.open(path, options).await
} else {
self.memory.open(path, options).await
}
}
async fn mkdir(&self, path: &str) -> Result<()> {
if reserved(path) {
denied()
} else {
self.memory.mkdir(path).await
}
}
async fn remove_file(&self, path: &str) -> Result<()> {
if reserved(path) {
denied()
} else {
self.memory.remove_file(path).await
}
}
async fn remove_dir(&self, path: &str) -> Result<()> {
if reserved(path) || path == "/" {
denied()
} else {
self.memory.remove_dir(path).await
}
}
async fn rename(&self, from: &str, to: &str) -> Result<()> {
if reserved(from) || reserved(to) || from == "/" || to == "/" {
denied()
} else {
self.memory.rename(from, to).await
}
}
}
+253 -27
View File
@@ -1,31 +1,257 @@
//! The interface graph (PROTOCOL.md section 10.4): the `rpc.discover`
//! documents of all plugins, their imports, and the result of each check.
//!
//! The `/proc` layer reads the files of `/proc/core/` through these
//! functions. They must be cheap and must not block: they return cached data.
use crate::Host;
use serde_json::Value;
//! Cached discovery documents and import checks under PROTOCOL.md section 10.4.
//! Refreshes use immutable copies. The proc layer only reads serialized cache entries.
use crate::{Host, RpcResult};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeMap, HashMap},
path::Path,
};
// The /proc layer (T6) uses these functions. Remove this when it does.
#[allow(dead_code)]
impl Host {
/// The content of `/proc/core/graph.json`.
pub(crate) fn graph_json(&self) -> Vec<u8> {
b"{}\n".to_vec()
}
#[derive(Clone)]
pub(crate) struct Interface {
pub(crate) hash: String,
pub(crate) document: Value,
pub(crate) error: Option<Value>,
}
pub(crate) struct Graph {
pub(crate) plugins: BTreeMap<String, Interface>,
pub(crate) versions: HashMap<String, Value>,
json: Vec<u8>,
dot: Vec<u8>,
}
/// The content of `/proc/core/graph.dot`.
pub(crate) fn graph_dot(&self) -> Vec<u8> {
b"digraph plugins {}\n".to_vec()
}
/// The plugins that have an `rpc.discover` document, for `/proc/core/plugins/`.
pub(crate) fn discovered_plugins(&self) -> Vec<String> {
Vec::new()
}
/// The `rpc.discover` document of the current `plugin.wasm` of `plugin`.
pub(crate) fn discover_document(&self, _plugin: &str) -> Option<Value> {
None
impl Default for Graph {
/// Exposes a valid empty graph before background discovery completes.
fn default() -> Self {
Self {
plugins: BTreeMap::new(),
versions: HashMap::new(),
json: b"{\"plugins\":{}}".to_vec(),
dot: b"digraph plugins {}\n".to_vec(),
}
}
}
/// Hashes exactly the bytes copied for discovery or installation.
pub(crate) fn hash(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
/// Lists broken imports, optionally restricted to changed consumers or providers.
pub(crate) fn problems(
plugins: &BTreeMap<String, Interface>,
changed: Option<&[String]>,
) -> Vec<Value> {
let mut failures = Vec::new();
for (consumer, interface) in plugins {
if let Some(imports) = interface.document["x-sausage-imports"].as_object() {
for (provider, expected) in imports {
if changed
.is_some_and(|names| !names.contains(consumer) && !names.contains(provider))
{
continue;
}
let Some(actual) = plugins.get(provider) else {
failures.push(json!({"consumer":consumer,"provider":provider,"status":"missing","message":"the provider does not exist"}));
continue;
};
if let Some(error) = &actual.error {
failures.push(json!({"consumer":consumer,"provider":provider,"status":"unavailable","message":"discovery failed","error":error}));
continue;
}
for problem in sausage_interface::compat::check(expected, &actual.document) {
failures.push(json!({"consumer":consumer,"provider":provider,"status":"broken","method":problem.method,"message":problem.message,"counterexample":problem.counterexample,"display":problem.to_string()}));
}
}
}
}
failures
}
impl Host {
/// The cached JSON graph. This does not perform filesystem or RPC operations.
pub(crate) fn graph_json(&self) -> Vec<u8> {
self.0.graph.lock().unwrap().json.clone()
}
/// The cached DOT graph, with an edge for every import.
pub(crate) fn graph_dot(&self) -> Vec<u8> {
self.0.graph.lock().unwrap().dot.clone()
}
/// Plugin names currently present in the discovery cache.
pub(crate) fn discovered_plugins(&self) -> Vec<String> {
self.0
.graph
.lock()
.unwrap()
.plugins
.keys()
.cloned()
.collect()
}
/// The current document, if discovery succeeded.
pub(crate) fn discover_document(&self, plugin: &str) -> Option<Value> {
self.0
.graph
.lock()
.unwrap()
.plugins
.get(plugin)
.filter(|entry| entry.error.is_none())
.map(|entry| entry.document.clone())
}
/// Discovers a private copy and caches successful documents by file hash.
pub(crate) async fn discover_copy(
&self,
plugin: &str,
path: &Path,
bytes: &[u8],
) -> Result<Interface, Value> {
let hash = hash(bytes);
let cache_key = format!("{plugin}/{hash}");
let document = self
.route_from(
"core".into(),
Some(json!(uuid::Uuid::now_v7().to_string())),
format!("{plugin}.rpc.discover"),
json!({}),
true,
Some(path.to_path_buf()),
)
.await?;
if !document["methods"].is_array() {
return Err(crate::error(
-32003,
"rpc.discover did not return an OpenRPC document",
));
}
self.0
.graph
.lock()
.unwrap()
.versions
.insert(cache_key, document.clone());
Ok(Interface {
hash,
document,
error: None,
})
}
/// Refreshes every installed plugin while the caller holds the activation mutex.
pub(crate) async fn refresh_graph(&self) -> RpcResult {
let mut directory = tokio::fs::read_dir(self.0.world.root.join("plugins"))
.await
.map_err(io_error)?;
let mut plugins = BTreeMap::new();
while let Some(entry) = directory.next_entry().await.map_err(io_error)? {
let name = entry.file_name().to_string_lossy().into_owned();
let source = entry.path().join("plugin.wasm");
let bytes = match tokio::fs::read(&source).await {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(io_error(error)),
};
let hash = hash(&bytes);
let previous = self.0.graph.lock().unwrap().plugins.get(&name).cloned();
let interface = if let Some(previous) =
previous.filter(|entry| entry.hash == hash && entry.error.is_none())
{
previous
} else {
let copy = entry
.path()
.join(format!("discover-{}.wasm", uuid::Uuid::now_v7()));
tokio::fs::write(&copy, &bytes).await.map_err(io_error)?;
let result = self.discover_copy(&name, &copy, &bytes).await;
let _ = tokio::fs::remove_file(&copy).await;
match result {
Ok(interface) => interface,
Err(error) => Interface {
hash,
document: Value::Null,
error: Some(error),
},
}
};
plugins.insert(name, interface);
}
let admission = self.0.admission.write().await;
let old_plugins = self.0.graph.lock().unwrap().plugins.clone();
let mut upgrades = Vec::new();
for (plugin, current) in &plugins {
if let Some(old) = old_plugins
.get(plugin)
.filter(|old| old.hash != current.hash && current.error.is_none())
{
let grace = old.document["x-sausage-grace"]
.as_f64()
.filter(|value| value.is_finite() && *value >= 0.0)
.unwrap_or(60.0);
upgrades.push(self.snapshot_upgrade(
plugin,
old.hash.clone(),
&current.hash,
grace,
));
}
}
self.publish_graph(plugins);
drop(admission);
for upgrade in upgrades {
self.begin_upgrade(upgrade);
}
Ok(json!({}))
}
/// Rebuilds serialized snapshots after a watcher refresh or an activation.
pub(crate) fn publish_graph(&self, plugins: BTreeMap<String, Interface>) {
let failures = problems(&plugins, None);
for failure in &failures {
self.0
.journal
.write("core", "core", "core", &format!("broken import {failure}"));
}
let mut entries = serde_json::Map::new();
let mut dot = String::from("digraph plugins {\n");
for (name, interface) in &plugins {
dot.push_str(&format!(" {};\n", json!(name)));
let mut imports = serde_json::Map::new();
for provider in interface.document["x-sausage-imports"]
.as_object()
.into_iter()
.flat_map(|imports| imports.keys())
{
let issues: Vec<_> = failures
.iter()
.filter(|problem| {
problem["consumer"] == *name && problem["provider"] == *provider
})
.cloned()
.collect();
let status = issues
.first()
.and_then(|problem| problem["status"].as_str())
.unwrap_or("compatible");
imports.insert(provider.clone(), json!({"status":status,"problems":issues}));
dot.push_str(&format!(
" {} -> {} [label={}];\n",
json!(name),
json!(provider),
json!(status)
));
}
entries.insert(
name.clone(),
json!({"hash":interface.hash,"imports":imports,"error":interface.error}),
);
}
dot.push_str("}\n");
let mut graph = self.0.graph.lock().unwrap();
graph.json = serde_json::to_vec(&json!({"plugins":entries})).unwrap();
graph.dot = dot.into_bytes();
graph.plugins = plugins;
}
}
/// Converts a filesystem failure into a JSON-RPC internal error.
pub(crate) fn io_error(error: std::io::Error) -> Value {
crate::error(-32603, &error.to_string())
}
+17
View File
@@ -17,6 +17,7 @@ pub(crate) struct State {
pub(crate) processes: HashMap<String, Process>,
pub(crate) locks: HashMap<String, Lock>,
pub(crate) native: HashMap<String, NativePlugin>,
pub(crate) services: Vec<crate::services::Service>,
}
pub(crate) struct Inner {
pub(crate) world: World,
@@ -26,6 +27,15 @@ pub(crate) struct Inner {
pub(crate) notifications: tokio::sync::broadcast::Sender<Value>,
pub(crate) exit: watch::Sender<bool>,
pub(crate) components: components::Components,
/// Calls hold it for reading from the choice of a version until their
/// process is in the table. Activation holds it for writing while it
/// installs a version, so that it sees every process of the old version.
pub(crate) admission: tokio::sync::RwLock<()>,
pub(crate) run: sausage_vfs::Memory,
pub(crate) graph: Mutex<crate::graph::Graph>,
pub(crate) activations: tokio::sync::Mutex<()>,
pub(crate) watcher: Mutex<Option<notify::RecommendedWatcher>>,
pub(crate) retirements: Mutex<std::collections::HashSet<(String, String)>>,
}
#[derive(Clone)]
/// A cloneable handle to a world and its running processes.
@@ -60,10 +70,17 @@ impl Host {
processes: HashMap::new(),
locks: HashMap::new(),
native: HashMap::new(),
services: Vec::new(),
}),
notifications: tokio::sync::broadcast::channel(1024).0,
exit: watch::channel(false).0,
components: components::Components::default(),
admission: tokio::sync::RwLock::new(()),
run: sausage_vfs::Memory::default(),
graph: Mutex::new(crate::graph::Graph::default()),
activations: tokio::sync::Mutex::new(()),
watcher: Mutex::new(None),
retirements: Mutex::new(std::collections::HashSet::new()),
}));
host.boot().await?;
Ok(host)
+3
View File
@@ -5,6 +5,7 @@ mod components;
pub mod console;
mod core;
mod execution;
mod filesystem;
mod graph;
mod host;
mod journal;
@@ -14,6 +15,8 @@ mod process;
mod routing;
pub mod runtime;
mod services;
mod upgrades;
mod watcher;
pub mod world;
pub use host::Host;
pub use native::{NativeContext, NativeFuture, NativePlugin};
+18 -14
View File
@@ -24,6 +24,9 @@ pub struct ProcessInfo {
}
pub(crate) struct Process {
pub(crate) info: ProcessInfo,
pub(crate) memory: sausage_vfs::Memory,
pub(crate) files: sausage_vfs::Process,
pub(crate) store_ended: watch::Sender<bool>,
pub(crate) input: Option<mpsc::UnboundedSender<Value>>,
pub(crate) abort: Option<tokio::task::AbortHandle>,
pub(crate) caller: String,
@@ -31,19 +34,19 @@ pub(crate) struct Process {
pub(crate) answer: Option<oneshot::Sender<RpcResult>>,
pub(crate) ended: watch::Sender<bool>,
}
pub(crate) struct ProcessGuard {
pub(crate) host: Host,
pub(crate) pid: String,
}
impl Drop for ProcessGuard {
fn drop(&mut self) {
self.host.finish(&self.pid, "process task ended");
}
}
impl Host {
/// Remove a process and release its locks before asynchronous journal completion.
pub(crate) fn finish(&self, pid: &str, status: &str) {
/// Called only after the execution task ends. Wait for the store and file operations before releasing locks.
pub(crate) async fn finish(&self, pid: &str, status: &str) {
let (files, mut store_ended) = {
let mut state = self.0.state.lock().unwrap();
let Some(p) = state.processes.get_mut(pid) else {
return;
};
p.info.state = "stopping".into();
(p.files.clone(), p.store_ended.subscribe())
};
let _ = store_ended.wait_for(|ended| *ended).await;
files.idle().await;
let mut state = self.0.state.lock().unwrap();
let process = state.processes.remove(pid);
if let Some(p) = process {
@@ -106,13 +109,14 @@ impl Host {
}
/// Abort a process task and perform its ended-state cleanup.
pub(crate) fn kill(&self, pid: &str) {
if let Some(p) = self.0.state.lock().unwrap().processes.get(pid) {
if let Some(p) = self.0.state.lock().unwrap().processes.get_mut(pid) {
p.info.state = "stopping".into();
p.input.take();
self.0.journal.write(pid, &p.info.plugin, "core", "kill");
if let Some(a) = &p.abort {
a.abort();
}
}
self.finish(pid, "killed");
}
/// Stop every process in parallel and wait for journal writes to finish.
pub async fn stop(&self) {
+24 -1
View File
@@ -93,6 +93,7 @@ impl Host {
{
return Err(error(-32601, "invalid plugin name"));
}
let admission = host.0.admission.read().await;
let native = host.0.state.lock().unwrap().native.get(plugin).cloned();
let (component, version) = if native.is_some() {
(None, "native".into())
@@ -141,6 +142,9 @@ impl Host {
pid.clone(),
Process {
info: info.clone(),
memory: sausage_vfs::Memory::default(),
files: sausage_vfs::Process::default(),
store_ended: watch::channel(true).0,
input: Some(input),
abort: None,
caller,
@@ -153,15 +157,34 @@ impl Host {
let h = host.clone();
let p = pid.clone();
let has_id = id.is_some();
let (start, ready) = oneshot::channel();
let task = tokio::spawn(async move {
let _ = ready.await;
h.run_process(p, info, component, native, incoming, has_id)
.await;
.await
});
if let Some(p) = host.0.state.lock().unwrap().processes.get_mut(&pid) {
p.abort = Some(task.abort_handle());
if p.info.state == "stopping" {
task.abort();
}
} else {
task.abort();
}
drop(admission);
let cleanup = host.clone();
let finished_pid = pid.clone();
tokio::spawn(async move {
let status = task.await.unwrap_or_else(|error| {
if error.is_cancelled() {
"killed".into()
} else {
format!("host task failed: {error}")
}
});
cleanup.finish(&finished_pid, &status).await;
});
let _ = start.send(());
if id.is_none() {
return Ok(Value::Null);
}
+31 -16
View File
@@ -3,7 +3,7 @@
//! An instance is one run of a `wasi:cli/command` component. The host writes
//! NDJSON lines to its stdin and reads NDJSON lines from its stdout.
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@@ -17,7 +17,7 @@ use wasmtime_wasi::cli::{AsyncStdinStream, AsyncStdoutStream};
use wasmtime_wasi::p2::bindings::Command;
use wasmtime_wasi::{FsPerms, I32Exit, WasiCtx, WasiCtxView, WasiView};
use wasmtime_wasi::{I32Exit, WasiCtx, WasiCtxView, WasiView};
use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, WasiHttpView};
/// The HTTP transport hooks supplied to WASI processes.
@@ -61,6 +61,7 @@ pub struct ProcessState {
table: ResourceTable,
http: WasiHttpCtx,
hooks: HttpHooks,
fs: sausage_vfs::adapter::Adapter,
}
impl WasiView for ProcessState {
@@ -111,8 +112,10 @@ impl std::fmt::Display for Exit {
/// The filesystem, network, and environment supplied to one process.
pub struct InstanceConfig {
/// The world folder to preopen as the guest root.
pub world: PathBuf,
/// The virtual namespace exposed through the single root preopen.
pub filesystem: sausage_vfs::Mounts,
/// The store completion barrier used by process cleanup.
pub store_ended: tokio::sync::watch::Sender<bool>,
/// Outgoing HTTP, and TCP, UDP, and DNS through `wasi:sockets`.
pub network: bool,
/// The environment variables supplied to the guest.
@@ -127,12 +130,20 @@ pub struct Instance {
pub stdout: mpsc::Receiver<String>,
/// The stderr lines that the process writes.
pub stderr: mpsc::Receiver<String>,
task: JoinHandle<Exit>,
task: Option<JoinHandle<Exit>>,
store_ended: tokio::sync::watch::Sender<bool>,
}
impl Drop for Instance {
fn drop(&mut self) {
self.task.abort();
if let Some(task) = self.task.take() {
task.abort();
let ended = self.store_ended.clone();
tokio::spawn(async move {
let _ = task.await;
ended.send_replace(true);
});
}
}
}
@@ -166,7 +177,10 @@ impl Instance {
/// Wait until the guest ends.
pub async fn wait(&mut self) -> Exit {
match (&mut self.task).await {
let result = self.task.as_mut().unwrap().await;
self.task.take();
self.store_ended.send_replace(true);
match result {
Ok(exit) => exit,
Err(e) if e.is_cancelled() => Exit::Trap("killed".into()),
Err(e) => Exit::Trap(format!("host task failed: {e}")),
@@ -198,6 +212,10 @@ impl Runtime {
let mut linker: Linker<ProcessState> = Linker::new(&engine);
wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
sausage_vfs::adapter::add_to_linker(&mut linker, |state| sausage_vfs::adapter::View {
adapter: &mut state.fs,
table: &mut state.table,
})?;
wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?;
// Time slices: every 10 ms each running guest yields to the scheduler.
let runtime = Arc::new(Self { engine, linker });
@@ -220,7 +238,7 @@ impl Runtime {
Ok(Component::new(&self.engine, bytes)?)
}
/// Prepare a world preopen off the async workers, then start the WASI task.
/// Start a WASI task with the supplied virtual namespace.
pub async fn start(
self: &Arc<Self>,
component: &Component,
@@ -243,18 +261,13 @@ impl Runtime {
for (k, v) in &cfg.env {
b.env(k, v);
}
// Filesystem setup lives here. T6 replaces this preopen with the VFS.
let world = cfg.world;
let mut b = tokio::task::spawn_blocking(move || -> Result<_> {
b.preopened_dir(world, "/", FsPerms::ReadWrite)?;
Ok(b)
})
.await??;
let state = ProcessState {
ctx: b.build(),
table: ResourceTable::new(),
http: WasiHttpCtx::new(),
hooks: HttpHooks { allow: cfg.network },
fs: sausage_vfs::adapter::Adapter::new(cfg.filesystem),
};
let (line_tx, line_rx) = mpsc::channel(1024);
@@ -285,6 +298,7 @@ impl Runtime {
});
let rt = self.clone();
let component = component.clone();
cfg.store_ended.send_replace(false);
let task = tokio::spawn(async move {
let mut store = Store::new(&rt.engine, state);
store.epoch_deadline_async_yield_and_update(1);
@@ -311,7 +325,8 @@ impl Runtime {
stdin: Some(host_in),
stdout: line_rx,
stderr: err_rx,
task,
task: Some(task),
store_ended: cfg.store_ended,
})
}
}
+34 -19
View File
@@ -1,31 +1,46 @@
//! Services: the calls in `/config/boot.json` (PROTOCOL.md section 8). The
//! core starts them in order at boot.
//! Ordered boot services and their identities for upgrade restarts (PROTOCOL.md section 8).
use crate::Host;
use serde_json::{Value, json};
use uuid::Uuid;
#[derive(Clone)]
pub(crate) struct Service {
pub(crate) method: String,
pub(crate) params: Value,
pub(crate) request: Value,
}
impl Host {
/// Start each service of `/config/boot.json` as a detached call with the
/// parent `core`, in order. A failure goes to the journal, and boot
/// continues.
/// Starts discovery in the background, then waits for each boot service in order.
pub(crate) async fn boot(&self) -> anyhow::Result<()> {
self.watch_plugins()?;
let boot: Vec<Value> = serde_json::from_slice(
&tokio::fs::read(self.0.world.root.join("config/boot.json")).await?,
)?;
for service in boot {
let result = self
.route(
"core".into(),
Some(json!(Uuid::now_v7().to_string())),
service["method"].as_str().unwrap_or("").into(),
service["params"].clone(),
true,
)
.await;
if let Err(e) = result {
self.0.journal.write("core", "core", "core", &e.to_string());
}
self.start_service(
service["method"].as_str().unwrap_or("").into(),
service["params"].clone(),
)
.await;
}
Ok(())
}
/// Records a unique request identity before the service can answer or exit.
pub(crate) async fn start_service(&self, method: String, params: Value) {
let request = json!(uuid::Uuid::now_v7().to_string());
self.0.state.lock().unwrap().services.push(Service {
method: method.clone(),
params: params.clone(),
request: request.clone(),
});
self.0
.journal
.write("core", "core", "core", &format!("service start {method}"));
if let Err(error) = self
.route("core".into(), Some(request), method, params, true)
.await
{
self.0
.journal
.write("core", "core", "core", &format!("service failed {error}"));
}
}
}
+157
View File
@@ -0,0 +1,157 @@
//! Grace periods, service restarts, and retirement of replaced versions (PROTOCOL.md section 3.7).
use crate::{Host, services::Service};
use serde_json::json;
use std::{collections::BTreeMap, time::Duration};
use tokio::sync::watch;
pub(crate) struct OldProcess {
pid: String,
version: String,
ended: watch::Receiver<bool>,
}
pub(crate) struct Upgrade {
plugin: String,
previous: String,
grace: Duration,
processes: Vec<OldProcess>,
services: Vec<(Service, Option<watch::Receiver<bool>>)>,
}
impl Host {
/// Snapshots old admissions while activation holds the admission write guard.
pub(crate) fn snapshot_upgrade(
&self,
plugin: &str,
previous: String,
current: &str,
grace: f64,
) -> Upgrade {
let mut state = self.0.state.lock().unwrap();
let processes = state
.processes
.iter()
.filter(|(_, process)| process.info.plugin == plugin && process.info.version != current)
.map(|(pid, process)| OldProcess {
pid: pid.clone(),
version: process.info.version.clone(),
ended: process.ended.subscribe(),
})
.collect();
let mut services = Vec::new();
let mut retained = Vec::new();
for service in std::mem::take(&mut state.services) {
if service
.method
.split_once('.')
.is_some_and(|(name, _)| name == plugin)
{
let ended = state
.processes
.values()
.find(|process| process.id.as_ref() == Some(&service.request))
.map(|process| process.ended.subscribe());
services.push((service, ended));
} else {
retained.push(service);
}
}
state.services = retained;
Upgrade {
plugin: plugin.into(),
previous,
grace: Duration::from_secs_f64(grace),
processes,
services,
}
}
/// Lets old calls finish, stops them after grace, and starts each lifecycle hook once.
pub(crate) fn begin_upgrade(&self, upgrade: Upgrade) {
let host = self.clone();
tokio::spawn(async move {
host.0.journal.write(
"core",
&upgrade.plugin,
"core",
&format!("upgrade grace {} seconds", upgrade.grace.as_secs_f64()),
);
let mut versions: BTreeMap<String, Vec<watch::Receiver<bool>>> = BTreeMap::new();
versions.entry(upgrade.previous).or_default();
for process in upgrade.processes {
versions
.entry(process.version)
.or_default()
.push(process.ended.clone());
let host = host.clone();
let grace = upgrade.grace;
tokio::spawn(async move {
let mut ended = process.ended;
tokio::select! {
_ = ended.wait_for(|ended|*ended) => (),
_ = tokio::time::sleep(grace) => { host.begin_stop(&process.pid); }
}
});
}
for (service, ended) in upgrade.services {
let host = host.clone();
tokio::spawn(async move {
if let Some(mut ended) = ended {
let _ = ended.wait_for(|ended| *ended).await;
}
host.start_service(service.method, service.params).await;
});
}
for (version, processes) in versions {
let host = host.clone();
let plugin = upgrade.plugin.clone();
let retirement = (plugin.clone(), version.clone());
if !host
.0
.retirements
.lock()
.unwrap()
.insert(retirement.clone())
{
continue;
}
tokio::spawn(async move {
for mut ended in processes {
let _ = ended.wait_for(|ended| *ended).await;
}
if host
.0
.graph
.lock()
.unwrap()
.plugins
.get(&plugin)
.is_some_and(|current| current.hash == version)
{
host.0.retirements.lock().unwrap().remove(&retirement);
return;
}
host.0.journal.write(
"core",
&plugin,
"core",
&format!("retired start {version}"),
);
let result = host
.route(
"core".into(),
Some(json!(uuid::Uuid::now_v7().to_string())),
format!("{plugin}.rpc.retired"),
json!({"previous":version}),
true,
)
.await;
host.0.journal.write(
"core",
&plugin,
"core",
&format!("retired result {result:?}"),
);
host.0.retirements.lock().unwrap().remove(&retirement);
});
}
});
}
}
+48
View File
@@ -0,0 +1,48 @@
//! Background discovery and file notifications for installed plugin versions.
use crate::Host;
use notify::{RecursiveMode, Watcher};
impl Host {
/// Starts background discovery and retains the watcher for this host's lifetime.
pub(crate) fn watch_plugins(&self) -> anyhow::Result<()> {
let (events, mut incoming) = tokio::sync::mpsc::unbounded_channel();
let mut watcher =
notify::recommended_watcher(move |event: notify::Result<notify::Event>| {
if let Ok(event) = event
&& !matches!(event.kind, notify::EventKind::Access(_))
&& event
.paths
.iter()
.any(|path| path.file_name().is_some_and(|name| name == "plugin.wasm"))
{
let _ = events.send(());
}
})?;
watcher.watch(&self.0.world.root.join("plugins"), RecursiveMode::Recursive)?;
*self.0.watcher.lock().unwrap() = Some(watcher);
let weak = std::sync::Arc::downgrade(&self.0);
tokio::spawn(async move {
loop {
let Some(inner) = weak.upgrade() else {
break;
};
let host = Host(inner);
let _activation = host.0.activations.lock().await;
if let Err(error) = host.refresh_graph().await {
host.0.journal.write(
"core",
"core",
"core",
&format!("discovery failed {error}"),
);
}
drop(_activation);
drop(host);
if incoming.recv().await.is_none() {
break;
}
while incoming.try_recv().is_ok() {}
}
});
Ok(())
}
}