From 7c0256a8a4463d3845549616fea9993801ff96b6 Mon Sep 17 00:00:00 2001 From: Surma Date: Thu, 12 Nov 2026 11:13:14 +0000 Subject: [PATCH] Add a read-only cache layer for precompiled tools --- docs/RUNNING.md | 5 + host/src/main.rs | 7 ++ host/src/process/cache.rs | 224 ++++++++++++++++++++++++++++++++++++-- host/src/process/mod.rs | 2 +- 4 files changed, 229 insertions(+), 9 deletions(-) diff --git a/docs/RUNNING.md b/docs/RUNNING.md index 6d2619b..fe33b20 100644 --- a/docs/RUNNING.md +++ b/docs/RUNNING.md @@ -25,6 +25,11 @@ For `init`, you can override it with `--userland BUNDLE` or the environment vari A bundle contains `bin/`, `src/`, and the seeded `etc/` tree. It is not a world folder. Closing stdin only detaches the optional terminal console. The host and chat service continue. +The package sets `SAUSAGE_PRECOMPILED` to its read-only cache of compiled tools. +The host checks `--cache` or `~/.cache/sausage` first, then this folder, then compiles missing tools into the writable cache. +If Wasmtime rejects an artifact, the host logs one line and compiles the program lazily. +For development or tests, run `sausage precompile BUNDLE OUT` and set `SAUSAGE_PRECOMPILED=OUT`. + The source-tree bundle builder is: ```sh diff --git a/host/src/main.rs b/host/src/main.rs index f5a844d..2d2b40f 100644 --- a/host/src/main.rs +++ b/host/src/main.rs @@ -14,6 +14,8 @@ struct Cli { #[derive(Subcommand)] enum Command { + /// Compile the bundled tools into a reusable read-only cache. + Precompile { bundle: PathBuf, out: PathBuf }, /// Create a world with default tools and /etc/sausage/config.yaml. Init { world: PathBuf, @@ -44,6 +46,11 @@ async fn main() -> anyhow::Result<()> { sausage::process::configure_cache(cache)?; } match cli.command { + Command::Precompile { bundle, out } => { + let count = sausage::process::precompile(&bundle, &out)?; + println!("Precompiled programs: {count}"); + Ok(()) + } Command::Init { world, userland } => { let bundle = userland .or_else(|| std::env::var_os("SAUSAGE_USERLAND").map(PathBuf::from)) diff --git a/host/src/process/cache.rs b/host/src/process/cache.rs index 5ea598b..9e376ac 100644 --- a/host/src/process/cache.rs +++ b/host/src/process/cache.rs @@ -1,7 +1,9 @@ -//! Compiles programs on first start and shares cached results across worlds in one host process. +//! Shares lazy compilation across worlds, with writable and read-only precompiled artifact layers. use super::{Compiled, State, adapter, bindings}; +use sha2::{Digest, Sha256}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet, hash_map::DefaultHasher}, + hash::{Hash, Hasher}, path::{Path, PathBuf}, sync::{Arc, Mutex, OnceLock}, }; @@ -44,8 +46,47 @@ pub(super) fn shared() -> anyhow::Result> { } Ok(shared.as_ref().unwrap().clone()) } +/// Compile each distinct bundled program into the runtime's artifact format and keys. +pub fn precompile(bundle: &Path, out: &Path) -> anyhow::Result { + let compiler = Compiler::with_precompiled(out, None)?; + std::fs::create_dir_all(out)?; + let mut paths = std::fs::read_dir(bundle.join("bin"))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>()?; + paths.sort(); + let mut keys = HashSet::new(); + for path in paths { + if path + .extension() + .is_some_and(|extension| extension == "wasm") + { + let bytes = std::fs::read(&path)?; + let key = compiler.key(&bytes); + if keys.insert(key.clone()) { + write_artifact(out, &key, &compiler.artifact(&bytes)?)?; + } + } + } + Ok(keys.len()) +} + +// Publish complete artifacts, including when separate hosts share the writable cache. +fn write_artifact(directory: &Path, key: &str, bytes: &[u8]) -> std::io::Result<()> { + std::fs::create_dir_all(directory)?; + let temporary = directory.join(format!("{}.tmp", uuid::Uuid::new_v4())); + let result = std::fs::write(&temporary, bytes) + .and_then(|()| std::fs::rename(&temporary, directory.join(key))); + if result.is_err() { + let _ = std::fs::remove_file(temporary); + } + result +} + pub(super) struct Compiler { pub engine: Engine, + directory: PathBuf, + precompiled: Option, + fingerprint: u64, programs: Mutex, Compilation>>, #[cfg(test)] before_compile: Mutex>>, @@ -54,16 +95,30 @@ pub(super) struct Compiler { } impl Compiler { pub fn new(directory: &Path) -> anyhow::Result> { + Self::with_precompiled( + directory, + std::env::var_os("SAUSAGE_PRECOMPILED").map(PathBuf::from), + ) + } + fn with_precompiled( + directory: &Path, + precompiled: Option, + ) -> anyhow::Result> { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let mut config = wasmtime::Config::new(); config.epoch_interruption(true); // Keep Cranelift's default optimization: compiles are cached per content, and guests run // many times. OptLevel::None saved only 12.7% of compile time (T15) but slows every run. - let mut cache = wasmtime::CacheConfig::new(); - cache.with_directory(directory); - config.cache(Some(wasmtime::Cache::new(cache)?)); + let engine = Engine::new(&config)?; + let mut fingerprint = DefaultHasher::new(); + engine + .precompile_compatibility_hash() + .hash(&mut fingerprint); Ok(Arc::new(Self { - engine: Engine::new(&config)?, + engine, + directory: directory.to_owned(), + precompiled, + fingerprint: fingerprint.finish(), programs: Mutex::new(HashMap::new()), #[cfg(test)] before_compile: Mutex::new(None), @@ -95,7 +150,36 @@ impl Compiler { result.changed().await?; } } + fn key(&self, bytes: &[u8]) -> String { + format!( + "{:016x}-{}.cwasm", + self.fingerprint, + hex::encode(Sha256::digest(bytes)) + ) + } fn compile(&self, bytes: &[u8]) -> anyhow::Result { + let key = self.key(bytes); + for directory in std::iter::once(&self.directory).chain(self.precompiled.iter()) { + let path = directory.join(&key); + let result = match std::fs::read(&path) { + Ok(artifact) => self.link(bytes, &artifact), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => Err(error.into()), + }; + match result { + Ok(compiled) => return Ok(compiled), + Err(error) => eprintln!( + "Ignore precompiled cache {}: {}", + path.display(), + error.to_string().replace('\n', " ") + ), + } + } + let artifact = self.artifact(bytes)?; + write_artifact(&self.directory, &key, &artifact)?; + self.link(bytes, &artifact) + } + fn artifact(&self, bytes: &[u8]) -> wasmtime::Result> { #[cfg(test)] { self.compilations @@ -105,14 +189,24 @@ impl Compiler { } } if bytes.get(..8) == Some(b"\0asm\x01\0\0\0") { - let module = wasmtime::Module::new(&self.engine, bytes)?; + self.engine.precompile_module(bytes) + } else { + self.engine.precompile_component(bytes) + } + } + fn link(&self, bytes: &[u8], artifact: &[u8]) -> anyhow::Result { + // SAFETY: artifacts come from this host's compiler or its trusted host/Nix cache, never + // from the world. Wasmtime checks engine compatibility before it loads their code. + if bytes.get(..8) == Some(b"\0asm\x01\0\0\0") { + let module = unsafe { wasmtime::Module::deserialize(&self.engine, artifact)? }; let mut linker = wasmtime::Linker::new(&self.engine); sausage_vfs::preview1::add_to_linker(&mut linker, |s: &mut State| { s.preview1.as_mut().unwrap() })?; Ok(Compiled::Module(linker.instantiate_pre(&module)?)) } else { - let component = Component::new(&self.engine, bytes)?; + // SAFETY: the artifact has the same trusted provenance described above. + let component = unsafe { Component::deserialize(&self.engine, artifact)? }; let mut linker = Linker::new(&self.engine); wasmtime_wasi::p2::add_to_linker_async(&mut linker)?; wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?; @@ -136,6 +230,120 @@ mod tests { use crate::exec::{Command, Exec, Options}; use std::{future::Future, sync::atomic::Ordering}; + fn fixture(component: bool) -> Vec { + wat::parse_str(if component { + r#"(component + (core module $m (func (export "run") (result i32) i32.const 0)) + (core instance $i (instantiate $m)) + (func $run (result (result)) (canon lift (core func $i "run"))) + (instance $cli (export "run" (func $run))) + (export "wasi:cli/run@0.2.6" (instance $cli)))"# + } else { + r#"(module (memory (export "memory") 1) (func (export "_start")))"# + }) + .unwrap() + } + + #[tokio::test] + async fn precompile_deduplicates_and_readonly_hits_compile_and_write_nothing() + -> anyhow::Result<()> { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir()?; + let bundle = temp.path().join("bundle"); + std::fs::create_dir_all(bundle.join("bin"))?; + for (name, component) in [("one", false), ("alias", false), ("component", true)] { + std::fs::write(bundle.join(format!("bin/{name}.wasm")), fixture(component))?; + } + std::fs::write(bundle.join("bin/not-wasm"), b"not a program")?; + let readonly = temp.path().join("precompiled"); + assert_eq!(precompile(&bundle, &readonly)?, 2); + assert_eq!(std::fs::read_dir(&readonly)?.count(), 2); + let compiler = + Compiler::with_precompiled(&temp.path().join("writable"), Some(readonly.clone()))?; + for component in [false, true] { + let bytes = fixture(component); + let path = readonly.join(compiler.key(&bytes)); + let before = std::fs::read(&path)?; + let modified = std::fs::metadata(&path)?.modified()?; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o444))?; + std::fs::set_permissions(&readonly, std::fs::Permissions::from_mode(0o555))?; + let directory_modified = std::fs::metadata(&readonly)?.modified()?; + let result = compiler.get(bytes).await; + std::fs::set_permissions(&readonly, std::fs::Permissions::from_mode(0o755))?; + result?; + assert_eq!(std::fs::read(&path)?, before); + assert_eq!(std::fs::metadata(&path)?.modified()?, modified); + assert_eq!( + std::fs::metadata(&readonly)?.modified()?, + directory_modified + ); + } + assert_eq!(compiler.compilations.load(Ordering::Relaxed), 0); + assert!(!temp.path().join("writable").exists()); + assert_eq!(std::fs::read_dir(&readonly)?.count(), 2); + Ok(()) + } + + #[tokio::test] + async fn writable_hit_precedes_readonly_and_changed_content_compiles_lazily() + -> anyhow::Result<()> { + let temp = tempfile::tempdir()?; + let writable = temp.path().join("writable"); + let readonly = temp.path().join("readonly"); + let compiler = Compiler::with_precompiled(&writable, Some(readonly.clone()))?; + let bytes = fixture(false); + write_artifact( + &writable, + &compiler.key(&bytes), + &compiler.engine.precompile_module(&bytes)?, + )?; + write_artifact(&readonly, &compiler.key(&bytes), b"invalid artifact")?; + compiler.get(bytes).await?; + assert_eq!(compiler.compilations.load(Ordering::Relaxed), 0); + let changed = + wat::parse_str(r#"(module (memory (export "memory") 2) (func (export "_start")))"#)?; + compiler.get(changed.clone()).await?; + assert_eq!(compiler.compilations.load(Ordering::Relaxed), 1); + assert!(writable.join(compiler.key(&changed)).is_file()); + let reopened = Compiler::with_precompiled(&writable, None)?; + reopened.get(changed).await?; + assert_eq!(reopened.compilations.load(Ordering::Relaxed), 0); + assert_eq!(std::fs::read_dir(&readonly)?.count(), 1); + Ok(()) + } + + #[tokio::test] + async fn corrupted_and_incompatible_readonly_entries_fall_back_to_lazy_compilation() + -> anyhow::Result<()> { + for component in [false, true] { + for incompatible in [false, true] { + let temp = tempfile::tempdir()?; + let writable = temp.path().join("writable"); + let readonly = temp.path().join("readonly"); + let compiler = Compiler::with_precompiled(&writable, Some(readonly.clone()))?; + let bytes = fixture(component); + let artifact = if incompatible { + // The runtime enables epoch interruption. This code deliberately does not. + let engine = Engine::new(&wasmtime::Config::new())?; + if component { + engine.precompile_component(&bytes)? + } else { + engine.precompile_module(&bytes)? + } + } else { + b"corrupted artifact".to_vec() + }; + let key = compiler.key(&bytes); + write_artifact(&readonly, &key, &artifact)?; + compiler.get(bytes).await?; + assert_eq!(compiler.compilations.load(Ordering::Relaxed), 1); + assert_eq!(std::fs::read(readonly.join(&key))?, artifact); + assert!(writable.join(key).is_file()); + } + } + Ok(()) + } + #[tokio::test] async fn startup_is_lazy_and_identical_programs_share_first_compilation() -> anyhow::Result<()> { diff --git a/host/src/process/mod.rs b/host/src/process/mod.rs index 92faba4..469f40e 100644 --- a/host/src/process/mod.rs +++ b/host/src/process/mod.rs @@ -55,7 +55,7 @@ pub mod net; pub fn default_script() -> Arc { Arc::new(crate::cred::quickjs::QuickJs::default()) } -pub use cache::configure as configure_cache; +pub use cache::{configure as configure_cache, precompile}; mod services; mod watcher; pub use watcher::Watcher;