From f2df0ce362ca52252bc6998f210531ea4b2068a2 Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 6 Nov 2026 19:51:44 +0000 Subject: [PATCH] Prepare v3 Nix packaging and OpenSSL build inputs --- flake.nix | 239 ++++++++++++++++++++---------------------------------- 1 file changed, 90 insertions(+), 149 deletions(-) diff --git a/flake.nix b/flake.nix index e61ce59..4be7cb2 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "Sausage: grow a system of WASI plugins by talking to it"; + description = "Sausage: a world of WASI programs and durable agents"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; @@ -16,18 +16,10 @@ inherit system; overlays = [ rust-overlay.overlays.default ]; }; - # The toolchain that builds the host and, at runtime, the plugins. rust = pkgs.rust-bin.stable.latest.default.override { targets = [ "wasm32-wasip2" ]; extensions = [ "rust-src" ]; }; - # The native cargo plugin needs Cargo, rustc (with wasm32-wasip2), and - # the native linker, even when the host itself is started with a bare - # PATH. Keep this same runtime path in the packaged host and exercise - # the installed binary below with PATH deliberately empty. - # mold links host code much faster than GNU ld. Only on Linux, and only - # for the host target: plugins for wasm32-wasip2 link with rust-lld. Nix - # brings mold along, so a build without Nix keeps the default linker. mold = pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.mold ]; hostTarget = pkgs.lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] pkgs.stdenv.hostPlatform.rust.rustcTarget); @@ -35,176 +27,125 @@ moldEnv = pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { ${moldVar} = "-C link-arg=-fuse-ld=mold"; }; - runtimePath = pkgs.lib.makeBinPath ([ rust pkgs.stdenv.cc ] ++ mold); - # A WASI clang with wasi-libc, so that build scripts (the `cc` crate) - # can compile C for wasm32-wasip2. Its headers are WASI 0.1: C code - # that needs sockets does not build. wasiCc = pkgs.pkgsCross.wasi32.stdenv.cc; wasiCcEnv = { CC_wasm32_wasip2 = "${wasiCc}/bin/${wasiCc.targetPrefix}cc"; AR_wasm32_wasip2 = "${wasiCc}/bin/${wasiCc.targetPrefix}ar"; }; craneLib = (crane.mkLib pkgs).overrideToolchain rust; + # Preserve the embedded JavaScript and WIT alongside Cargo's sources. + hostSrc = pkgs.lib.cleanSourceWith { + src = ./.; + filter = path: type: + let relative = pkgs.lib.removePrefix (toString ./. + "/") path; + in !(pkgs.lib.hasPrefix "userland/" relative || pkgs.lib.hasPrefix "spikes/" relative) + && (craneLib.filterCargoSources path type + || pkgs.lib.hasSuffix ".wit" path || pkgs.lib.hasSuffix ".js" path); + }; + v8Archive = pkgs.fetchurl { + url = "https://github.com/denoland/rusty_v8/releases/download/v150.4.0/librusty_v8_release_${pkgs.stdenv.hostPlatform.rust.rustcTarget}.a.gz"; + hash = { + x86_64-linux = "sha256-WGn9twcbHyHyAKl86X0gElh34PMc2ALtmd4sU/SIsGw="; + aarch64-linux = "sha256-txd9Uq0zNycv4NO453gjnIIalcJdWVnexiue/WVPfdM="; + x86_64-darwin = "sha256-5ex9E/kUgT6/IB1Ee/j9J2h7exkuFsR/KCb+VBUXHyk="; + aarch64-darwin = "sha256-zNj4FIW4IsWxiuun+d65KaM4LYasZzu/DzZvBod+axA="; + }.${system}; + }; common = { pname = "sausage"; - version = "0.1.0"; - src = ./.; - nativeBuildInputs = mold; + version = "0.3.0"; + src = hostSrc; + strictDeps = true; + RUSTY_V8_ARCHIVE = v8Archive; + buildInputs = [ pkgs.openssl ]; + nativeBuildInputs = [ pkgs.pkg-config pkgs.cmake pkgs.python3 pkgs.perl ] ++ mold; } // moldEnv; - # The test packages for `checks.tests`. - testPackages = "--workspace"; cargoArtifacts = craneLib.buildDepsOnly (common // { doCheck = false; - # The dependencies of the host binary and of the tests. - buildPhaseCargoCommand = '' - cargoWithProfile build --locked -p sausage - cargoWithProfile test --locked --no-run ${testPackages} + }); + brushSource = pkgs.fetchFromGitHub { + owner = "reubeno"; + repo = "brush"; + rev = "943553433ae04f5b3bd55d2cf207e39eb9ef2af7"; + hash = "sha256-a1eMkqx4nBqJ8R+h+rxtU35pAungcEnXoxwZnb1poH8="; + }; + userlandSrc = pkgs.lib.cleanSourceWith { + src = ./.; + filter = path: type: + let relative = pkgs.lib.removePrefix (toString ./. + "/") path; + in type == "directory" || pkgs.lib.hasPrefix "userland/" relative + || pkgs.lib.hasPrefix "host/wit/" relative; + }; + userlandCommon = { + pname = "sausage-userland"; + version = "0.3.0"; + src = userlandSrc; + strictDeps = true; + cargoVendorDir = craneLib.vendorMultipleCargoDeps { + cargoLockList = [ ./userland/Cargo.lock ./userland/brush/Cargo.lock ]; + }; + nativeBuildInputs = [ pkgs.nushell pkgs.wasm-tools pkgs.git pkgs.pkg-config ] ++ mold; + postPatch = '' + cp -r ${brushSource} userland/brush/source + chmod -R u+w userland/brush/source + # The fixed-output source fetch replaces the development checkout's Git check. + substituteInPlace userland/brush/build.nu --replace-fail \ + ${pkgs.lib.escapeShellArg "if (^git -C $source rev-parse HEAD | str trim) != $revision { error make {msg: 'unexpected brush revision'} }"} \ + '# Revision verified by the fixed-output Nix fetch.' + patchShebangs userland/cwd-init/link.nu ''; + preBuild = '' + export CARGO_TARGET_DIR="$PWD/target" + ''; + buildPhaseCargoCommand = "nu userland/bundle.nu"; + doCheck = false; + } // wasiCcEnv // moldEnv; + # Keep the real local patch crates: registry dependencies use their full APIs. + userlandArtifacts = craneLib.buildDepsOnly (userlandCommon // { + dummySrc = userlandSrc; + }); + userland = craneLib.mkCargoDerivation (userlandCommon // { + cargoArtifacts = userlandArtifacts; + installPhaseCommand = '' + mkdir -p $out + cp -a target/userland/bin $out/bin + ''; + doInstallCargoArtifacts = false; }); sausage = craneLib.buildPackage (common // { inherit cargoArtifacts; - doNotRemoveReferencesToRustToolchain = true; cargoExtraArgs = "--locked -p sausage"; doCheck = false; - nativeBuildInputs = [ pkgs.makeWrapper ] ++ mold; - # The native `cargo` plugin runs cargo at runtime. Build scripts need a C linker, - # and C code for WASI needs the WASI clang. + nativeBuildInputs = common.nativeBuildInputs ++ [ pkgs.makeWrapper ]; postInstall = '' - wrapProgram $out/bin/sausage --prefix PATH : ${runtimePath} \ - --set-default CC_wasm32_wasip2 ${wasiCcEnv.CC_wasm32_wasip2} \ - --set-default AR_wasm32_wasip2 ${wasiCcEnv.AR_wasm32_wasip2} \ - ${pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isLinux "--set-default ${moldVar} '-C link-arg=-fuse-ld=mold'"} + wrapProgram $out/bin/sausage --set-default SAUSAGE_USERLAND ${userland}/bin ''; meta.mainProgram = "sausage"; }); - # `nix run .#loop -- ` runs the newest `sausage run ` - # (without a folder: bare `sausage`, i.e. ./world, created if needed) - # from the public repo, again and again: after a crash or a call of - # `core.restart` (exit code 75) it refreshes the flake and starts the host again, so - # the world updates itself. `quit` (exit code 0) ends the loop. - # SAUSAGE_FLAKE overrides where it fetches from (any flake URL, - # e.g. a fork or a local checkout). - repoUrl = "git+https://gitea.surma.technology/surma/sausage"; - loop = pkgs.writeShellScriptBin "sausage-loop" '' - url="''${SAUSAGE_FLAKE:-${repoUrl}}" - # No folder: bare `sausage` (creates ./world first if needed). - if [ $# -gt 0 ]; then set -- run "$@"; fi - while true; do - SAUSAGE_LOOP=1 nix run --refresh "$url#default" -- "$@" - code=$? - if [ "$code" -eq 0 ]; then exit 0; fi - echo "sausage-loop: sausage exited with $code, starting the newest version in 2 s …" >&2 - sleep 2 - done - ''; - in - { + in { packages.default = sausage; - - # `nix flake check` runs every test of the workspace. The tests build - # WASM plugins with cargo, offline, from the vendored crates of - # Cargo.lock. - # The binary the host loop runs (`nix run`), built from the same - # cached dependencies as the tests. checks.package = sausage; checks.tests = craneLib.cargoTest (common // wasiCcEnv // { inherit cargoArtifacts; - cargoExtraArgs = "--locked ${testPackages}"; - # Nothing builds on the test run's target folder: don't store it. + cargoExtraArgs = "--workspace --lib --test harness --test p1 --test layers --test regressions --test streams --test wasi"; + preCheck = '' + mkdir -p target/userland + ln -s ${userland}/bin target/userland/bin + ''; doInstallCargoArtifacts = false; - # The SDK tests run sample plugins with the wasmtime CLI. - nativeBuildInputs = [ pkgs.wasmtime ] ++ mold; }); - # Exercise the installed host exactly as the loop launches it: the - # caller has no Cargo on PATH, so only the packaged runtime toolchain - # can run `cargo.build`: the tests under WASI and the wasm32-wasip2 - # build, with C code from the packaged WASI clang. - checks.runtime-toolchain = pkgs.runCommand "sausage-runtime-toolchain" { } '' - set -eu - root="$TMPDIR/runtime-toolchain" - world="$root/world" - emptyPath="$root/empty-bin" - mkdir -p "$emptyPath" "$root/home" "$root/cargo-home" - mkdir -p "$world/plugins" "$world/state" "$world/config" - echo "[]" > "$world/config/boot.yaml" - source="$world/plugins/probe/source" - mkdir -p "$source/src" "$source/tests" - cat > "$source/Cargo.toml" <<'TOML' - [package] - name = "runtime-probe" - version = "0.1.0" - edition = "2024" - TOML - # C code for WASI, compiled with the packaged WASI clang. - printf 'int probe_add(int a, int b) { return a + b; }\n' > "$source/src/add.c" - cat > "$source/build.rs" <<'RUST' - use std::process::Command; - fn main() { - if std::env::var("TARGET").unwrap() != "wasm32-wasip2" { - return; - } - let out = std::env::var("OUT_DIR").unwrap(); - let cc = std::env::var("CC_wasm32_wasip2").expect("CC_wasm32_wasip2"); - let ar = std::env::var("AR_wasm32_wasip2").expect("AR_wasm32_wasip2"); - let object = format!("{out}/add.o"); - assert!(Command::new(cc).args(["-c", "src/add.c", "-o", &object]).status().unwrap().success()); - assert!(Command::new(ar).args(["crs", &format!("{out}/libadd.a"), &object]).status().unwrap().success()); - println!("cargo:rustc-link-search=native={out}"); - println!("cargo:rustc-link-lib=static=add"); - } - RUST - cat > "$source/src/main.rs" <<'RUST' - #[cfg(target_os = "wasi")] - unsafe extern "C" { - fn probe_add(a: i32, b: i32) -> i32; - } - fn main() { - #[cfg(target_os = "wasi")] - assert_eq!(unsafe { probe_add(2, 3) }, 5); - } - RUST - cat > "$source/tests/probe.rs" <<'RUST' - #[test] - fn test_runs_under_wasi() { - assert_eq!(2 + 2, 4); - } - RUST - echo "runtime-check: run packaged host with an empty PATH" - coproc host { - ${pkgs.coreutils}/bin/env -i PATH="$emptyPath" HOME="$root/home" TMPDIR="$root" \ - CARGO_HOME="$root/cargo-home" ${sausage}/bin/sausage run "$world" - } - hostInput=''${host[1]} - hostOutput=''${host[0]} - hostPid=$host_PID - trap 'kill "$hostPid" 2>/dev/null || true' EXIT - printf '%s\n' 'cargo.build {"plugin":"probe"}' >&"$hostInput" - built=0 - while IFS= read -r -t 600 line <&"$hostOutput"; do - echo "$line" - case "$line" in - *'error:'*) - echo "packaged cargo.build failed with an empty caller PATH" >&2 - exit 1 - ;; - *'"passed": 1'*) built=1; break ;; - esac - done - if [ "$built" -ne 1 ]; then - echo "packaged host did not build the probe, or its test did not pass" >&2 - exit 1 - fi - printf '%s\n' 'quit' >&"$hostInput" - wait "$hostPid" - mkdir -p "$out" - touch "$out/passed" + checks.shell = pkgs.runCommand "sausage-shell" { } '' + ${sausage}/bin/sausage sh "$TMPDIR/world" -c \ + 'ls /bin | wc -l; echo hi | tr a-z A-Z; git --version; rg --version' > output + cat output + grep -qx HI output + grep -q 'git version' output + grep -q ripgrep output + touch $out ''; - apps.default = { type = "app"; program = "${sausage}/bin/sausage"; }; - apps.loop = { type = "app"; program = "${loop}/bin/sausage-loop"; }; - devShells.default = pkgs.mkShell (wasiCcEnv // moldEnv // { - packages = [ rust pkgs.wasmtime pkgs.wasm-tools ] ++ mold; + packages = [ rust pkgs.wasmtime pkgs.wasm-tools pkgs.openssl pkgs.pkg-config ] ++ mold; shellHook = '' export CARGO_HOME="$PWD/.cargo-home" '';