Merge v3-t87: builder preserves symlinks, tolerates incomplete caches, shows world paths (T87)

host/src/builder/README.md

@@ -7,7 +7,9 @@ The response contains newline-delimited JSON events.
 The final `exit` event contains a code and component locations.
 Each location contains an executable path and working directory.
 
-The CLI forwards human output immediately.
+The CLI forwards human output as Cargo emits complete lines.
+Displayed source and artifact paths use the world's workspace and target directories.
+The builder hides Cargo's native capture-runner command line.
 The builder captures compiler JSON for artifact discovery without forwarding it to stdout.
 Compiler diagnostics remain on stderr. `cargo run` reserves stdout for the program.
 `cargo new` and `cargo init` use native Cargo, including `--lib`, `--name`, and `--edition`.
@@ -20,6 +22,7 @@ Other commands retain their `--` arguments, such as `fmt -- --check` and Clippy
 
 The host finds the Cargo workspace above the requested directory and copies its sources through the shared VFS.
 It excludes `target` and `.git`.
+The copy preserves source symlinks, including dangling links, without following them.
 
 Each workspace owns a source copy and target directory.
 It also owns a registry and kache cache.
@@ -36,6 +39,7 @@ The PID namespace removes compiler descendants and the private kache daemon afte
 
 The builder copies changed and new source files back through the VFS.
 It preserves other source files, including concurrent edits in the world.
+It leaves source symlinks unchanged rather than publishing them as ordinary files.
 It publishes WASI executables into the world's workspace target directory.
 After `cargo clean`, it removes the published artifacts that Cargo removed from the native target.
 Every target build uses `wasm32-wasip2` and links the shared cwd-init object, including its numeric exit-code interface.
@@ -46,6 +50,7 @@ The host kills timed-out commands before it sends their timeout message.
 A closed response cancels the native command and its process group.
 
 The host retains at most 20 GiB of build directories and evicts the oldest workspace first.
+An incomplete cache entry without a usage marker uses its directory timestamp for eviction.
 
 The Nix wrapper supplies these tools on PATH:
 

host/src/builder/files.rs

@@ -49,7 +49,9 @@ pub async fn copy_out(root: &dyn Layer, source: &str, destination: &Path) -> Res
         }
         let from = sausage_vfs::resolve(source, &entry.name);
         let to = destination.join(&entry.name);
-        if entry.directory {
+        if entry.symlink {
+            std::os::unix::fs::symlink(root.readlink(&from).await?, &to)?;
+        } else if entry.directory {
             before.extend(Box::pin(copy_out(root, &from, &to)).await?);
         } else {
             let bytes = read(root, &from).await?;
@@ -111,6 +113,9 @@ pub async fn copy_back(
             continue;
         }
         let to = sausage_vfs::resolve(destination, &name);
+        if entry.file_type()?.is_symlink() {
+            continue;
+        }
         if entry.file_type()?.is_dir() {
             Box::pin(copy_back(root, &entry.path(), &to, before)).await?;
         } else if before.is_some() || entry.path().extension().is_some_and(|e| e == "wasm") {

host/src/builder/mod.rs

@@ -12,7 +12,7 @@ use std::{
     time::Duration,
 };
 use tokio::{
-    io::AsyncReadExt,
+    io::{AsyncBufReadExt, AsyncReadExt, BufReader},
     process::Command,
     sync::{Mutex, mpsc},
     time::Instant,
@@ -121,6 +121,8 @@ impl Builder {
         if source.exists() {
             std::fs::remove_dir_all(&source)?;
         }
+        std::fs::create_dir_all(&base)?;
+        std::fs::write(base.join("last-used"), [])?;
         let before = files::copy_out(self.root.as_ref(), &workspace, &source).await?;
         for path in [&target, &registry, &cache, &temporary] {
             std::fs::create_dir_all(path)?;
@@ -129,7 +131,6 @@ impl Builder {
             target.join("CACHEDIR.TAG"),
             b"Signature: 8a477f597d28d172789f06886806bc55\n",
         )?;
-        std::fs::write(base.join("last-used"), [])?;
         let relative = world_cwd
             .strip_prefix(&workspace)
             .expect("workspace is an ancestor of cwd")
@@ -139,6 +140,12 @@ impl Builder {
         } else {
             source.join(relative)
         };
+        let display = DisplayPaths {
+            replacements: [
+                (source.to_string_lossy().into_owned(), workspace.clone()),
+                (target.to_string_lossy().into_owned(), world_target.clone()),
+            ],
+        };
         let mut fetch = Command::new("cargo");
         if action == "add" {
             // Resolve the new manifest before fetch, including transitive dependencies.
@@ -150,7 +157,7 @@ impl Builder {
         let mut code = if scaffold || matches!(action, "clean" | "fmt" | "remove") {
             0
         } else {
-            run(fetch, tx, deadline, false).await?.0
+            run(fetch, tx, deadline, false, Some(&display)).await?.0
         };
         if code == 0 && action == "add" {
             let mut fetch = Command::new("cargo");
@@ -158,7 +165,7 @@ impl Builder {
                 .arg("fetch")
                 .current_dir(&cwd)
                 .env("CARGO_HOME", &registry);
-            code = run(fetch, tx, deadline, false).await?.0;
+            code = run(fetch, tx, deadline, false, Some(&display)).await?.0;
         }
         let mut executables = Vec::new();
         if code == 0 {
@@ -198,7 +205,7 @@ impl Builder {
             )?;
             let mut startup = Command::new("nu");
             startup.arg(tools.join("build.nu"));
-            let (startup_code, _) = run(startup, tx, deadline, false).await?;
+            let (startup_code, _) = run(startup, tx, deadline, false, Some(&display)).await?;
             if startup_code != 0 {
                 files::copy_back(self.root.as_ref(), &source, &workspace, Some(&before)).await?;
                 self.evict(20 * 1024 * 1024 * 1024)?;
@@ -257,8 +264,7 @@ impl Builder {
             .args(["--setenv", "SAUSAGE_CWD_INIT"])
             .arg(tools.join("init.o"))
             .args(["--setenv", "SAUSAGE_RUN_PATH"])
-            .arg(&run_path)
-            .args(["--setenv", "CARGO_BUILD_JOBS", "2"]);
+            .arg(&run_path);
             for name in [
                 "CC_wasm32_wasip2",
                 "AR_wasm32_wasip2",
@@ -310,7 +316,7 @@ impl Builder {
                 ]);
             }
             cmd.args(&args[split..]);
-            let (status, stdout) = run(cmd, tx, deadline, compiles).await?;
+            let (status, stdout) = run(cmd, tx, deadline, compiles, Some(&display)).await?;
             code = status;
             if action == "clean" && code == 0 {
                 files::clean_artifacts(self.root.as_ref(), &target, &world_target).await?;
@@ -378,7 +384,15 @@ impl Builder {
             let size = files::size(&path)?;
             total += size;
             entries.push((
-                std::fs::metadata(path.join("last-used"))?.modified()?,
+                std::fs::metadata(path.join("last-used"))
+                    .or_else(|e| {
+                        if e.kind() == std::io::ErrorKind::NotFound {
+                            std::fs::metadata(&path)
+                        } else {
+                            Err(e)
+                        }
+                    })?
+                    .modified()?,
                 path,
                 size,
             ));
@@ -442,16 +456,46 @@ impl Drop for ProcessGroup {
         }
     }
 }
+struct DisplayPaths {
+    replacements: [(String, String); 2],
+}
+impl DisplayPaths {
+    fn render(&self, line: &[u8]) -> Vec<u8> {
+        let mut line = String::from_utf8_lossy(line).into_owned();
+        if line.trim_start().starts_with("Running `") && line.contains("/tools/run.nu") {
+            return Vec::new();
+        }
+        for (native, world) in &self.replacements {
+            line = line
+                .replace(
+                    &format!("{native}/"),
+                    &format!("{}/", world.trim_end_matches('/')),
+                )
+                .replace(native, world);
+        }
+        line.into_bytes()
+    }
+}
 async fn pump(
-    mut reader: impl tokio::io::AsyncRead + Unpin,
+    reader: impl tokio::io::AsyncRead + Unpin,
     tx: &mpsc::Sender<Event>,
     stderr: bool,
     silent_stdout: bool,
+    display: Option<&DisplayPaths>,
 ) -> Result<Vec<u8>> {
+    let mut reader = BufReader::new(reader);
     let mut all = Vec::new();
-    let mut bytes = vec![0; 8192];
+    let mut bytes = Vec::new();
     loop {
-        let n = reader.read(&mut bytes).await?;
+        bytes.clear();
+        let n = if display.is_some() {
+            reader.read_until(b'\n', &mut bytes).await?
+        } else {
+            bytes.resize(8192, 0);
+            let n = reader.read(&mut bytes).await?;
+            bytes.truncate(n);
+            n
+        };
         if n == 0 {
             return Ok(all);
         }
@@ -461,14 +505,14 @@ async fn pump(
         if !stderr && silent_stdout {
             continue;
         }
+        let visible = display.map_or_else(|| bytes.clone(), |display| display.render(&bytes));
+        if visible.is_empty() {
+            continue;
+        }
         tx.send(if stderr {
-            Event::Stderr {
-                bytes: bytes[..n].to_vec(),
-            }
+            Event::Stderr { bytes: visible }
         } else {
-            Event::Stdout {
-                bytes: bytes[..n].to_vec(),
-            }
+            Event::Stdout { bytes: visible }
         })
         .await?;
     }
@@ -478,6 +522,7 @@ async fn run(
     tx: &mpsc::Sender<Event>,
     deadline: Instant,
     silent_stdout: bool,
+    display: Option<&DisplayPaths>,
 ) -> Result<(i32, Vec<u8>)> {
     cmd.stdin(Stdio::null())
         .stdout(Stdio::piped())
@@ -491,8 +536,8 @@ async fn run(
     let collect = async {
         let (status, out, _) = tokio::try_join!(
             async { Ok::<_, anyhow::Error>(child.wait().await?) },
-            pump(out, tx, false, silent_stdout),
-            pump(err, tx, true, false)
+            pump(out, tx, false, silent_stdout, display),
+            pump(err, tx, true, false, display)
         )?;
         Ok::<_, anyhow::Error>((status.code().unwrap_or(1), out))
     };

host/src/builder/tests.rs

@@ -1,6 +1,48 @@
 //! Focused tests for source discovery, argument forwarding, eviction, and process cancellation.
 use super::*;
 #[tokio::test]
+async fn source_links_survive_round_trip() -> Result<()> {
+    let temp = tempfile::tempdir()?;
+    let world = crate::world::World::open(temp.path())?;
+    files::write(world.root.as_ref(), "/project/real.rs", b"source".to_vec()).await?;
+    for (name, target) in [("dangling", "missing"), ("linked.rs", "real.rs")] {
+        std::os::unix::fs::symlink(target, temp.path().join("world/project").join(name))?;
+    }
+    let copy = temp.path().join("copy");
+    let before = files::copy_out(world.root.as_ref(), "/project", &copy).await?;
+    assert_eq!(
+        std::fs::read_link(copy.join("dangling"))?,
+        Path::new("missing")
+    );
+    assert_eq!(std::fs::read(copy.join("linked.rs"))?, b"source");
+    files::copy_back(world.root.as_ref(), &copy, "/project", Some(&before)).await?;
+    assert_eq!(world.root.readlink("/project/dangling").await?, "missing");
+    assert_eq!(world.root.readlink("/project/linked.rs").await?, "real.rs");
+    Ok(())
+}
+#[test]
+fn display_world_paths_and_hide_runner() {
+    let display = DisplayPaths {
+        replacements: [
+            ("/private/source".into(), "/project".into()),
+            ("/private/target/cargo".into(), "/project/target".into()),
+        ],
+    };
+    assert_eq!(
+        display.render(b"Compiling app (/private/source/member)\n"),
+        b"Compiling app (/project/member)\n"
+    );
+    assert_eq!(
+        display.render(b"error: /private/target/cargo/app.wasm\n"),
+        b"error: /project/target/app.wasm\n"
+    );
+    assert!(
+        display
+            .render(b"     Running `nu /private/tools/run.nu /private/target/cargo/app.wasm`\n")
+            .is_empty()
+    );
+}
+#[tokio::test]
 async fn discovers_workspace_and_skips_target() -> Result<()> {
     let temp = tempfile::tempdir()?;
     let world = crate::world::World::open(temp.path())?;
@@ -169,6 +211,7 @@ fn disk_cap_removes_workspaces() -> Result<()> {
     let temp = tempfile::tempdir()?;
     let world = crate::world::World::open(temp.path())?;
     let builder = Builder::new(world.root, &world.host_dir)?;
+    std::fs::create_dir_all(builder.directory.join("incomplete/source"))?;
     for name in ["a", "b"] {
         let path = builder.directory.join(name);
         std::fs::create_dir_all(&path)?;
@@ -192,7 +235,14 @@ async fn timeout_kills_before_output_can_drain() -> Result<()> {
     let mut cmd = Command::new("sh");
     cmd.args(["-c", "echo $$; exec sleep 1000"]);
     let task = tokio::spawn(async move {
-        run(cmd, &output, Instant::now() + Duration::from_secs(1), false).await
+        run(
+            cmd,
+            &output,
+            Instant::now() + Duration::from_secs(1),
+            false,
+            None,
+        )
+        .await
     });
     let Some(Event::Stdout { bytes }) =
         tokio::time::timeout(Duration::from_secs(2), rx.recv()).await?
@@ -219,7 +269,14 @@ async fn closed_receiver_cancels_command() -> Result<()> {
     let mut cmd = Command::new("sh");
     cmd.args(["-c", "printf ready; exec sleep 1000"]);
     let task = tokio::spawn(async move {
-        run(cmd, &tx, Instant::now() + Duration::from_secs(60), false).await
+        run(
+            cmd,
+            &tx,
+            Instant::now() + Duration::from_secs(60),
+            false,
+            None,
+        )
+        .await
     });
     // Output must arrive before the command exits, not after collection completes.
     let event = tokio::time::timeout(Duration::from_secs(2), rx.recv())

host/tests/it/builder.rs

@@ -116,6 +116,11 @@ async fn cargo_scaffold_and_human_output() -> anyhow::Result<()> {
             !stderr.contains("\"reason\":\"compiler-artifact\""),
             "{stderr}"
         );
+        assert!(!stderr.contains("/host/builds/"), "{stderr}");
+        assert!(!stderr.contains("tools/run.nu"), "{stderr}");
+        if script.contains("cargo run") {
+            assert!(stderr.contains("(/hello)"), "{stderr}");
+        }
         if script == "cargo build" {
             assert!(stderr.contains("Finished"), "{stderr}");
         }
@@ -168,6 +173,10 @@ async fn cargo_build_test_and_source_changes() -> anyhow::Result<()> {
     let source = temp.path().join("world/project");
     std::fs::create_dir_all(source.join("src"))?;
     std::fs::create_dir_all(source.join("tests"))?;
+    std::fs::create_dir_all(source.join("target"))?;
+    std::fs::write(source.join("target/keep.txt"), "existing target")?;
+    std::os::unix::fs::symlink("missing", source.join("dangling"))?;
+    std::fs::create_dir_all(temp.path().join("host/builds/incomplete/source"))?;
     for (path, bytes) in [
         (
             "Cargo.toml",
@@ -196,6 +205,16 @@ async fn cargo_build_test_and_source_changes() -> anyhow::Result<()> {
     let (code, out) = cargo(&runtime, &["build"]).await?;
     assert_eq!(code, 0, "{out}");
     assert!(out.contains("namespace assertions passed"), "{out}");
+    assert!(out.contains("(/project)"), "{out}");
+    assert!(!out.contains("/host/builds/"), "{out}");
+    assert_eq!(
+        std::fs::read_link(source.join("dangling"))?,
+        Path::new("missing")
+    );
+    assert_eq!(
+        std::fs::read_to_string(source.join("target/keep.txt"))?,
+        "existing target"
+    );
     assert!(source.join("Cargo.lock").exists());
     let mut shell_output = Vec::new();
     let shell_code = runtime
@@ -268,8 +287,10 @@ async fn cargo_build_test_and_source_changes() -> anyhow::Result<()> {
     assert_ne!(code, 0, "{out}");
     assert!(out.contains("stdout"), "{out}");
     let base = std::fs::read_dir(temp.path().join("host/builds"))?
-        .next()
-        .unwrap()?
+        .collect::<std::io::Result<Vec<_>>>()?
+        .into_iter()
+        .find(|entry| entry.path().join("last-used").is_file())
+        .unwrap()
         .path();
     let before = cache_hits(&base)?;
     let (code, out) = cargo(&runtime, &["clean"]).await?;