Merge v3-t56 (T56)

# Conflicts:
#	host/js/package.json

host/js/p4-test.ts

@@ -0,0 +1,86 @@
+/** P4 fixture provider for the real console: all project work uses the installed bash tool. */
+import { fauxProvider, fauxAssistantMessage, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
+
+const main = `fn greeting(name: &str) -> String { format!("hello {name}") }
+fn main() {
+    let Some(name) = std::env::args().nth(1) else { std::process::exit(7) };
+    println!("{}", serde_json::json!({"greeting": greeting(&name)}));
+}
+#[test]
+fn greeting_test() { assert_eq!(greeting("world"), "hello world", "P4_expected_failure"); }
+`;
+const write = (path: string, text: string) => `cat > ${path} <<'P4_FILE'\n${text}\nP4_FILE\n`;
+const phases = [
+  `set -e
+mkdir -p /src/demo-cli/src /src/demo-node /tmp
+cd /src/demo-cli
+if test -d target; then cargo clean; fi
+${write('Cargo.toml', '[package]\nname="demo-cli"\nversion="0.1.0"\nedition="2024"\n[workspace]\n')}
+${write('src/main.rs', main.replace('"hello world", "P4_expected_failure"', '"hello wrong", "P4_expected_failure"'))}
+cargo add serde_json
+cargo fmt
+set +e
+cargo test -- --nocapture > /tmp/p4-fail.stdout 2> /tmp/p4-fail.stderr
+code=$?
+set -e
+cat /tmp/p4-fail.stdout /tmp/p4-fail.stderr
+test "$code" = 101
+case "$(cat /tmp/p4-fail.stderr)" in *P4_expected_failure*) ;; *) exit 1 ;; esac
+printf 'P4_PHASE_0_OK\n'`,
+  `set -e
+cd /src/demo-cli
+${write('src/main.rs', main)}
+cargo fmt
+cargo test -- --nocapture
+cargo build --release
+test -f target/wasm32-wasip2/release/demo-cli.wasm
+sh -c '/src/demo-cli/target/wasm32-wasip2/release/demo-cli.wasm world' > /tmp/p4-cli.json
+case "$(cat /tmp/p4-cli.json)" in *'"greeting":"hello world"'*) ;; *) exit 1 ;; esac
+set +e
+sh -c '/src/demo-cli/target/wasm32-wasip2/release/demo-cli.wasm'
+code=$?
+set -e
+test "$code" = 7
+printf 'P4_PHASE_1_OK\n'`,
+  `set -e
+${write('/src/demo-node/package.json', '{"scripts":{"test":"node test.cjs"}}')}
+${write('/src/demo-node/test.cjs', 'const fs = require("node:fs");\nrequire("node:assert/strict").equal(2 + 2, 4);\nfs.writeFileSync("result.txt", "P4 npm test passed\\n");')}
+id=$(box create --ttl 600)
+printf '%s\n' "$id" > /tmp/p4-box-id
+trap 'box rm "$id"' EXIT
+box put "$id" /src/demo-node /work
+box exec "$id" --timeout 120 -- sh -c 'cd /work && npm test'
+box get "$id" /work/result.txt /src/demo-node/result.txt
+test "$(cat /src/demo-node/result.txt)" = 'P4 npm test passed'
+box rm "$id"
+trap - EXIT
+printf 'P4_PHASE_2_OK\n'`,
+  `set -e
+undo list --conversation 1 > /tmp/p4-undo.json
+box ls > /tmp/p4-boxes
+printf 'P4_PHASE_3_OK\n'`,
+];
+export default function p4(host: any) {
+  // Replace the console's transport before resume. No real model request can occur.
+  const faux = fauxProvider({ provider: 'openai', models: [{ id: 'gpt-6.1-sol' }], tokensPerSecond: 1000000 });
+  const respond = (transcript: any): any => {
+    faux.appendResponses([respond]);
+    const messages = transcript.messages;
+    const input = messages.findLast((message: any) => message.role === 'user');
+    const text = (message: any) => typeof message.content === 'string' ? message.content : (message.content ?? []).filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n');
+    const round = /P4 start (\d+)/.exec(text(input))?.[1];
+    if (!round) throw new Error(`Unexpected P4 input: ${JSON.stringify(input)}`);
+    const last = messages.findLast((message: any) => message.role !== 'system');
+    let phase = 0;
+    if (last.role !== 'user') {
+      const prior = /p4-\d+-(\d+)/.exec(last.toolCallId)?.[1];
+      if (last.isError || prior === undefined || !text(last).includes(`P4_PHASE_${prior}_OK`)) throw new Error(`P4 bash failed: ${JSON.stringify(last)}`);
+      phase = Number(prior) + 1;
+    }
+    if (phase === phases.length) return fauxAssistantMessage(`P4_COMPLETE_${round}`);
+    return fauxAssistantMessage([fauxToolCall('bash', { command: phases[phase], timeout: 300 }, { id: `p4-${round}-${phase}` })], { stopReason: 'toolUse' });
+  };
+  faux.setResponses([respond]);
+  host.models().setProvider(faux.provider);
+  return { name: 'p4' };
+}

host/js/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "type": "module",
-  "scripts": { "bundle": "nu apply-patch.nu && esbuild main.ts --bundle --format=esm --platform=browser --outfile=bundle.js && esbuild test.ts --bundle --format=esm --platform=browser --outfile=test-bundle.js && esbuild durable-test.ts --bundle --format=esm --platform=browser --outfile=durable-test-bundle.js && esbuild foundation-test.ts --bundle --format=esm --platform=browser --outfile=foundation-test-bundle.js && esbuild extensions-test.ts --bundle --format=esm --platform=browser --outfile=extensions-test-bundle.js && esbuild events-test.ts --bundle --format=esm --platform=browser --outfile=events-test-bundle.js && esbuild subagents-test.ts --bundle --format=esm --platform=browser --outfile=subagents-test-bundle.js && esbuild core-ext-test.ts --bundle --format=esm --platform=browser --outfile=core-ext-test-bundle.js && esbuild plan-todos-test.ts --bundle --format=esm --platform=browser --outfile=plan-todos-test-bundle.js && esbuild p2-test.ts --bundle --format=esm --platform=browser --outfile=p2-test-bundle.js && esbuild chain-test.ts --bundle --format=esm --platform=browser --outfile=chain-test-bundle.js && esbuild undo-harness-test.ts --bundle --format=esm --platform=browser --outfile=undo-harness-test-bundle.js && esbuild messages-test.ts --bundle --format=esm --platform=browser --outfile=messages-test-bundle.js && esbuild p3-test.ts --bundle --format=esm --platform=browser --outfile=p3-test-bundle.js && esbuild automations-test.ts --bundle --format=esm --platform=browser --outfile=automations-test-bundle.js && esbuild watchdog-test.ts --bundle --format=esm --platform=browser --outfile=watchdog-test-bundle.js && esbuild workspaces-test.ts --bundle --format=esm --platform=browser --outfile=workspaces-test-bundle.js && esbuild goal-questions-test.ts --bundle --format=esm --platform=browser --outfile=goal-questions-test-bundle.js && esbuild dream-test.ts --bundle --format=esm --platform=browser --outfile=dream-test-bundle.js" },
+  "scripts": { "bundle": "nu apply-patch.nu && esbuild main.ts --bundle --format=esm --platform=browser --outfile=bundle.js && esbuild test.ts --bundle --format=esm --platform=browser --outfile=test-bundle.js && esbuild durable-test.ts --bundle --format=esm --platform=browser --outfile=durable-test-bundle.js && esbuild foundation-test.ts --bundle --format=esm --platform=browser --outfile=foundation-test-bundle.js && esbuild extensions-test.ts --bundle --format=esm --platform=browser --outfile=extensions-test-bundle.js && esbuild events-test.ts --bundle --format=esm --platform=browser --outfile=events-test-bundle.js && esbuild subagents-test.ts --bundle --format=esm --platform=browser --outfile=subagents-test-bundle.js && esbuild core-ext-test.ts --bundle --format=esm --platform=browser --outfile=core-ext-test-bundle.js && esbuild plan-todos-test.ts --bundle --format=esm --platform=browser --outfile=plan-todos-test-bundle.js && esbuild p2-test.ts --bundle --format=esm --platform=browser --outfile=p2-test-bundle.js && esbuild chain-test.ts --bundle --format=esm --platform=browser --outfile=chain-test-bundle.js && esbuild undo-harness-test.ts --bundle --format=esm --platform=browser --outfile=undo-harness-test-bundle.js && esbuild messages-test.ts --bundle --format=esm --platform=browser --outfile=messages-test-bundle.js && esbuild p3-test.ts --bundle --format=esm --platform=browser --outfile=p3-test-bundle.js && esbuild automations-test.ts --bundle --format=esm --platform=browser --outfile=automations-test-bundle.js && esbuild watchdog-test.ts --bundle --format=esm --platform=browser --outfile=watchdog-test-bundle.js && esbuild workspaces-test.ts --bundle --format=esm --platform=browser --outfile=workspaces-test-bundle.js && esbuild goal-questions-test.ts --bundle --format=esm --platform=browser --outfile=goal-questions-test-bundle.js && esbuild dream-test.ts --bundle --format=esm --platform=browser --outfile=dream-test-bundle.js && esbuild p4-test.ts --bundle --format=esm --platform=browser --outfile=p4-test-bundle.js" },
   "dependencies": {
     "@earendil-works/pi-durable": "1.5.0",
     "@earendil-works/pi-ai": "1.5.0",

host/src/builder/README.md

@@ -63,6 +63,6 @@ It does not depend on a user-profile path or a host HOME file.
 Development commands retain the global Cargo configuration and its kache wrapper.
 The sandbox uses a private Cargo home and selects kache through `--config build.rustc-wrapper`.
 
-For `cargo add`, the network phase performs a dry run to populate the registry index.
-The offline command makes the actual manifest change inside the namespaces.
+For `cargo add`, the network phase updates the private source copy and fetches the new dependencies, including transitive dependencies.
+The command then repeats offline inside the namespaces before source publication.
 The disk cap applies after each completed command, including compiler failures.

host/src/builder/mod.rs

@@ -134,7 +134,8 @@ impl Builder {
         };
         let mut fetch = Command::new("cargo");
         if action == "add" {
-            fetch.args(&request.argv).arg("--dry-run");
+            // Resolve the new manifest before fetch, including transitive dependencies.
+            fetch.args(&request.argv);
         } else {
             fetch.arg("fetch");
         }
@@ -144,6 +145,14 @@ impl Builder {
         } else {
             run(fetch, tx, deadline).await?.0
         };
+        if code == 0 && action == "add" {
+            let mut fetch = Command::new("cargo");
+            fetch
+                .arg("fetch")
+                .current_dir(&cwd)
+                .env("CARGO_HOME", &registry);
+            code = run(fetch, tx, deadline).await?.0;
+        }
         let mut executables = Vec::new();
         if code == 0 {
             let tools = base.join("tools");

host/tests/p4.rs

@@ -0,0 +1,277 @@
+//! P4 exit: the real console agent builds Rust and Node projects, then inspects undo history.
+use anyhow::{Context, ensure};
+use sausage::world::World;
+use sausage_vfs::journal::Entry;
+use std::{
+    fs,
+    io::Write,
+    path::{Path, PathBuf},
+    process::{Child, Command, Stdio},
+    time::{Duration, Instant},
+};
+
+fn checked(command: &mut Command) -> anyhow::Result<()> {
+    ensure!(command.status()?.success(), "command failed: {command:?}");
+    Ok(())
+}
+fn prepare(dir: &Path) -> anyhow::Result<PathBuf> {
+    let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
+    let userland = source.join("userland");
+    let target = PathBuf::from(std::env::var("CARGO_TARGET_DIR")?);
+    let bundle = Path::new(env!("CARGO_TARGET_TMPDIR")).join("p4-bundle");
+    let revision = Command::new("git")
+        .current_dir(&source)
+        .args(["rev-parse", "HEAD"])
+        .output()?;
+    ensure!(revision.status.success(), "fixture revision");
+    if fs::read(bundle.join("revision")).ok().as_deref() != Some(revision.stdout.as_slice()) {
+        if !userland.join("cwd-init/init.o").exists() {
+            checked(Command::new("nu").arg(userland.join("cwd-init/build.nu")))?;
+        }
+        checked(Command::new("cargo").current_dir(&userland).args([
+            "build",
+            "--release",
+            "--locked",
+            "-p",
+            "git",
+            "-p",
+            "world-cargo",
+            "-p",
+            "box-cli",
+            "-p",
+            "undo",
+            "-p",
+            "sausage-coreutils",
+        ]))?;
+        checked(Command::new("nu").arg(userland.join("brush/build.nu")))?;
+        fs::create_dir_all(bundle.join("bin"))?;
+        fs::create_dir_all(bundle.join("src"))?;
+        for (binary, name) in [
+            ("brush", "sh"),
+            ("brush", "bash"),
+            ("git", "git"),
+            ("cargo", "cargo"),
+            ("box", "box"),
+            ("undo", "undo"),
+            ("cat", "cat"),
+            ("mkdir", "mkdir"),
+        ] {
+            fs::copy(
+                target.join(format!("wasm32-wasip2/release/{binary}.wasm")),
+                bundle.join(format!("bin/{name}.wasm")),
+            )?;
+        }
+        fs::write(bundle.join("revision"), revision.stdout)?;
+    }
+    World::open_with_userland(dir, Some(&bundle))?;
+    let repo = dir.join("world/src/sausage");
+    fs::create_dir_all(repo.join("extensions/p4"))?;
+    fs::write(
+        repo.join("extensions/p4/server.ts"),
+        include_str!("../js/p4-test-bundle.js"),
+    )?;
+    for args in [
+        vec!["init", "-b", "main"],
+        vec!["add", "."],
+        vec!["commit", "--no-gpg-sign", "-m", "P4 fixture"],
+    ] {
+        checked(
+            Command::new("git")
+                .current_dir(&repo)
+                .args(args)
+                .env("GIT_AUTHOR_NAME", "P4 test")
+                .env("GIT_AUTHOR_EMAIL", "p4@example.com")
+                .env("GIT_COMMITTER_NAME", "P4 test")
+                .env("GIT_COMMITTER_EMAIL", "p4@example.com"),
+        )?;
+    }
+    Ok(bundle)
+}
+struct Running(Child);
+impl Drop for Running {
+    fn drop(&mut self) {
+        let _ = self.0.kill();
+        let _ = self.0.wait();
+    }
+}
+fn round(dir: &Path, bundle: &Path, number: usize) -> anyhow::Result<()> {
+    let stdout = dir.join(format!("round-{number}.stdout"));
+    let stderr = dir.join(format!("round-{number}.stderr"));
+    let mut child = Running(
+        Command::new(env!("CARGO_BIN_EXE_sausage"))
+            .arg("run")
+            .arg(dir)
+            .arg("--userland")
+            .arg(bundle)
+            .args(["--api-listen", "127.0.0.1:0"])
+            .env("PI_PROXY_BASE_URL", "https://proxy.invalid")
+            .env("PI_PROXY_API_KEY", "faux-token")
+            .env_remove("CREDENTIALS_DIRECTORY")
+            .stdin(Stdio::piped())
+            .stdout(fs::File::create(&stdout)?)
+            .stderr(fs::File::create(&stderr)?)
+            .spawn()?,
+    );
+    writeln!(
+        child.0.stdin.take().context("console stdin")?,
+        "P4 start {number}"
+    )?;
+    let deadline = Instant::now() + Duration::from_secs(600);
+    loop {
+        if fs::read_to_string(&stdout)?.contains(&format!("P4_COMPLETE_{number}")) {
+            break;
+        }
+        if let Some(status) = child.0.try_wait()? {
+            anyhow::bail!("console exited: {status}");
+        }
+        ensure!(
+            Instant::now() < deadline,
+            "P4 round {number} exceeded its deadline"
+        );
+        std::thread::yield_now();
+    }
+    checked(Command::new("kill").args(["-TERM", &child.0.id().to_string()]))?;
+    let deadline = Instant::now() + Duration::from_secs(20);
+    loop {
+        if let Some(status) = child.0.try_wait()? {
+            ensure!(status.success(), "console shutdown: {status}");
+            break;
+        }
+        ensure!(
+            Instant::now() < deadline,
+            "console shutdown exceeded its deadline"
+        );
+        std::thread::yield_now();
+    }
+    Ok(())
+}
+fn boxes_removed(dir: &Path, bundle: &Path) -> anyhow::Result<()> {
+    let path = dir.join("world/tmp/p4-box-id");
+    if !path.exists() {
+        return Ok(());
+    }
+    let id = fs::read_to_string(path)?;
+    let id = id.trim();
+    ensure!(
+        id.starts_with("tc-") && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'),
+        "invalid box ID"
+    );
+    // The agent's EXIT trap is primary cleanup. Recover its box even if the console fails.
+    let output = Command::new(env!("CARGO_BIN_EXE_sausage"))
+        .arg("sh")
+        .arg(dir)
+        .arg("--userland")
+        .arg(bundle)
+        .arg("-c")
+        .arg("box ls")
+        .env_remove("CREDENTIALS_DIRECTORY")
+        .output()?;
+    ensure!(
+        output.status.success(),
+        "box ls: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    if String::from_utf8_lossy(&output.stdout).contains(id) {
+        checked(
+            Command::new(env!("CARGO_BIN_EXE_sausage"))
+                .arg("sh")
+                .arg(dir)
+                .arg("--userland")
+                .arg(bundle)
+                .arg("-c")
+                .arg(format!("box rm {id}"))
+                .env_remove("CREDENTIALS_DIRECTORY"),
+        )?;
+        anyhow::bail!("Agent left {id} active. Test cleanup removed it");
+    }
+    println!("P4 box absent from service inventory: {id}");
+    Ok(())
+}
+fn cache_hits(dir: &Path) -> anyhow::Result<u64> {
+    let workspace = fs::read_dir(dir.join("host/builds"))?
+        .next()
+        .context("builder workspace")??
+        .path();
+    let output = Command::new("kache")
+        .arg("stats")
+        .env("KACHE_CACHE_DIR", workspace.join("kache"))
+        .output()?;
+    ensure!(
+        output.status.success(),
+        "kache: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let text = String::from_utf8(output.stdout)?;
+    println!("P4 PRIVATE CACHE:\n{text}");
+    let words: Vec<_> = text.split_whitespace().collect();
+    words
+        .windows(2)
+        .find_map(|pair| {
+            if pair[1] == "of" {
+                pair[0].replace(',', "").parse().ok()
+            } else {
+                None
+            }
+        })
+        .context("cache hit count")
+}
+#[test]
+fn agent_builds_rust_and_node_twice() -> anyhow::Result<()> {
+    let dir = tempfile::tempdir()?;
+    let bundle = prepare(dir.path())?;
+    let mut first_hits = 0;
+    for number in 1..=2 {
+        let outcome = round(dir.path(), &bundle, number);
+        let cleanup = boxes_removed(dir.path(), &bundle);
+        if outcome.is_err() {
+            for stream in ["stdout", "stderr"] {
+                eprintln!(
+                    "P4 round {number} {stream}:\n{}",
+                    fs::read_to_string(dir.path().join(format!("round-{number}.{stream}")))?
+                );
+            }
+        }
+        outcome?;
+        cleanup?;
+        let world = dir.path().join("world");
+        ensure!(
+            world
+                .join("src/demo-cli/target/wasm32-wasip2/release/demo-cli.wasm")
+                .is_file(),
+            "missing world artifact"
+        );
+        ensure!(
+            fs::read_to_string(world.join("src/demo-cli/Cargo.toml"))?.contains("serde_json"),
+            "cargo add did not reach world"
+        );
+        ensure!(
+            fs::read_to_string(world.join("src/demo-node/result.txt"))? == "P4 npm test passed\n",
+            "Node result missing"
+        );
+        let entries: Vec<Entry> =
+            serde_json::from_slice(&fs::read(world.join("tmp/p4-undo.json"))?)?;
+        for (phase, path) in [
+            (0, "/src/demo-cli/Cargo.toml"),
+            (1, "/src/demo-cli/src/main.rs"),
+        ] {
+            let tag = format!("p4-{number}-{phase}");
+            ensure!(
+                entries.iter().any(|entry| entry.tag == tag
+                    && entry.program == "bash"
+                    && entry.changed_paths.iter().any(|p| p == path)),
+                "undo lost {tag}: {path}"
+            );
+        }
+        let hits = cache_hits(dir.path())?;
+        if number == 1 {
+            first_hits = hits;
+        } else {
+            ensure!(
+                hits > first_hits,
+                "second run missed kache: {first_hits} -> {hits}"
+            );
+            println!("P4 CACHE SECOND RUN: {first_hits} -> {hits} hits");
+        }
+    }
+    Ok(())
+}