sausage/host/tests/it/watchdog.rs

//! World-loaded watchdog recovery with SIGKILL during its durable wait.
use sausage::{harness::run_module, world::World};
use std::{
    path::Path,
    process::Command,
    time::{Duration, Instant},
};
const BUNDLE: &str = include_str!("../../js/watchdog-test-bundle.js");
static PREPARE: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn prepare(dir: &Path) -> anyhow::Result<()> {
    let _guard = PREPARE.lock().unwrap();
    let disk = dir.join("world");
    let repo = disk.join("src/sausage");
    if repo.join(".git").exists() {
        return Ok(());
    }
    let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
    let target = crate::common::build_git();
    std::fs::create_dir_all(disk.join("bin"))?;
    for binary in ["git", "tar"] {
        std::fs::copy(
            target.join(format!("wasm32-wasip2/release/{binary}.wasm")),
            disk.join(format!("bin/{binary}.wasm")),
        )?;
    }
    for directory in [
        "extensions/subagents",
        "extensions/messages",
        "extensions/watchdog",
        "extensions/lineage",
        "extensions/core",
        "extensions/conversations",
        "extensions/models",
        "agents",
    ] {
        std::fs::create_dir_all(repo.join(directory))?;
    }
    for directory in [
        "extensions/subagents",
        "extensions/messages",
        "extensions/watchdog",
        "extensions/lineage",
        "extensions/models",
    ] {
        for entry in std::fs::read_dir(source.join(directory))? {
            let entry = entry?;
            if entry.path().extension().is_some_and(|ext| ext == "ts") {
                std::fs::copy(entry.path(), repo.join(directory).join(entry.file_name()))?;
            }
        }
    }
    std::fs::create_dir_all(repo.join("extensions/probe"))?;
    std::fs::write(
        repo.join("extensions/probe/server.ts"),
        "export default host => { globalThis.testHost = host; globalThis.subagentService = host.need({name:'subagents'}); globalThis.messageService = host.need({name:'messages'}); globalThis.watchdogService = host.need({name:'watchdog'}); globalThis.lineageService = host.need({name:'lineage'}); host.implement({name:'conversations'}, {summaryModel: async () => globalThis.summaryRef, concludedIn: async () => new Set(), generatedTitle: async () => {}, setStanding: async () => {}, setAutomaticSummaries: async () => {}}); return {name:'probe'}; }",
    )?;
    std::fs::copy(
        source.join("extensions/core/agents.ts"),
        repo.join("extensions/core/agents.ts"),
    )?;
    std::fs::copy(
        source.join("extensions/conversations/api.ts"),
        repo.join("extensions/conversations/api.ts"),
    )?;
    crate::common::write_config(
        dir,
        "standing",
        serde_json::json!([
          {
            "name": "scout",
            "description": "Search",
            "resident": true,
            "denyTools": [
              "subagent"
            ],
            "isolate": "none",
            "model": "faux/faux-1",
            "prompt": "Scout instructions"
          },
          {
            "name": "standing",
            "description": "Primary",
            "resident": true,
            "isolate": "none",
            "model": "faux/faux-1",
            "prompt": "Primary agent"
          },
          {
            "name": "coordinator",
            "description": "Coordinate",
            "isolate": "none",
            "model": "faux/faux-1",
            "prompt": "Coordinator agent"
          }
        ]),
    )?;
    for args in [
        vec!["init", "-b", "main"],
        vec!["add", "."],
        vec!["commit", "--no-gpg-sign", "-m", "watchdog fixture"],
    ] {
        assert!(
            Command::new("git")
                .args(args)
                .current_dir(&repo)
                .env("GIT_AUTHOR_NAME", "Test")
                .env("GIT_AUTHOR_EMAIL", "test@example.com")
                .env("GIT_COMMITTER_NAME", "Test")
                .env("GIT_COMMITTER_EMAIL", "test@example.com")
                .status()?
                .success()
        );
    }
    Ok(())
}
async fn run(dir: &Path, mode: &str) -> anyhow::Result<()> {
    let world = World::open(dir)?;
    prepare(dir)?;
    let exec = sausage::process::exec(World {
        root: world.root.clone(),
        host_dir: world.host_dir.clone(),
        processes: world.processes.clone(),
    })?;
    run_module(
        world,
        exec,
        &format!("globalThis.watchdogMode={mode:?};\n{BUNDLE}"),
    )
    .await
}
#[tokio::test(flavor = "current_thread")]
async fn coordination() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    run(dir.path(), "normal").await
}
#[tokio::test(flavor = "current_thread")]
async fn restart_child() -> anyhow::Result<()> {
    if let Ok(dir) = std::env::var("WATCHDOG_DIR") {
        run(Path::new(&dir), &std::env::var("WATCHDOG_MODE")?).await?;
    }
    Ok(())
}
#[test]
fn sigkill_during_durable_wait() -> anyhow::Result<()> {
    let dir = tempfile::tempdir()?;
    World::open(dir.path())?;
    prepare(dir.path())?;
    let spawn = |mode: &str| {
        Command::new(std::env::current_exe().unwrap())
            .args(["--exact", "watchdog::restart_child", "--nocapture"])
            .env("WATCHDOG_DIR", dir.path())
            .env("WATCHDOG_MODE", mode)
            .spawn()
    };
    let mut child = spawn("kill")?;
    let deadline = Instant::now() + Duration::from_secs(30);
    while !dir.path().join("world/watchdog-ready").exists() {
        if let Some(status) = child.try_wait()? {
            anyhow::bail!("Child exited before reply: {status}");
        }
        if Instant::now() > deadline {
            child.kill()?;
            child.wait()?;
            anyhow::bail!("Delivery did not reach reply");
        }
        std::thread::yield_now();
    }
    child.kill()?;
    child.wait()?;
    for _ in 0..2 {
        let mut child = spawn("resume")?;
        let deadline = Instant::now() + Duration::from_secs(30);
        loop {
            if let Some(status) = child.try_wait()? {
                assert!(status.success());
                break;
            }
            if Instant::now() > deadline {
                child.kill()?;
                child.wait()?;
                anyhow::bail!("Reply recovery timed out");
            }
            std::thread::yield_now();
        }
    }
    Ok(())
}