Supervise boot services with uniform five-second upgrade grace

cli/tests/host/activation.rs

@@ -160,9 +160,9 @@ impl Drop for TestWorld {
         let _ = std::fs::remove_dir_all(&self.0);
     }
 }
-/// Builds a method-only document without an upgrade delay.
+/// Builds a method-only document; all plugins use the host's five-second grace.
 fn document(name: &str) -> Value {
-    json!({"methods":[{"name":name,"params":[],"result":{"schema":{"type":"string"}}}],"x-sausage-grace":0})
+    json!({"methods":[{"name":name,"params":[],"result":{"schema":{"type":"string"}}}]})
 }
 /// Adds an import of a provider's method list.
 fn consumer(provider: &str, expected: Value) -> Value {
@@ -484,7 +484,7 @@ async fn review_dropped_activation_completes() {
     host.stop().await;
 }
 /// PROTOCOL 10.1: discovery fails after its fixed ten-second deadline and receives EOF.
-/// PROTOCOL 3.7: old processes retain their declared grace while new calls use new code.
+/// PROTOCOL 3.7: old processes get five seconds regardless of declarations; new calls use new code.
 /// PROTOCOL 3.7: retirement waits for all live admissions, including reverted versions.
 /// PROTOCOL 3.7: rejected discovery candidates never receive retirement hooks.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -604,7 +604,7 @@ async fn source_changes_do_not_change_candidate() {
     );
     host.stop().await;
 }
-/// PROTOCOL 3.7: plugins without a declared grace period retain old calls before the default deadline.
+/// PROTOCOL 3.7: missing metadata also retains old calls until the uniform deadline.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn default_grace_retains_old_process() {
     let world = TestWorld::new();
@@ -623,6 +623,104 @@ async fn default_grace_retains_old_process() {
     assert_eq!(host.processes()[&pid].state, "running");
     host.stop().await;
 }
+/// PROTOCOL 3.7: metadata cannot shorten or extend the actual grace for active requests.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn uniform_grace_ignores_discovery_declarations() {
+    let world = TestWorld::new();
+    let declarations = [
+        ("missing", None),
+        ("default", Some(json!(60))),
+        ("long", Some(json!(3600))),
+        ("zero", Some(json!(0))),
+        ("negative", Some(json!(-1))),
+        ("invalid", Some(json!("long"))),
+        ("huge", Some(json!(1e300))),
+    ];
+    for (name, grace) in &declarations {
+        let mut old = document("work");
+        if let Some(grace) = grace {
+            old["x-sausage-grace"] = grace.clone();
+        }
+        world.plugin(name, old, document("work"));
+    }
+    let host = world.start().await;
+    let mut ready = host.subscribe();
+    let mut calls = Vec::new();
+    for (name, _) in &declarations {
+        let host = host.clone();
+        let method = format!("{name}.work");
+        calls.push(tokio::spawn(async move {
+            let result = host.call(&method, json!({})).await;
+            (result, std::time::Instant::now())
+        }));
+    }
+    // Guest notifications are events, not process/file polling.
+    for _ in &declarations {
+        let event = tokio::time::timeout(Duration::from_secs(30), ready.recv())
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(event["params"]["name"], "ready");
+    }
+    let before = std::time::Instant::now();
+    let plugins: Vec<_> = declarations
+        .iter()
+        .map(|(name, _)| json!({"plugin":name,"wasm":format!("/state/{name}/next.wasm")}))
+        .collect();
+    call(&host, "core.activate", json!({"plugins":plugins})).await;
+    // Use the existing discovery+stop test budget, not a new execution limit.
+    tokio::time::timeout(Duration::from_secs(15), async {
+        for task in calls {
+            let (result, ended) = task.await.unwrap();
+            assert_eq!(result.unwrap_err()["code"], -32002);
+            assert!(
+                ended.duration_since(before) >= Duration::from_secs(5),
+                "every declaration gets the full grace"
+            );
+        }
+    })
+    .await
+    .unwrap();
+    host.stop().await;
+}
+
+/// PROTOCOL 10.4/3.7: a watcher refresh uses the same policy as explicit activation.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn manual_copy_uses_uniform_grace() {
+    let world = TestWorld::new();
+    let mut old = document("work");
+    old["x-sausage-grace"] = json!(3600);
+    world.plugin("notes", old, document("work"));
+    let host = world.start().await;
+    // Wait for initial discovery without changing the installed version.
+    call(
+        &host,
+        "core.activate",
+        json!({"plugin":"notes","wasm":"/plugins/notes/plugin.wasm"}),
+    )
+    .await;
+    let mut ready = host.subscribe();
+    let work_host = host.clone();
+    let work = tokio::spawn(async move { work_host.call("notes.work", json!({})).await });
+    let event = tokio::time::timeout(Duration::from_secs(30), ready.recv())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(event["params"]["name"], "ready");
+    let before = std::time::Instant::now();
+    std::fs::copy(fixture(2), world.0.join("plugins/notes/plugin.wasm")).unwrap();
+    assert_eq!(
+        tokio::time::timeout(Duration::from_secs(15), work)
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap_err()["code"],
+        -32002
+    );
+    assert!(before.elapsed() >= Duration::from_secs(5));
+    host.stop().await;
+}
+
 /// PROTOCOL 10.3: candidate imports must fit installed providers.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn checks_candidate_imports() {
@@ -818,11 +916,11 @@ async fn graph_missing_import() {
     assert!(dot.as_str().unwrap().contains("absent"));
     host.stop().await;
 }
-/// Checks the declared upgrade grace and dispatch to the new version.
+/// Checks that an explicit long declaration cannot extend grace; new calls use the new version.
 async fn old_process_grace_and_new_calls() {
     let world = TestWorld::new();
     let mut old = document("add");
-    old["x-sausage-grace"] = json!(0.5);
+    old["x-sausage-grace"] = json!(3600);
     world.plugin("notes", old, document("add"));
     let host = world.start().await;
     call(&host, "notes.hold", json!({})).await;
@@ -868,7 +966,7 @@ async fn superseded_notifies_each_old_process() {
         json!({"jsonrpc":"2.0","method":"rpc.superseded","params":{"current":fixture_hash(2)}})
             .to_string();
     for pid in pids {
-        // marker's existing 30-second bound is shorter than the old 60-second grace.
+        // The notice arrives while the old process is still in its five-second window.
         marker(
             world.0.join(format!("state/notes/superseded-{pid}")),
             &expected,
@@ -993,7 +1091,7 @@ async fn upgrades_enter_the_journal() {
                 .as_array()
                 .unwrap()
                 .iter()
-                .any(|e| e["line"] == "upgrade grace 60 seconds")
+                .any(|e| e["line"] == "upgrade grace 5 seconds")
             {
                 break entries;
             }
@@ -1117,6 +1215,36 @@ async fn old_consumer_can_call_removed_provider_method() {
     host.stop().await;
 }
 
+/// PROTOCOL 8: supervise actual WASI exits on either side of the readiness response.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn crashed_boot_service_restarts_before_and_after_ready() {
+    for when in ["before", "after"] {
+        let world = TestWorld::new();
+        world.plugin("notes", document("serve"), document("serve"));
+        std::fs::write(
+            world.0.join("config/boot.yaml"),
+            json!([{"method":"notes.serve","params":{"crash_once":when}}]).to_string(),
+        )
+        .unwrap();
+        let host = world.start().await;
+        marker(world.0.join("state/notes/service"), "1").await;
+        let entries = call(
+            &host,
+            "core.log",
+            json!({"grep":"service start notes.serve"}),
+        )
+        .await;
+        assert_eq!(entries.as_array().unwrap().len(), 2);
+        let processes = host.processes();
+        let services: Vec<_> = processes.values().filter(|p| p.method == "serve").collect();
+        assert_eq!(services.len(), 1);
+        assert_eq!(services[0].params, json!({"crash_once":when}));
+        let exits = call(&host, "core.log", json!({"plugin":"notes","grep":"exit 1"})).await;
+        assert_eq!(exits.as_array().unwrap().len(), 1);
+        host.stop().await;
+    }
+}
+
 /// PROTOCOL 10.2: activation and revert check interfaces but leave persistent state unchanged.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn state_is_outside_interface_checks() {

cli/tests/host/sdk.rs

@@ -178,6 +178,56 @@ async fn stop_runs_hook() {
     assert_eq!(std::fs::read(world.0.join("stopped")).unwrap(), b"yes");
     host.stop().await;
 }
+/// PROTOCOL 3.7: an idle SDK listener opts into drain and exits before EOF/grace.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn superseded_idle_listener_drains_before_grace() {
+    let world = TestWorld::new();
+    let host = world.start().await;
+    let (notice, mut notices) = tokio::sync::mpsc::unbounded_channel();
+    host.register_native(
+        "other",
+        std::sync::Arc::new(move |_, method, _| {
+            let notice = notice.clone();
+            Box::pin(async move {
+                assert_eq!(method, "superseded");
+                notice.send(std::time::Instant::now()).unwrap();
+                Ok(json!({}))
+            })
+        }),
+    );
+    let pid = call(&host, "probe.drain_service", json!({})).await;
+    // Replace with a different real component; force only skips unrelated test consumers.
+    std::fs::copy(
+        fixtures().join("notes.wasm"),
+        world.0.join("state/next.wasm"),
+    )
+    .unwrap();
+    call(
+        &host,
+        "core.activate",
+        json!({"plugin":"probe","wasm":"/state/next.wasm","force":true}),
+    )
+    .await;
+    // Activation preparation/discovery is outside upgrade grace. Time from the
+    // hook's outgoing call, not from the start of candidate compilation.
+    let before = tokio::time::timeout(Duration::from_secs(60), notices.recv())
+        .await
+        .unwrap()
+        .unwrap();
+    // Waiting on its lifetime lock is an exit event, not process-liveness polling.
+    call(&host, "core.lock", json!({"path":"/drain-owner"})).await;
+    assert!(before.elapsed() < Duration::from_secs(5));
+    assert!(!host.processes().contains_key(pid.as_str().unwrap()));
+    assert!(
+        !world
+            .0
+            .join(format!("stopped-{}", pid.as_str().unwrap()))
+            .exists(),
+        "drain must not run on_stop (discovery processes may run their own hooks)"
+    );
+    host.stop().await;
+}
+
 /// SDK framing carries payloads above 64 KiB in both directions.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn large_messages() {

cli/tests/plugins/activation/src/main.rs

@@ -84,8 +84,31 @@ fn main() {
             .unwrap();
             json!({"retired":version})
         }
+        "work" => {
+            // Keep an actual request unanswered through grace; notices do not drain it.
+            send(
+                json!({"jsonrpc":"2.0","method":"core.notify","params":{"name":"ready","params":{}}}),
+            );
+            for _ in lines {}
+            return;
+        }
         "serve" | "hold" => {
+            // One genuine process failure, followed by a healthy service, without
+            // timers or liveness polling in either the fixture or supervisor.
+            let crash = request["params"]["crash_once"].as_str();
+            let first = crash.is_some()
+                && std::fs::OpenOptions::new()
+                    .write(true)
+                    .create_new(true)
+                    .open(format!("/state/{plugin}/crashed-once"))
+                    .is_ok();
+            if first && crash == Some("before") {
+                std::process::exit(1);
+            }
             send(json!({"jsonrpc":"2.0","id":request["id"],"result":{"version":version}}));
+            if first && crash == Some("after") {
+                std::process::exit(1);
+            }
             if request["method"] == "serve" {
                 std::fs::write(format!("/state/{plugin}/service"), version).unwrap();
             }