Supervise boot services with uniform five-second upgrade grace

cli/src/process.rs

@@ -33,6 +33,8 @@ pub(crate) struct Process {
     pub(crate) id: Option<Value>,
     pub(crate) answer: Option<oneshot::Sender<RpcResult>>,
     pub(crate) ended: watch::Sender<bool>,
+    pub(crate) stopped: bool,
+    pub(crate) superseded: bool,
 }
 impl Host {
     /// Called only after the execution task ends. Wait for the store and file operations before releasing locks.
@@ -57,6 +59,7 @@ impl Host {
                 .map(|_| self.0.journal.query_later(json!({"pid":pid,"limit":20})));
             self.0.journal.forget(pid);
             Self::release_locks(&mut state, pid);
+            self.service_ended(&mut state, pid, status, p.stopped, p.superseded);
             let status = status.to_string();
             tokio::spawn(async move {
                 if let Some(answer) = p.answer {
@@ -84,13 +87,24 @@ impl Host {
     }
     /// Close stdin and start the same five-second deadline for each attached child.
     pub(crate) fn begin_stop(&self, pid: &str) -> Option<watch::Receiver<bool>> {
+        self.stop_process(pid, false)
+    }
+    /// Grace expiry is retirement, not an operator stopping the service.
+    pub(crate) fn begin_retire(&self, pid: &str) -> Option<watch::Receiver<bool>> {
+        self.stop_process(pid, true)
+    }
+    fn stop_process(&self, pid: &str, retirement: bool) -> Option<watch::Receiver<bool>> {
         let mut state = self.0.state.lock().unwrap();
         let p = state.processes.get_mut(pid)?;
+        if !retirement {
+            p.superseded = false;
+        }
         let ended = p.ended.subscribe();
         if p.info.state == "stopping" {
             return Some(ended);
         }
         p.info.state = "stopping".into();
+        p.stopped = true;
         p.input.take();
         self.0.journal.write(pid, &p.info.plugin, "core", "stop");
         for lock in state.locks.values_mut() {
@@ -110,17 +124,24 @@ impl Host {
         let pid = pid.to_string();
         tokio::spawn(async move {
             tokio::time::sleep(std::time::Duration::from_secs(5)).await;
-            h.kill(&pid);
+            h.kill_process(&pid, retirement);
         });
         Some(ended)
     }
     /// Abort a process task and perform its ended-state cleanup.
     pub(crate) fn kill(&self, pid: &str) {
+        self.kill_process(pid, false);
+    }
+    fn kill_process(&self, pid: &str, retirement: bool) {
         let mut state = self.0.state.lock().unwrap();
         for lock in state.locks.values_mut() {
             lock.queue.retain(|(queued, _)| queued != pid);
         }
         if let Some(p) = state.processes.get_mut(pid) {
+            if !retirement {
+                p.superseded = false;
+            }
+            p.stopped = true;
             p.info.state = "stopping".into();
             p.input.take();
             self.0.journal.write(pid, &p.info.plugin, "core", "kill");