host: preserve journals, full traps and embedded agent output

Write new journal entries under logs/core and merge read-only legacy
history without moving SQLite/WAL files. Atomically spool failed batches,
retaining unspoolable entries for event-driven retries. Journal full trap
diagnostics before UTF-8-safe status display. Port session overflow archives
and followable line/byte excerpts to the still-shipped embedded agent.
Document retention/recovery and cover the contracts with regressions.
Include one rustfmt-only normalization in the existing SDK WASI test.

Card: SAU-143

agent/src/main.rs

@@ -94,19 +94,38 @@ impl Agent {
                 return Ok(final_text);
             }
             for (id, name, arguments) in calls {
-                let output = match serde_json::from_str(&arguments) {
-                    Ok(args) => tools::run(&name, &args)
+                let args = serde_json::from_str::<serde_json::Value>(&arguments);
+                let output = match &args {
+                    Ok(args) => tools::run(&name, args)
                         .await
                         .unwrap_or_else(|error| json!({"error":error})),
                     Err(error) => json!({"error":format!("arguments are not valid JSON: {error}")}),
                 };
-                append(
-                    &mut entries,
-                    Entry::ToolResult {
-                        id,
-                        output: tools::limit_output(output.to_string()),
-                    },
-                )?;
+                let output = if name == "read_file"
+                    && args
+                        .as_ref()
+                        .ok()
+                        .and_then(|a| a["path"].as_str())
+                        .is_some_and(|p| p.starts_with("/logs/"))
+                {
+                    output["content"]
+                        .as_str()
+                        .map(str::to_owned)
+                        .unwrap_or_else(|| output.to_string())
+                } else {
+                    output.to_string()
+                };
+                let tail = args
+                    .as_ref()
+                    .ok()
+                    .and_then(|a| a["method"].as_str())
+                    .is_some_and(|m| {
+                        [".exec", ".log", ".build", ".test"]
+                            .iter()
+                            .any(|suffix| m.ends_with(suffix))
+                    });
+                let output = tools::overflow::guard(output, "default", &id, tail);
+                append(&mut entries, Entry::ToolResult { id, output })?;
             }
         }
     }

agent/src/overflow.rs

@@ -0,0 +1,258 @@
+//! Context-size guard; full output is in the referenced file. Never a data limit.
+use std::io::{self, Write};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+// One policy point: Pi's 50 KiB / 2000 lines, including room for the reference.
+const CONTEXT_LIMIT: (usize, usize) = (50 * 1024, 2000);
+static NEXT: AtomicU64 = AtomicU64::new(0);
+
+fn component(s: &str) -> String {
+    s.bytes()
+        .take(80)
+        .map(|b| {
+            if b.is_ascii_alphanumeric() || b == b'-' || b == b'_' {
+                b as char
+            } else {
+                '_'
+            }
+        })
+        .collect()
+}
+
+pub fn exceeds_limit(output: &str) -> bool {
+    output.len() > CONTEXT_LIMIT.0 || output.lines().count() > CONTEXT_LIMIT.1
+}
+
+/// Save before excerpting. On storage failure return all data, not a false reference.
+pub fn guard(output: String, session: &str, id: &str, tail: bool) -> String {
+    if !exceeds_limit(&output) {
+        return output;
+    }
+    // Decode only results that crossed the context guard. Archive physical
+    // string lines, not JSON escapes, even if decoding makes the text smaller.
+    let output = serde_json::from_str::<serde_json::Value>(&output)
+        .map(|value| super::render(&value))
+        .unwrap_or(output);
+    let pid = std::env::var("SAUSAGE_PID").unwrap_or_else(|_| "test".into());
+    let dir = format!("/logs/agent/{}/{}", date(), component(session));
+    let saved = (|| -> io::Result<String> {
+        std::fs::create_dir_all(&dir)?;
+        // create_new prevents overwriting even if an ID is reused or a process restarts.
+        loop {
+            let path = format!(
+                "{dir}/{}-{}-{}.txt",
+                component(id),
+                component(&pid),
+                NEXT.fetch_add(1, Ordering::Relaxed)
+            );
+            match std::fs::OpenOptions::new()
+                .write(true)
+                .create_new(true)
+                .open(&path)
+            {
+                Ok(mut f) => {
+                    f.write_all(output.as_bytes())?;
+                    return Ok(path);
+                }
+                Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
+                Err(e) => return Err(e),
+            }
+        }
+    })();
+    match saved {
+        Ok(path) => excerpt(&output, &path, tail, 0, output.len()),
+        Err(e) => {
+            eprintln!("overflow archive failed: {e}; retaining complete tool result");
+            output
+        }
+    }
+}
+
+fn date() -> String {
+    let days = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+        / 86400;
+    let z = days as i64 + 719468;
+    let era = z / 146097;
+    let doe = z - era * 146097;
+    let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
+    let y = yoe + era * 400;
+    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+    let mp = (5 * doy + 2) / 153;
+    let d = doy - (153 * mp + 2) / 5 + 1;
+    let m = mp + if mp < 10 { 3 } else { -9 };
+    format!("{:04}-{m:02}-{d:02}", y + i64::from(m <= 2))
+}
+
+/// Excerpt an existing archive in place. Byte/line ranges refer to that file,
+/// including when a caller requests an oversized line or the entire archive.
+pub fn existing(bytes: &[u8], path: &str, args: &serde_json::Value) -> String {
+    let text = String::from_utf8_lossy(bytes);
+    let (mut from, mut to) = if let Some(line) = args["line_offset"].as_u64() {
+        let from = text
+            .split_inclusive('\n')
+            .take(line as usize)
+            .map(str::len)
+            .sum::<usize>();
+        let len = text[from..]
+            .split_inclusive('\n')
+            .take(args["line_limit"].as_u64().unwrap_or(200) as usize)
+            .map(str::len)
+            .sum::<usize>();
+        (from, from + len)
+    } else {
+        let from = (args["offset"].as_u64().unwrap_or(0) as usize).min(text.len());
+        (
+            from,
+            args["max_bytes"].as_u64().map_or(text.len(), |n| {
+                from.saturating_add(n as usize).min(text.len())
+            }),
+        )
+    };
+    while !text.is_char_boundary(from) {
+        from += 1;
+    }
+    while to > from && !text.is_char_boundary(to) {
+        to -= 1;
+    }
+    to = to.max(from);
+    excerpt(&text, path, false, from, to)
+}
+
+fn excerpt(output: &str, path: &str, tail: bool, from: usize, to: usize) -> String {
+    let selected = &output[from..to];
+    let lines: Vec<&str> = selected.split_inclusive('\n').collect();
+    let budget = CONTEXT_LIMIT.0.saturating_sub(1024);
+    let mut start = 0;
+    let mut end = lines.len();
+    let mut bytes = 0;
+    if tail {
+        start = end;
+        while start > 0
+            && end - start < CONTEXT_LIMIT.1.saturating_sub(2)
+            && bytes + lines[start - 1].len() <= budget
+        {
+            start -= 1;
+            bytes += lines[start].len();
+        }
+    } else {
+        end = 0;
+        while end < lines.len()
+            && end < CONTEXT_LIMIT.1.saturating_sub(2)
+            && bytes + lines[end].len() <= budget
+        {
+            bytes += lines[end].len();
+            end += 1;
+        }
+    }
+    let mut shown = lines[start..end].concat();
+    // A single huge line: preserve valid UTF-8 and explicitly label a partial line.
+    let partial = start == end;
+    if partial && !lines.is_empty() {
+        if tail {
+            start = lines.len() - 1;
+            let line = lines[start];
+            let mut from = line.len().saturating_sub(budget);
+            while !line.is_char_boundary(from) {
+                from += 1;
+            }
+            shown.push_str(&line[from..]);
+        } else {
+            let mut to = budget.min(lines[0].len());
+            while !lines[0].is_char_boundary(to) {
+                to -= 1;
+            }
+            shown.push_str(&lines[0][..to]);
+        }
+    }
+    let mut byte_start = from + lines[..start].iter().map(|s| s.len()).sum::<usize>();
+    if partial && tail {
+        byte_start = to - shown.len();
+    }
+    let byte_end = byte_start + shown.len();
+    let line_start = output[..byte_start].bytes().filter(|b| *b == b'\n').count();
+    let line_end = line_start + shown.split_inclusive('\n').count();
+    let total_lines = output.split_inclusive('\n').count();
+    let partial = partial
+        || (from > 0 && !output[..from].ends_with('\n'))
+        || (to < output.len() && !output[..to].ends_with('\n'));
+    let read = if partial {
+        format!(
+            "Shown byte range [{byte_start},{byte_end}); continue with read_file path={path}, offset={byte_end}, max_bytes={budget} (or choose an earlier byte offset)"
+        )
+    } else {
+        format!(
+            "Read with read_file path={path}, line_offset={line_start}, line_limit={}; next line_offset={line_end}",
+            line_end - line_start
+        )
+    };
+    format!(
+        "{shown}\n[Showing lines {}-{} of {}{}; total {} bytes. Full output: {path}. {read}]",
+        line_start + 1,
+        line_end,
+        total_lines,
+        if partial { " (partial line)" } else { "" },
+        output.len()
+    )
+}
+
+/// Zero-based line offsets; byte-based reads remain unchanged when absent.
+pub fn read_lines(bytes: &[u8], offset: usize, limit: usize) -> String {
+    String::from_utf8_lossy(bytes)
+        .split_inclusive('\n')
+        .skip(offset)
+        .take(limit)
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn full_archive_and_context_guard() {
+        let original = "😀 line\n".repeat(5400);
+        for tail in [false, true] {
+            let shown = guard(original.clone(), "guard-test", "call", tail);
+            assert!(shown.len() <= CONTEXT_LIMIT.0);
+            assert!(shown.lines().count() <= CONTEXT_LIMIT.1);
+            assert!(shown.contains(if tail {
+                "Showing lines 3403-5400"
+            } else {
+                "Showing lines 1-1998"
+            }));
+            let path = shown
+                .split("Full output: ")
+                .nth(1)
+                .unwrap()
+                .split(". Read with")
+                .next()
+                .unwrap();
+            assert_eq!(std::fs::read_to_string(path).unwrap(), original);
+        }
+        assert_eq!(guard("small".into(), "test", "id", false), "small");
+    }
+    #[test]
+    fn archive_failure_does_not_discard_data() {
+        std::fs::create_dir_all(format!("/logs/agent/{}", date())).unwrap();
+        std::fs::write(format!("/logs/agent/{}/not-a-directory", date()), "block").unwrap();
+        let output = "no loss\n".repeat(6000);
+        assert_eq!(
+            guard(output.clone(), "not-a-directory", "test", false),
+            output
+        );
+    }
+    #[test]
+    fn huge_line_and_exact_boundaries() {
+        for tail in [false, true] {
+            let result = guard("😀".repeat(40000), "test", "utf8", tail);
+            assert!(result.len() <= CONTEXT_LIMIT.0);
+            assert!(result.contains("partial line"));
+        }
+        let exact = "a\n".repeat(CONTEXT_LIMIT.1);
+        assert_eq!(guard(exact.clone(), "test", "exact", false), exact);
+        assert_eq!(read_lines(b"one\ntwo\nthree", 1, 1), "two\n");
+        assert_eq!(read_lines(b"one", 3, 1), "");
+    }
+}

agent/src/tools.rs

@@ -4,21 +4,47 @@ use crate::llm::Tool;
 use serde_json::{Value, json};
 use std::io;
 
-/// Maximum UTF-8 bytes in a serialized tool result, including its truncation marker.
-pub const MAX_OUTPUT_BYTES: usize = 65_536;
+#[path = "overflow.rs"]
+pub mod overflow;
 
-/// Bound every tool result before it enters history or a provider request.
-pub fn limit_output(mut output: String) -> String {
-    const MARKER: &str = "\n[tool output truncated at 65536 bytes]";
-    if output.len() > MAX_OUTPUT_BYTES {
-        let mut end = MAX_OUTPUT_BYTES - MARKER.len();
-        while !output.is_char_boundary(end) {
-            end -= 1;
+/// Human-readable, lossless text representation of tool JSON. String values
+/// retain real newlines, with JSON-quoted field/index labels for structure.
+fn render(value: &Value) -> String {
+    fn visit(value: &Value, label: &str, out: &mut String) {
+        match value {
+            Value::Object(fields) => {
+                if fields.is_empty() {
+                    out.push_str(&format!("{label}: {{}}\n"));
+                }
+                for (key, value) in fields {
+                    visit(
+                        value,
+                        &format!("{label}[{}]", serde_json::to_string(key).unwrap()),
+                        out,
+                    );
+                }
+            }
+            Value::Array(items) => {
+                if items.is_empty() {
+                    out.push_str(&format!("{label}: []\n"));
+                }
+                for (i, value) in items.iter().enumerate() {
+                    visit(value, &format!("{label}[{i}]"), out);
+                }
+            }
+            Value::String(text) => {
+                out.push_str(&format!("{label} (string, {} bytes):\n", text.len()));
+                out.push_str(text);
+                out.push('\n');
+            }
+            _ => {
+                out.push_str(&format!("{label}: {value}\n"));
+            }
         }
-        output.truncate(end);
-        output.push_str(MARKER);
     }
-    output
+    let mut out = String::new();
+    visit(value, "$", &mut out);
+    out
 }
 
 /// Describe the five tools sent to either inference API.
@@ -26,7 +52,7 @@ pub fn definitions() -> Vec<Tool> {
     vec![
         Tool {
             name: "call",
-            description: "Call a plugin method through the core. Await its result. Set detach to let its process survive this caller. Output is capped at 65536 bytes with a truncation marker.",
+            description: "Call a plugin method through the core. Await its result. Set detach to let its process survive this caller. Large results show an excerpt with a full-output /logs reference.",
             parameters: schema(
                 json!({
                     "method": {"type": "string", "description": "Qualified plugin method"},
@@ -38,19 +64,21 @@ pub fn definitions() -> Vec<Tool> {
         },
         Tool {
             name: "read_file",
-            description: "Read a text file in the world, including `/proc`. `offset` and `max_bytes` are optional. Output is capped at 65536 bytes with a truncation marker.",
+            description: "Read a text file in the world, including `/proc`. `offset`/`max_bytes` select bytes; `line_offset`/`line_limit` select zero-based lines (default 200). Large results show an excerpt with a full-output /logs reference.",
             parameters: schema(
                 json!({
                     "path": {"type": "string"},
                     "offset": {"type": "integer", "description": "Byte offset (default 0)"},
-                    "max_bytes": {"type": "integer", "description": "Optional maximum number of bytes to read"}
+                    "max_bytes": {"type": "integer", "description": "Optional maximum number of bytes to read"},
+                    "line_offset": {"type": "integer", "description": "Zero-based starting line; overrides bytes"},
+                    "line_limit": {"type": "integer", "description": "Number of lines; default 200"}
                 }),
                 &["path"],
             ),
         },
         Tool {
             name: "write_file",
-            description: "Write a text file in the world. Parent folders are created. Output is capped at 65536 bytes with a truncation marker.",
+            description: "Write a text file in the world. Parent folders are created. Large results show an excerpt with a full-output /logs reference.",
             parameters: schema(
                 json!({"path": {"type": "string"}, "content": {"type": "string"}}),
                 &["path", "content"],
@@ -58,12 +86,12 @@ pub fn definitions() -> Vec<Tool> {
         },
         Tool {
             name: "list_dir",
-            description: "List entries in a world directory. Returns each name, type, and size. Output is capped at 65536 bytes with a truncation marker.",
+            description: "List entries in a world directory. Returns each name, type, and size. Large results show an excerpt with a full-output /logs reference.",
             parameters: schema(json!({"path": {"type": "string"}}), &["path"]),
         },
         Tool {
             name: "copy_dir",
-            description: "Copy a directory tree, for example a plugin source to a draft folder. Skips `target/` folders. Output is capped at 65536 bytes with a truncation marker.",
+            description: "Copy a directory tree, for example a plugin source to a draft folder. Skips `target/` folders. Large results show an excerpt with a full-output /logs reference.",
             parameters: schema(
                 json!({"from": {"type": "string"}, "to": {"type": "string"}}),
                 &["from", "to"],
@@ -103,6 +131,16 @@ pub async fn run(name: &str, args: &Value) -> sausage_plugin::Result<Value> {
 fn read_file(args: &Value) -> io::Result<Value> {
     let path = args["path"].as_str().unwrap_or_default();
     let bytes = std::fs::read(path)?;
+    if path.starts_with("/logs/") {
+        return Ok(
+            json!({"path":path,"size":bytes.len(),"content":overflow::existing(&bytes, path, args)}),
+        );
+    }
+    if let Some(line) = args["line_offset"].as_u64() {
+        return Ok(
+            json!({"path":path,"size":bytes.len(),"line_offset":line,"content":overflow::read_lines(&bytes, line as usize, args["line_limit"].as_u64().unwrap_or(200) as usize)}),
+        );
+    }
     let offset = (args["offset"].as_u64().unwrap_or(0) as usize).min(bytes.len());
     let end = args["max_bytes"].as_u64().map_or(bytes.len(), |max| {
         offset.saturating_add(max as usize).min(bytes.len())
@@ -185,11 +223,35 @@ fn copy_tree(from: &str, to: &str) -> io::Result<u64> {
 #[cfg(test)]
 mod tests {
     #[test]
-    fn tool_output_limit_preserves_utf8_and_marks_truncation() {
-        let limited = super::limit_output("😀".repeat(40_000));
-        assert!(limited.len() <= super::MAX_OUTPUT_BYTES);
-        assert!(limited.ends_with("[tool output truncated at 65536 bytes]"));
-        assert_eq!(super::limit_output("small".into()), "small");
+    fn overflow_json_can_be_followed_by_line_without_rearchiving() {
+        let value = serde_json::json!({"stdout":"hello 😀\n".repeat(4000),"exit":1,"error":{"text":"failure\n".repeat(4000)}});
+        let expected = super::render(&value);
+        let shown = super::overflow::guard(value.to_string(), "json-test", "call", true);
+        assert!(shown.len() <= 50 * 1024 && shown.lines().count() <= 2000);
+        let path = shown
+            .split("Full output: ")
+            .nth(1)
+            .unwrap()
+            .split(". Read with")
+            .next()
+            .unwrap();
+        assert_eq!(std::fs::read_to_string(path).unwrap(), expected);
+        let parent = std::path::Path::new(path).parent().unwrap();
+        let count = std::fs::read_dir(parent).unwrap().count();
+        let args = serde_json::json!({"path":path,"line_offset":30,"line_limit":5});
+        let read = super::read_file(&args).unwrap();
+        let read = read["content"].as_str().unwrap();
+        assert!(read.contains("Showing lines 31-35"));
+        assert!(read.contains(path));
+        assert_eq!(std::fs::read_dir(parent).unwrap().count(), count);
+        assert!(
+            super::overflow::existing(
+                expected.as_bytes(),
+                path,
+                &serde_json::json!({"line_offset":u64::MAX})
+            )
+            .contains("Full output:")
+        );
     }
     #[test]
     fn copy_rejects_a_destination_inside_the_source() {