From e61358e7aadae347b5bc4df80bdbb7f7851430b1 Mon Sep 17 00:00:00 2001 From: Surma Date: Thu, 12 Nov 2026 19:47:31 +0000 Subject: [PATCH] Add a packaged real-model end-to-end check --- AGENTS.md | 15 ++++ host/tests/e2e-real.nu | 191 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 host/tests/e2e-real.nu diff --git a/AGENTS.md b/AGENTS.md index 7525d14..043ad1d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,21 @@ Coreutils, diffutils, procps, and brush use relative symlinks to one binary each Sandbox-incompatible tests print an explicit skip reason instead of a flake-level filter. The check sets `SAUSAGE_NIX_CHECK=1` for tests that need network access or unvendored fixture dependencies. +## Real end-to-end check + +Before you report packaging, config, or model changes as complete, run this check outside the Nix sandbox. +Use the required build slot wrapper and approved real provider keys. + +```sh +nu --no-config-file host/tests/e2e-real.nu /path/to/config.yaml +``` + +The script builds the package and tests a fresh world, real models, a shell tool, and browser chat. +It copies only the supplied config's providers and never changes the source world. +It uses a free loopback port, not port 8080. +A failure retains a private temporary folder with provider keys. Treat that folder as sensitive. +The result table reports times in seconds. + ## Tests Host integration tests share one executable: diff --git a/host/tests/e2e-real.nu b/host/tests/e2e-real.nu new file mode 100644 index 0000000..38dd7a0 --- /dev/null +++ b/host/tests/e2e-real.nu @@ -0,0 +1,191 @@ +#!/usr/bin/env nu +# Check the installed package, its web UI, real roster models, and a real tool call. + +def check [name: string, action: closure] { + let start = date now + try { + let value = (do $action) + {step: $name, result: PASS, time: ((date now) - $start), detail: '', value: $value} + } catch {|error| + {step: $name, result: FAIL, time: ((date now) - $start), detail: $error.msg, value: null} + } +} + +def require [condition: bool, message: string] { + if not $condition { error make {msg: $message} } +} + +def api [base: string, route: string, body?: record] { + if $body == null { + http get --max-time 10sec $'($base)/api/harness/($route)' + } else { + http post --max-time 10sec --content-type application/json $'($base)/api/harness/($route)' ($body | to json --raw) + } +} + +def entries [base: string, id: int, after: int] { + api $base $'conversations/($id)/entries?after=($after)' | get entries +} + +def answer [base: string, id: int, after: int, directory: path] { + let deadline = (date now) + 180sec + loop { + let found = (entries $base $id $after | where conversationId == $id) + $found | to json | save --force ($directory | path join $'conversation-($id)-after-($after).json') + let messages = ($found | where kind == 'pi.assistant' | get model | flatten) + let errors = ($messages | where {|message| ($message.stopReason? | default '') == 'error' }) + if not ($errors | is-empty) { + error make {msg: $'model_error: ($errors | first | get errorMessage)'} + } + let conversation = (api $base conversations | get conversations | where id == $id | first) + let answered = ($messages | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) }) + if not $conversation.running and $answered { + return $found + } + require ((date now) < $deadline) $'No answer from conversation ($id) within 180 seconds' + } +} + +def submit [base: string, id: int, content: string, directory: path] { + let previous = (entries $base $id 0) + let after = ($previous | get id | append 0 | math max) + api $base $'conversations/($id)/submit' {content: $content, requestId: (random uuid)} | ignore + answer $base $id $after $directory +} + +# The caller supplies one config file. Its provider values never enter the report. +def main [config: path] { + hide-env --ignore-errors SAUSAGE_USERLAND SAUSAGE_PRECOMPILED SAUSAGE_NIX_CHECK + mut rows = [] + let directory = (^mktemp -d -t sausage-e2e-real.XXXXXX | str trim) + let build = (check 'nix build' { + let result = (^nix build --max-jobs auto --cores 0 | complete) + require ($result.exit_code == 0) $'nix build failed: ($result.stderr)' + }) + $rows = ($rows | append $build) + if $build.result == FAIL { + print ($rows | reject value) + print $'Failure evidence: ($directory)' + exit 1 + } + let binary = ($env.PWD | path join result bin sausage) + let world = ($directory | path join world) + let root = ($world | path join world) + let session = $'sausage-e2e-(random uuid)' + let init = (check 'fresh init' { + let result = (^$binary init $world | complete) + require ($result.exit_code == 0) 'Packaged init failed' + }) + $rows = ($rows | append $init) + let setup = (check 'providers and loopback port' { + let providers = (try { open $config | get providers } catch { error make {msg: 'Cannot read the supplied providers section'} }) + let destination = ($root | path join etc sausage config.yaml) + open $destination | upsert providers $providers | to yaml | save --force $destination + let listen = (port 20000 60000) + let service = ($root | path join etc systemd system web.service) + open --raw $service | str replace -r '(?m)^ExecStart=.*$' $'ExecStart=/bin/web.wasm --listen 127.0.0.1:($listen)' | save --force $service + $'http://127.0.0.1:($listen)' + }) + $rows = ($rows | append $setup) + if $init.result == FAIL or $setup.result == FAIL { + print ($rows | reject value) + print $'Failure evidence: ($directory)' + exit 1 + } + let base = $setup.value + let job = (job spawn { + '' | ^bash -c 'echo $$ > "$1"; exec "$2" run "$3"' e2e ($directory | path join server.pid) $binary $world out> ($directory | path join server.stdout) err> ($directory | path join server.stderr) + }) + let ready = (check 'server readiness' { + let deadline = (date now) + 60sec + loop { + let response = (try { http get --allow-errors --full --max-time 1sec $'($base)/api/whoami' } catch { null }) + if $response != null and $response.status == 200 { break } + require ((date now) < $deadline) 'The web server did not answer within 60 seconds' + } + }) + $rows = ($rows | append $ready) + if $ready.result == PASS { + $rows = ($rows | append (check 'GET / chat page' { + let response = (http get --allow-errors --full --raw --max-time 10sec $base) + require ($response.status == 200) $'GET / returned ($response.status), expected 200' + require ($response.body | str contains 'id="compose"') 'GET / did not contain the chat page' + })) + $rows = ($rows | append (check 'GET /api/whoami' { + let response = (http get --allow-errors --full --max-time 10sec $'($base)/api/whoami') + require ($response.status == 200) $'whoami returned ($response.status)' + })) + let main = (check 'main agent answer' { + let main = (api $base conversations | get main_agent) + let roster = (glob ($root | path join etc sausage roster '*.md') | each {|file| + let front = (open --raw $file | split row '---' | get 1 | from yaml) + {name: ($file | path basename | str replace '.md' ''), model: $front.model} + }) + require (not ($roster | any {|agent| $agent.model | str starts-with 'openrouter/' })) 'OpenRouter calls are not approved' + let current = ($roster | where name == $main.name | first) + let others = ($roster | where model != $current.model | uniq-by model) + let request = if ($others | is-empty) { 'Reply with E2E_OK only. Do not use tools.' } else { + let names = ($others | get name | str join ', ') + $'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Do not edit files or run checks. Report completion briefly.' + } + let result = (submit $base $main.conversationId $request $directory) + {id: $main.conversationId, others: $others, entries: $result} + }) + $rows = ($rows | append $main) + if $main.result == PASS { + for agent in $main.value.others { + $rows = ($rows | append (check $'model ($agent.model) via @($agent.name)' { + let deadline = (date now) + 180sec + mut id = -1 + loop { + let conversations = (api $base conversations | get conversations) + let matches = ($conversations | where {|c| + ($c.agent?.model? | default '') == $agent.model + }) + if not ($matches | is-empty) { $id = ($matches | first | get id); break } + require ((date now) < $deadline) $'No conversation for ($agent.model)' + } + submit $base $id 'Reply E2E_OK only. Do not use tools.' $directory | ignore + })) + } + $rows = ($rows | append (check 'shell tool nonce' { + let nonce = $'e2e-tool-(random uuid)' + let result = (submit $base $main.value.id $'Use bash to run exactly: printf "($nonce)\\n". Do not merely repeat the nonce. Then answer done.' $directory) + require ($result | any {|entry| $entry.kind == 'pi.tool-result' and not ($entry | to json --raw | str contains '"isError":true') and ($entry | to json --raw | str contains $nonce) }) 'No tool result contained the random nonce' + })) + } else { + let main_id = (try { api $base conversations | get main_agent.conversationId } catch { null }) + if $main_id != null { try { api $base $'conversations/($main_id)/stop' {} | ignore } } + $rows = ($rows | append {step: 'other roster models and shell nonce', result: FAIL, time: 0sec, detail: 'Blocked by the main model failure', value: null}) + } + let page_ok = (($rows | where step == 'GET / chat page' | first | get result) == PASS) + $rows = ($rows | append (check 'browser message and reply' { + require $page_ok 'The chat page is unavailable' + let nonce = $'e2e-browser-(random uuid)' + let condition = (r#'Array.from(document.querySelectorAll(".entry:not(.user) .markdown")).some(e=>e.textContent.includes("TOKEN"))'# | str replace TOKEN $nonce) + for args in [[open $base] [wait '#input'] [fill '#input' $'Reply only ($nonce), without tools.'] [click '#compose button:first-of-type'] [wait --fn $condition]] { + let result = (with-env {AGENT_BROWSER_DEFAULT_TIMEOUT: '180000'} { ^agent-browser --session $session ...$args | complete }) + require ($result.exit_code == 0) $'Browser check failed: ($result.stderr)' + } + })) + } + $rows = ($rows | append (check 'stop server' { + let pid = (open --raw ($directory | path join server.pid) | str trim | into int) + ^kill -INT $pid + let deadline = (date now) + 15sec + loop { + if (ps | where pid == $pid | is-empty) { break } + require ((date now) < $deadline) 'The server did not stop within 15 seconds' + } + })) + try { job kill $job } + try { ^agent-browser --session $session close | complete | ignore } + let table = ($rows | reject value | update time {|row| ($row.time / 1sec) | math round --precision 3 }) + $table | to json | save --force ($directory | path join results.json) + print ($table | table --expand --width 180) + if ($rows | any {|row| $row.result == FAIL }) { + print $'Failure evidence: ($directory)' + exit 1 + } + rm --recursive $directory +}