Add a packaged real-model end-to-end check
This commit is contained in:
@@ -63,6 +63,21 @@ Coreutils, diffutils, procps, and brush use relative symlinks to one binary each
|
||||
Sandbox-incompatible tests print an explicit skip reason instead of a flake-level filter.
|
||||
The check sets `SAUSAGE_NIX_CHECK=1` for tests that need network access or unvendored fixture dependencies.
|
||||
|
||||
## Real end-to-end check
|
||||
|
||||
Before you report packaging, config, or model changes as complete, run this check outside the Nix sandbox.
|
||||
Use the required build slot wrapper and approved real provider keys.
|
||||
|
||||
```sh
|
||||
nu --no-config-file host/tests/e2e-real.nu /path/to/config.yaml
|
||||
```
|
||||
|
||||
The script builds the package and tests a fresh world, real models, a shell tool, and browser chat.
|
||||
It copies only the supplied config's providers and never changes the source world.
|
||||
It uses a free loopback port, not port 8080.
|
||||
A failure retains a private temporary folder with provider keys. Treat that folder as sensitive.
|
||||
The result table reports times in seconds.
|
||||
|
||||
## Tests
|
||||
|
||||
Host integration tests share one executable:
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env nu
|
||||
# Check the installed package, its web UI, real roster models, and a real tool call.
|
||||
|
||||
def check [name: string, action: closure] {
|
||||
let start = date now
|
||||
try {
|
||||
let value = (do $action)
|
||||
{step: $name, result: PASS, time: ((date now) - $start), detail: '', value: $value}
|
||||
} catch {|error|
|
||||
{step: $name, result: FAIL, time: ((date now) - $start), detail: $error.msg, value: null}
|
||||
}
|
||||
}
|
||||
|
||||
def require [condition: bool, message: string] {
|
||||
if not $condition { error make {msg: $message} }
|
||||
}
|
||||
|
||||
def api [base: string, route: string, body?: record] {
|
||||
if $body == null {
|
||||
http get --max-time 10sec $'($base)/api/harness/($route)'
|
||||
} else {
|
||||
http post --max-time 10sec --content-type application/json $'($base)/api/harness/($route)' ($body | to json --raw)
|
||||
}
|
||||
}
|
||||
|
||||
def entries [base: string, id: int, after: int] {
|
||||
api $base $'conversations/($id)/entries?after=($after)' | get entries
|
||||
}
|
||||
|
||||
def answer [base: string, id: int, after: int, directory: path] {
|
||||
let deadline = (date now) + 180sec
|
||||
loop {
|
||||
let found = (entries $base $id $after | where conversationId == $id)
|
||||
$found | to json | save --force ($directory | path join $'conversation-($id)-after-($after).json')
|
||||
let messages = ($found | where kind == 'pi.assistant' | get model | flatten)
|
||||
let errors = ($messages | where {|message| ($message.stopReason? | default '') == 'error' })
|
||||
if not ($errors | is-empty) {
|
||||
error make {msg: $'model_error: ($errors | first | get errorMessage)'}
|
||||
}
|
||||
let conversation = (api $base conversations | get conversations | where id == $id | first)
|
||||
let answered = ($messages | any {|message| ($message.stopReason? | default '') == stop and ($message.content | any {|part| $part.type == text and ($part.text? | default '' | str trim | is-not-empty) }) })
|
||||
if not $conversation.running and $answered {
|
||||
return $found
|
||||
}
|
||||
require ((date now) < $deadline) $'No answer from conversation ($id) within 180 seconds'
|
||||
}
|
||||
}
|
||||
|
||||
def submit [base: string, id: int, content: string, directory: path] {
|
||||
let previous = (entries $base $id 0)
|
||||
let after = ($previous | get id | append 0 | math max)
|
||||
api $base $'conversations/($id)/submit' {content: $content, requestId: (random uuid)} | ignore
|
||||
answer $base $id $after $directory
|
||||
}
|
||||
|
||||
# The caller supplies one config file. Its provider values never enter the report.
|
||||
def main [config: path] {
|
||||
hide-env --ignore-errors SAUSAGE_USERLAND SAUSAGE_PRECOMPILED SAUSAGE_NIX_CHECK
|
||||
mut rows = []
|
||||
let directory = (^mktemp -d -t sausage-e2e-real.XXXXXX | str trim)
|
||||
let build = (check 'nix build' {
|
||||
let result = (^nix build --max-jobs auto --cores 0 | complete)
|
||||
require ($result.exit_code == 0) $'nix build failed: ($result.stderr)'
|
||||
})
|
||||
$rows = ($rows | append $build)
|
||||
if $build.result == FAIL {
|
||||
print ($rows | reject value)
|
||||
print $'Failure evidence: ($directory)'
|
||||
exit 1
|
||||
}
|
||||
let binary = ($env.PWD | path join result bin sausage)
|
||||
let world = ($directory | path join world)
|
||||
let root = ($world | path join world)
|
||||
let session = $'sausage-e2e-(random uuid)'
|
||||
let init = (check 'fresh init' {
|
||||
let result = (^$binary init $world | complete)
|
||||
require ($result.exit_code == 0) 'Packaged init failed'
|
||||
})
|
||||
$rows = ($rows | append $init)
|
||||
let setup = (check 'providers and loopback port' {
|
||||
let providers = (try { open $config | get providers } catch { error make {msg: 'Cannot read the supplied providers section'} })
|
||||
let destination = ($root | path join etc sausage config.yaml)
|
||||
open $destination | upsert providers $providers | to yaml | save --force $destination
|
||||
let listen = (port 20000 60000)
|
||||
let service = ($root | path join etc systemd system web.service)
|
||||
open --raw $service | str replace -r '(?m)^ExecStart=.*$' $'ExecStart=/bin/web.wasm --listen 127.0.0.1:($listen)' | save --force $service
|
||||
$'http://127.0.0.1:($listen)'
|
||||
})
|
||||
$rows = ($rows | append $setup)
|
||||
if $init.result == FAIL or $setup.result == FAIL {
|
||||
print ($rows | reject value)
|
||||
print $'Failure evidence: ($directory)'
|
||||
exit 1
|
||||
}
|
||||
let base = $setup.value
|
||||
let job = (job spawn {
|
||||
'' | ^bash -c 'echo $$ > "$1"; exec "$2" run "$3"' e2e ($directory | path join server.pid) $binary $world out> ($directory | path join server.stdout) err> ($directory | path join server.stderr)
|
||||
})
|
||||
let ready = (check 'server readiness' {
|
||||
let deadline = (date now) + 60sec
|
||||
loop {
|
||||
let response = (try { http get --allow-errors --full --max-time 1sec $'($base)/api/whoami' } catch { null })
|
||||
if $response != null and $response.status == 200 { break }
|
||||
require ((date now) < $deadline) 'The web server did not answer within 60 seconds'
|
||||
}
|
||||
})
|
||||
$rows = ($rows | append $ready)
|
||||
if $ready.result == PASS {
|
||||
$rows = ($rows | append (check 'GET / chat page' {
|
||||
let response = (http get --allow-errors --full --raw --max-time 10sec $base)
|
||||
require ($response.status == 200) $'GET / returned ($response.status), expected 200'
|
||||
require ($response.body | str contains 'id="compose"') 'GET / did not contain the chat page'
|
||||
}))
|
||||
$rows = ($rows | append (check 'GET /api/whoami' {
|
||||
let response = (http get --allow-errors --full --max-time 10sec $'($base)/api/whoami')
|
||||
require ($response.status == 200) $'whoami returned ($response.status)'
|
||||
}))
|
||||
let main = (check 'main agent answer' {
|
||||
let main = (api $base conversations | get main_agent)
|
||||
let roster = (glob ($root | path join etc sausage roster '*.md') | each {|file|
|
||||
let front = (open --raw $file | split row '---' | get 1 | from yaml)
|
||||
{name: ($file | path basename | str replace '.md' ''), model: $front.model}
|
||||
})
|
||||
require (not ($roster | any {|agent| $agent.model | str starts-with 'openrouter/' })) 'OpenRouter calls are not approved'
|
||||
let current = ($roster | where name == $main.name | first)
|
||||
let others = ($roster | where model != $current.model | uniq-by model)
|
||||
let request = if ($others | is-empty) { 'Reply with E2E_OK only. Do not use tools.' } else {
|
||||
let names = ($others | get name | str join ', ')
|
||||
$'This is an authorized end-to-end smoke check. Use branch with repo /src/sausage and isolated false. Ask the coordinator to use its subagent tool to start exactly one agent for each of these roster names: ($names). Each must reply E2E_OK only without tools. Do not edit files or run checks. Report completion briefly.'
|
||||
}
|
||||
let result = (submit $base $main.conversationId $request $directory)
|
||||
{id: $main.conversationId, others: $others, entries: $result}
|
||||
})
|
||||
$rows = ($rows | append $main)
|
||||
if $main.result == PASS {
|
||||
for agent in $main.value.others {
|
||||
$rows = ($rows | append (check $'model ($agent.model) via @($agent.name)' {
|
||||
let deadline = (date now) + 180sec
|
||||
mut id = -1
|
||||
loop {
|
||||
let conversations = (api $base conversations | get conversations)
|
||||
let matches = ($conversations | where {|c|
|
||||
($c.agent?.model? | default '') == $agent.model
|
||||
})
|
||||
if not ($matches | is-empty) { $id = ($matches | first | get id); break }
|
||||
require ((date now) < $deadline) $'No conversation for ($agent.model)'
|
||||
}
|
||||
submit $base $id 'Reply E2E_OK only. Do not use tools.' $directory | ignore
|
||||
}))
|
||||
}
|
||||
$rows = ($rows | append (check 'shell tool nonce' {
|
||||
let nonce = $'e2e-tool-(random uuid)'
|
||||
let result = (submit $base $main.value.id $'Use bash to run exactly: printf "($nonce)\\n". Do not merely repeat the nonce. Then answer done.' $directory)
|
||||
require ($result | any {|entry| $entry.kind == 'pi.tool-result' and not ($entry | to json --raw | str contains '"isError":true') and ($entry | to json --raw | str contains $nonce) }) 'No tool result contained the random nonce'
|
||||
}))
|
||||
} else {
|
||||
let main_id = (try { api $base conversations | get main_agent.conversationId } catch { null })
|
||||
if $main_id != null { try { api $base $'conversations/($main_id)/stop' {} | ignore } }
|
||||
$rows = ($rows | append {step: 'other roster models and shell nonce', result: FAIL, time: 0sec, detail: 'Blocked by the main model failure', value: null})
|
||||
}
|
||||
let page_ok = (($rows | where step == 'GET / chat page' | first | get result) == PASS)
|
||||
$rows = ($rows | append (check 'browser message and reply' {
|
||||
require $page_ok 'The chat page is unavailable'
|
||||
let nonce = $'e2e-browser-(random uuid)'
|
||||
let condition = (r#'Array.from(document.querySelectorAll(".entry:not(.user) .markdown")).some(e=>e.textContent.includes("TOKEN"))'# | str replace TOKEN $nonce)
|
||||
for args in [[open $base] [wait '#input'] [fill '#input' $'Reply only ($nonce), without tools.'] [click '#compose button:first-of-type'] [wait --fn $condition]] {
|
||||
let result = (with-env {AGENT_BROWSER_DEFAULT_TIMEOUT: '180000'} { ^agent-browser --session $session ...$args | complete })
|
||||
require ($result.exit_code == 0) $'Browser check failed: ($result.stderr)'
|
||||
}
|
||||
}))
|
||||
}
|
||||
$rows = ($rows | append (check 'stop server' {
|
||||
let pid = (open --raw ($directory | path join server.pid) | str trim | into int)
|
||||
^kill -INT $pid
|
||||
let deadline = (date now) + 15sec
|
||||
loop {
|
||||
if (ps | where pid == $pid | is-empty) { break }
|
||||
require ((date now) < $deadline) 'The server did not stop within 15 seconds'
|
||||
}
|
||||
}))
|
||||
try { job kill $job }
|
||||
try { ^agent-browser --session $session close | complete | ignore }
|
||||
let table = ($rows | reject value | update time {|row| ($row.time / 1sec) | math round --precision 3 })
|
||||
$table | to json | save --force ($directory | path join results.json)
|
||||
print ($table | table --expand --width 180)
|
||||
if ($rows | any {|row| $row.result == FAIL }) {
|
||||
print $'Failure evidence: ($directory)'
|
||||
exit 1
|
||||
}
|
||||
rm --recursive $directory
|
||||
}
|
||||
Reference in New Issue
Block a user