Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
56d02befc8 | ||
|
|
0be152dda9 | ||
|
|
83bbbb33a8 |
@@ -1,11 +0,0 @@
|
|||||||
# Runs on a self-hosted Nix runner (label `nixos`, host executor).
|
|
||||||
name: ci
|
|
||||||
on: [push, pull_request]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
runs-on: nixos
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
- name: nix flake check
|
|
||||||
run: nix flake check -L --show-trace
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
# Working on Sausage
|
|
||||||
|
|
||||||
Sausage is meant to be read and changed by agents, including its own agent.
|
|
||||||
Keep it that way.
|
|
||||||
|
|
||||||
## Priorities
|
|
||||||
|
|
||||||
1. **Correct.** Every behavior in `PROTOCOL.md` has a test in `cli/tests/`.
|
|
||||||
2. **Simple and short.** Prefer the obvious code. No abstraction for one
|
|
||||||
caller. No configuration that nobody asked for.
|
|
||||||
3. **Readable.** A new reader must understand a file without the others.
|
|
||||||
Each module starts with a doc comment that says what it owns.
|
|
||||||
4. **Efficient** on the paths that run per message.
|
|
||||||
5. **Raw and minimal.** Sausage has no permissions, sandboxing beyond WASI,
|
|
||||||
limits, or authentication. Do not add them to the core. Add nothing to
|
|
||||||
the host that an actor (or the agent, by changing itself) could do.
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
- `cli/`: the host (`sausage` binary and library).
|
|
||||||
- `lib.rs`: `Host`, which starts and stops a world.
|
|
||||||
- `broker.rs`: routing, mailboxes, the version store, broker requests.
|
|
||||||
- `supervisor.rs`: runs one WASM actor (restarts, swaps, rollbacks).
|
|
||||||
- `runtime.rs`: Wasmtime instances and what they get (mounts, network).
|
|
||||||
- `candidate.rs`: tests a new version. `cargo_actor.rs`: builds Rust.
|
|
||||||
- `world.rs`: the on-disk layout. `console.rs`: the terminal.
|
|
||||||
- `sdk/`: `sausage-actor`, the SDK for Rust actors. Never published: the
|
|
||||||
`cargo` actor points `sausage-actor = "0.1"` at its local copy.
|
|
||||||
- `agent/`: the agent actor. `sausage init` embeds and builds it.
|
|
||||||
- `examples/`: small actors. They are not workspace members.
|
|
||||||
- `PROTOCOL.md`: the contract between the host and the actors. Change it in
|
|
||||||
the same commit as the behavior. `DECISIONS.md` says why; when the two
|
|
||||||
disagree, `PROTOCOL.md` and the code win.
|
|
||||||
|
|
||||||
## Checks
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nix flake check -L # all tests below; CI runs this on every push
|
|
||||||
nix develop -c cargo test -p sausage -p sausage-actor -p agent
|
|
||||||
nix develop -c cargo build --release -p agent --target wasm32-wasip2
|
|
||||||
nix develop -c cargo clippy --workspace
|
|
||||||
```
|
|
||||||
|
|
||||||
`nix run -- init` and `nix run -- run` must keep working without other
|
|
||||||
setup.
|
|
||||||
Generated
+1
-125
@@ -15,7 +15,7 @@ dependencies = [
|
|||||||
name = "agent"
|
name = "agent"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"sausage-actor",
|
"async-channel",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"wstd",
|
"wstd",
|
||||||
@@ -678,15 +678,6 @@ dependencies = [
|
|||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fsevent-sys"
|
|
||||||
version = "4.1.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures"
|
name = "futures"
|
||||||
version = "0.3.34"
|
version = "0.3.34"
|
||||||
@@ -1075,26 +1066,6 @@ dependencies = [
|
|||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "inotify"
|
|
||||||
version = "0.11.5"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"inotify-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "inotify-sys"
|
|
||||||
version = "0.1.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "instant"
|
name = "instant"
|
||||||
version = "0.1.13"
|
version = "0.1.13"
|
||||||
@@ -1194,26 +1165,6 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "kqueue"
|
|
||||||
version = "1.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea"
|
|
||||||
dependencies = [
|
|
||||||
"kqueue-sys",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "kqueue-sys"
|
|
||||||
version = "1.1.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"libc",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leb128"
|
name = "leb128"
|
||||||
version = "0.2.7"
|
version = "0.2.7"
|
||||||
@@ -1308,7 +1259,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"log",
|
|
||||||
"wasi",
|
"wasi",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
@@ -1319,33 +1269,6 @@ version = "1.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "notify"
|
|
||||||
version = "8.2.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"fsevent-sys",
|
|
||||||
"inotify",
|
|
||||||
"kqueue",
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"mio",
|
|
||||||
"notify-types",
|
|
||||||
"walkdir",
|
|
||||||
"windows-sys 0.60.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "notify-types"
|
|
||||||
version = "2.1.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "object"
|
name = "object"
|
||||||
version = "0.40.0"
|
version = "0.40.0"
|
||||||
@@ -1657,15 +1580,6 @@ version = "1.0.23"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "same-file"
|
|
||||||
version = "1.0.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
|
||||||
dependencies = [
|
|
||||||
"winapi-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sausage"
|
name = "sausage"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -1676,26 +1590,14 @@ dependencies = [
|
|||||||
"http",
|
"http",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"include_dir",
|
"include_dir",
|
||||||
"notify",
|
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
"toml_edit",
|
|
||||||
"wasmtime",
|
"wasmtime",
|
||||||
"wasmtime-wasi",
|
"wasmtime-wasi",
|
||||||
"wasmtime-wasi-http",
|
"wasmtime-wasi-http",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sausage-actor"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"async-channel",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"wstd",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "scopeguard"
|
name = "scopeguard"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
@@ -2027,19 +1929,6 @@ dependencies = [
|
|||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toml_edit"
|
|
||||||
version = "0.25.17+spec-1.1.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
|
|
||||||
dependencies = [
|
|
||||||
"indexmap",
|
|
||||||
"toml_datetime",
|
|
||||||
"toml_parser",
|
|
||||||
"toml_writer",
|
|
||||||
"winnow",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_parser"
|
name = "toml_parser"
|
||||||
version = "1.1.5+spec-1.1.0"
|
version = "1.1.5+spec-1.1.0"
|
||||||
@@ -2162,16 +2051,6 @@ version = "1.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
|
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "walkdir"
|
|
||||||
version = "2.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
|
||||||
dependencies = [
|
|
||||||
"same-file",
|
|
||||||
"winapi-util",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "want"
|
name = "want"
|
||||||
version = "0.3.2"
|
version = "0.3.2"
|
||||||
@@ -2912,9 +2791,6 @@ name = "winnow"
|
|||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
|
checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
|
||||||
dependencies = [
|
|
||||||
"memchr",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winx"
|
name = "winx"
|
||||||
|
|||||||
+1
-6
@@ -1,10 +1,5 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
resolver = "3"
|
resolver = "3"
|
||||||
members = ["cli", "agent", "sdk"]
|
members = ["cli", "agent"]
|
||||||
default-members = ["cli"]
|
default-members = ["cli"]
|
||||||
exclude = ["examples"]
|
exclude = ["examples"]
|
||||||
|
|
||||||
# Wasmtime compiles components with Cranelift. Optimize dependencies in dev
|
|
||||||
# builds, so that tests and debug runs start actors quickly.
|
|
||||||
[profile.dev.package."*"]
|
|
||||||
opt-level = 2
|
|
||||||
|
|||||||
+145
-46
@@ -1,55 +1,154 @@
|
|||||||
# Decisions
|
# Decisions
|
||||||
|
|
||||||
Why Sausage is the way it is. `PROTOCOL.md` and the code own the details;
|
This file lists the design decisions of the spike. **(you)** marks a decision
|
||||||
when this file disagrees with them, they win.
|
that Surma made. **(spike)** marks a decision that Scout made alone during the
|
||||||
|
night and that needs review.
|
||||||
|
|
||||||
## The shape
|
## Architecture
|
||||||
|
|
||||||
- **Actors are WASI commands, not components with interfaces.** stdin,
|
- **(you)** Actors are WASI commands (`wasm32-wasip2`). They use stdin, stdout,
|
||||||
stdout, files, HTTP, and sockets are enough, and every language can do
|
files, and HTTP. No WIT interfaces, no component composition.
|
||||||
them.
|
- **(you)** Actors exchange newline-delimited JSON through the host, which is
|
||||||
- **Messages are JSON lines through a broker, fire and forget.** Like
|
the broker. No built-in request/response. A reply is a message with
|
||||||
Erlang: no acknowledgments, no request/response. A reply is a message with
|
`reply_to`. Delivery is at most once.
|
||||||
`reply_to`. Sausage is not a reliable message bus.
|
- **(you)** The broker accepts only `.wasm` files. It knows no source language.
|
||||||
- **The broker accepts only `.wasm` files.** It knows no source language.
|
- **(you)** `cargo` is a built-in Rust compiler actor. **(spike)** It is native
|
||||||
`cargo` is one compiler actor; others can add more languages.
|
(host code), because cargo cannot run inside WASM.
|
||||||
- **The world folder is the source of truth.** Everything about an actor is
|
- **(you)** The agent is a normal WASM actor with the fixed ID `agent`. It can
|
||||||
in `actor/<id>/`. Copy a folder into another world, and it runs there.
|
read and upgrade its own source.
|
||||||
- **The agent is a normal WASM actor.** It can read and upgrade its own
|
- **(spike)** Three native actors: `broker`, `console`, `cargo`.
|
||||||
source, and change how it does inference.
|
- **(you)** One Cargo workspace with two crates: `cli` (binary `sausage`) and
|
||||||
|
`agent`. **(spike)** Example actors are in `examples/`, outside the workspace.
|
||||||
|
- **(you)** `sausage init` builds the agent and asks for the API shape, the
|
||||||
|
endpoint, the model, and the token. **(spike)** The token can also come from
|
||||||
|
`SAUSAGE_API_TOKEN`. The source contains no endpoint and no token.
|
||||||
|
|
||||||
## Minimal on purpose
|
## Messages
|
||||||
|
|
||||||
The harness does only what it needs to keep going. Everything else can be an
|
- **(spike)** Stdin line: `{"id", "from", "reply_to"?, "body"}`.
|
||||||
actor, or a change that the agent makes to itself.
|
Stdout line: `{"to"?, "id"?, "reply_to"?, "body"}`.
|
||||||
|
- **(spike)** The broker sets `from`. An actor can choose its own `id`. The
|
||||||
|
broker then uses `"<sender>/<id>"`, so IDs cannot collide or be faked.
|
||||||
|
Without an `id`, the broker assigns `m-<n>`.
|
||||||
|
- **(you)** Without `to`, a message goes to the subscribers of the sender.
|
||||||
|
**(spike)** `subscribe` / `unsubscribe` messages to the broker. The registry
|
||||||
|
stores subscriptions, so they survive restarts and upgrades. `init`
|
||||||
|
subscribes `console` to `agent`.
|
||||||
|
- **(spike)** Errors from routing come back as
|
||||||
|
`{"type": "undeliverable", "reason", "to"}` from `broker`.
|
||||||
|
- **(spike)** Lifecycle: `{"type": "shutdown"}` from `broker`, then EOF on stdin.
|
||||||
|
After 5 seconds the broker kills the instance.
|
||||||
|
- **(spike)** Lines on stdout that are not JSON objects with `body` go to the
|
||||||
|
actor's `stderr.log` with a note.
|
||||||
|
|
||||||
- **The host is not self-modifiable; actors are.** So the host keeps only
|
## Filesystem
|
||||||
what an actor cannot do for itself: routing, running instances, versions,
|
|
||||||
tests before an install, and the rollback of a version that crash-loops.
|
|
||||||
Without that rollback, a broken self-upgrade would end the agent.
|
|
||||||
- **No permissions, no limits, no hiding.** Any actor can message, upgrade,
|
|
||||||
or stop any actor, read every folder, and use the network. People who want rules
|
|
||||||
can build them.
|
|
||||||
- **No re-delivery and no state snapshots.** A crash loses the message, and
|
|
||||||
all versions of an actor share `data/` and `public/`. A rollback restores
|
|
||||||
code, not data.
|
|
||||||
- **The agent is dumb.** It has a short prompt, a few tools, an append-only
|
|
||||||
history that it sends in full, and no compaction. When the context is too
|
|
||||||
large, it stops sending the oldest turns, so it never gets stuck.
|
|
||||||
|
|
||||||
## Small choices
|
- **(you)** `/data` private, `/public` own public files,
|
||||||
|
`/actor/<id>/public` and `/actor/<id>/source` read-only for everyone.
|
||||||
|
- **(spike)** The broker also writes `/actor/<id>/status.json` and
|
||||||
|
`/actor/<id>/stderr.log`. `ls /actor` is the registry.
|
||||||
|
- **(you)** Commit on close: a write becomes visible only when the writer
|
||||||
|
closes the file. **(spike)** This also applies to `/data`. Implementation: a
|
||||||
|
wrapper around the `wasmtime-wasi` filesystem. An open for write goes to a
|
||||||
|
hidden `.sausage-tmp-*` file next to the target. The descriptor drop renames
|
||||||
|
it into place. Details:
|
||||||
|
- A normal exit commits every open file (process exit closes files).
|
||||||
|
- A trap or kill discards uncommitted writes.
|
||||||
|
- An append or an edit without truncate copies the old content first.
|
||||||
|
- Other descriptors see the old content until the close.
|
||||||
|
- Directory operations take effect immediately.
|
||||||
|
- The layer hides `.sausage-tmp-*` names from listings, `stat`, `open`,
|
||||||
|
`rename`, and `unlink`.
|
||||||
|
- Symlinks are disabled (they would break the path tracking).
|
||||||
|
- No `fsync`. A power loss can lose committed data.
|
||||||
|
- **(spike)** Raw sockets are not available (no network permission), but the
|
||||||
|
`wasi:sockets` interfaces are linked, so programs that import them still
|
||||||
|
start.
|
||||||
|
|
||||||
- **Plain POSIX files.** An earlier version staged every write until close.
|
## Broker requests
|
||||||
That broke `tempfile`, git, and SQLite, which expect normal files. Atomic
|
|
||||||
publishing is the usual temporary file + `rename`.
|
- **(spike)** `spawn`, `upgrade`, `rollback`, `stop`, `start`, `remove`,
|
||||||
- **Test first, then switch.** A new version is tested where it lies. Only
|
`subscribe`, `unsubscribe`, `list`.
|
||||||
a version that passes goes to `versions/`, and the old one runs until
|
- **(you)** `spawn` and `upgrade` take `wasm`, plus optional `source` and
|
||||||
then. `versions/` keeps every version that was installed, so a rollback
|
`built_by` metadata. **(spike)** The broker copies the `.wasm` file and the
|
||||||
can always go back. A file that fails stays where it was put.
|
source folder into `broker/versions/<id>/<n>/`, with `meta.json` (paths,
|
||||||
- **Tests are JSON transcripts** next to the source. They run on a copy of
|
builder, requester, time).
|
||||||
the actor's state, without network, so a test cannot send real requests.
|
- **(spike)** Path resolution: `/actor/...` is global, and `/public/...` and
|
||||||
- **`broker` and `cargo` never answer a reply,** so no two actors answer
|
`/data/...` belong to the sender. `..` is rejected. Native senders (the
|
||||||
each other forever. The agent starts a turn only for text that is not a
|
console) can give absolute host paths.
|
||||||
reply, for the same reason.
|
- **(you)** Anyone can upgrade anyone. No permissions.
|
||||||
- **The SDK is not published.** `cargo` points `sausage-actor = "0.1"` at its
|
- **(spike)** Actor IDs: `a-z`, `0-9`, `-`, `_`, at most 64 characters.
|
||||||
own copy, which it also shows in `/actor/cargo/public/sdk/`.
|
- **(spike)** `remove` archives every folder of the actor to `archive/`. Nothing
|
||||||
|
is deleted.
|
||||||
|
|
||||||
|
## Upgrades, tests, rollback
|
||||||
|
|
||||||
|
- **(spike)** The tests are message transcripts in `source/tests/*.jsonl`:
|
||||||
|
`{"send": body}` and `{"expect": subset-of-body, "to"?}`. Each file runs in
|
||||||
|
a fresh candidate on a copy of the actor's `/data` and `/public`. The
|
||||||
|
candidate has no HTTP access. An `expect` waits up to 10 s. Without tests,
|
||||||
|
the candidate must stay up for 2 s (smoke test).
|
||||||
|
- **(spike)** Swap: hold the mailbox, stop the old instance gracefully,
|
||||||
|
snapshot `/data` + `/public` into the old version's folder, install the new
|
||||||
|
source in `/actor/<id>/source`, start the new instance, release the mailbox.
|
||||||
|
- **(spike)** Restart after a crash: backoff 0.5 s × 2ⁿ, at most 30 s. The
|
||||||
|
counter resets after 60 s of uptime.
|
||||||
|
- **(spike)** Parked message: when an instance traps, the broker keeps the
|
||||||
|
last delivered message. It delivers that message again after the next
|
||||||
|
upgrade, not after a plain restart (that would loop).
|
||||||
|
- **(spike)** Crash loop: 3 crashes within 30 s after an upgrade → automatic
|
||||||
|
rollback to the previous version, with its data snapshot. The broker
|
||||||
|
informs the actor that requested the upgrade.
|
||||||
|
- **(spike)** `rollback` without `version` goes to the highest older version.
|
||||||
|
It restores that version's snapshot, and it first snapshots the current
|
||||||
|
state, so a roll-forward is possible.
|
||||||
|
- **(spike)** Versions are per actor, not global.
|
||||||
|
|
||||||
|
## Limits
|
||||||
|
|
||||||
|
- **(spike)** Mailbox: 1,000 messages. When it is full, the sender gets
|
||||||
|
`undeliverable`.
|
||||||
|
- **(spike)** The broker waits 30 s when it writes to an actor's stdin. Then it
|
||||||
|
drops the message and logs it.
|
||||||
|
- **(spike)** Time slices: the epoch advances every 10 ms, and each guest
|
||||||
|
yields at each epoch.
|
||||||
|
- **(spike)** One `sausage run` per world (file lock on `world/.lock`).
|
||||||
|
- **(spike)** HTTP for all running actors, with no allow-list yet.
|
||||||
|
|
||||||
|
## Agent
|
||||||
|
|
||||||
|
- **(you)** API shapes: OpenAI Responses and Anthropic Messages, chosen by
|
||||||
|
`"api"` in `/data/config.json`.
|
||||||
|
- **(spike)** Config fields: `api`, `base_url`, `model`, `token`, optional
|
||||||
|
`reasoning_effort` and `max_tokens` (16,000). OpenAI: `POST
|
||||||
|
{base_url}/v1/responses` with `store: false` and encrypted reasoning passed
|
||||||
|
back. Anthropic: `POST {base_url}/v1/messages` with `x-api-key`.
|
||||||
|
- **(spike)** One shared conversation in `/data/history.json`, in a neutral
|
||||||
|
format, so the API shape can change. The history is limited to 600 entries,
|
||||||
|
with the cut at a user message. There is no compaction yet.
|
||||||
|
- **(spike)** A turn starts only for `{"type": "text"}` messages *without*
|
||||||
|
`reply_to`. Replies never start turns, so two actors cannot keep the agent
|
||||||
|
in an endless loop. Other messages are collected and shown at the next
|
||||||
|
turn.
|
||||||
|
- **(spike)** Tools: `send_message` (with optional wait for the reply),
|
||||||
|
`wait_for_messages`, `read_file`, `list_dir`, `write_file`, `copy_dir`,
|
||||||
|
`remove_path`. At most 40 steps per turn. Tool output is cut at 30 KB.
|
||||||
|
- **(spike)** The final text of a turn goes to the sender, as a reply to the
|
||||||
|
message that started the turn.
|
||||||
|
- **(spike)** A shutdown during a turn saves the turn to `/data/turn.json`. The
|
||||||
|
next start (for example after a self-upgrade) resumes the turn and answers
|
||||||
|
the original message.
|
||||||
|
|
||||||
|
## Console
|
||||||
|
|
||||||
|
- **(spike)** `@<actor> <json|text>`, plain text to `agent`, `/deploy`,
|
||||||
|
`/list`, `/sleep`, `/await`, `/help`, `/quit`. `/sleep` and `/await` exist
|
||||||
|
for scripted tests.
|
||||||
|
|
||||||
|
## Known risks (accepted for now)
|
||||||
|
|
||||||
|
- Build scripts and proc macros run as native code in the host environment.
|
||||||
|
- Any actor can upgrade the agent with code that reads the token.
|
||||||
|
- Any actor can message, stop, upgrade, or remove any actor.
|
||||||
|
- A full copy of `/data` and `/public` at every upgrade is slow for large data.
|
||||||
|
- Mailboxes are in memory. A host crash loses queued messages.
|
||||||
|
|||||||
-252
@@ -1,252 +0,0 @@
|
|||||||
# The Sausage protocol
|
|
||||||
|
|
||||||
This is the contract between the host and the actors. An actor in any
|
|
||||||
language that follows it works in Sausage. The host publishes this file at
|
|
||||||
`/actor/broker/public/PROTOCOL.md`, so every actor can read it.
|
|
||||||
|
|
||||||
## Actors
|
|
||||||
|
|
||||||
An actor is a WASI 0.2 command (`wasm32-wasip2`): a program with `main`. The
|
|
||||||
host runs one instance per actor. An instance can run forever.
|
|
||||||
|
|
||||||
The instance gets:
|
|
||||||
|
|
||||||
- **stdin:** one message per line, from the broker
|
|
||||||
- **stdout:** one message per line, to the broker
|
|
||||||
- **stderr:** free text, appended to `/actor/<id>/stderr.log`
|
|
||||||
- **environment:** `SAUSAGE_ACTOR` (its ID), `SAUSAGE_VERSION` (its version),
|
|
||||||
and `SAUSAGE_CANDIDATE=1` during a test run
|
|
||||||
- **the network:** outgoing HTTP and HTTPS through `wasi:http` (the host does
|
|
||||||
TLS), and TCP, UDP, and DNS through `wasi:sockets`, including listening on
|
|
||||||
a port of the host
|
|
||||||
- **the filesystem** described below
|
|
||||||
|
|
||||||
## Messages
|
|
||||||
|
|
||||||
A line on **stdin**:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"id": "m-17", "from": "console", "reply_to": "agent/a5", "body": {"type": "text", "text": "hi"}}
|
|
||||||
```
|
|
||||||
|
|
||||||
A line on **stdout**:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"to": "console", "id": "a6", "reply_to": "m-17", "body": {"type": "text", "text": "hello"}}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `body` is required. By convention it is an object with a `"type"`.
|
|
||||||
Text for people is `{"type": "text", "text": "..."}`.
|
|
||||||
- `to` is the recipient. Without `to`, the broker publishes the message to
|
|
||||||
every subscriber of the sender.
|
|
||||||
- `id` is an optional string. If the actor sets it, the message ID is `"<sender>/<id>"`.
|
|
||||||
Otherwise, the broker assigns `m-<n>`. The broker always sets `from`.
|
|
||||||
- `reply_to` marks a reply: its value is the ID of the message it answers.
|
|
||||||
There is no request/response. A reply is a new message.
|
|
||||||
- **Never answer a reply,** or two actors can answer each other forever.
|
|
||||||
`broker` and `cargo` ignore messages that have a `reply_to`.
|
|
||||||
- Messages are fire and forget: delivery is at most once. Each actor has a
|
|
||||||
mailbox of 1,000 messages, which keeps messages while the actor restarts
|
|
||||||
or is upgraded. If the broker cannot deliver a message, the sender gets
|
|
||||||
`{"type": "undeliverable", "reason": "...", "to": "..."}` from `broker`.
|
|
||||||
- An actor must keep reading stdin. If the broker must stop or upgrade an
|
|
||||||
actor while a message waits for it to read stdin, it kills the actor.
|
|
||||||
|
|
||||||
Before the broker stops an instance, it sends
|
|
||||||
`{"id": "broker/shutdown", "from": "broker", "body": {"type": "shutdown"}}`
|
|
||||||
and then closes stdin. The broker still routes what the actor writes until
|
|
||||||
it exits. After 5 seconds, it kills the instance.
|
|
||||||
|
|
||||||
## Filesystem
|
|
||||||
|
|
||||||
Each instance sees three folders:
|
|
||||||
|
|
||||||
| Path | Access | Contents |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `/data` | read-write by its actor; readable by every actor | the actor's state |
|
|
||||||
| `/public` | read-write | the files that the actor publishes |
|
|
||||||
| `/actor` | read-only | the `actor/` folder of the world |
|
|
||||||
|
|
||||||
An actor can write only its own `/data`, but can read every actor's `/data` through `/actor/<id>/data/`. `/data` is not private: any unprotected secret file there is readable by every actor.
|
|
||||||
|
|
||||||
`/actor/<id>/` holds everything about an actor: `actor.wasm`, `actor.json`
|
|
||||||
(installed version, running, subscriptions, tapped, watches), `source/`, `public/`, `data/`,
|
|
||||||
`stderr.log`, and `versions/<n>/`. `ls /actor` lists all actors.
|
|
||||||
|
|
||||||
Files are plain POSIX files: a write is visible at once. To publish a file
|
|
||||||
so that readers never see half of it, write a temporary file and `rename` it
|
|
||||||
(the Rust SDK has `write_atomic`).
|
|
||||||
|
|
||||||
Send large or binary data as a path under `/actor/<id>/public/`, not inside a
|
|
||||||
message.
|
|
||||||
|
|
||||||
### Actor-served files: `/actor/<id>/fs/`
|
|
||||||
|
|
||||||
Files under `/actor/<id>/fs/` are not on disk: the actor `<id>` serves them.
|
|
||||||
Every actor can read and write them. The host turns each file operation into a
|
|
||||||
message from `fs` to `<id>` and waits up to 10 seconds for the reply. `path`
|
|
||||||
is relative to `fs/` and starts with `/` (the folder itself is `/`).
|
|
||||||
|
|
||||||
To make this virtual directory discoverable, an actor that serves it declares
|
|
||||||
`Actor-served files` and `fs_read` in its leading `//!` documentation in
|
|
||||||
`source/src/main.rs`. The host uses that declaration when listing
|
|
||||||
`/actor/<id>/`; it does not probe every actor (which could stall on actors that
|
|
||||||
do not handle filesystem messages). This is discovery metadata only: opening
|
|
||||||
and reading the path still requires a valid `fs_read` reply.
|
|
||||||
|
|
||||||
| Request | Reply |
|
|
||||||
| --- | --- |
|
|
||||||
| `{"type": "fs_read", "path"}` | `{"type": "fs_file", "text"}` (or `"base64"` instead of `"text"`), or `{"type": "fs_dir", "entries": [{"name", "type": "file" \| "dir"}]}` |
|
|
||||||
| `{"type": "fs_write", "path", "text"}` (or `"base64"` for non-UTF-8 data) | `{"type": "fs_done"}` |
|
|
||||||
| `{"type": "fs_mkdir", "path"}`, `{"type": "fs_remove", "path"}` | `{"type": "fs_done"}` |
|
|
||||||
| `{"type": "fs_rename", "path", "to"}` | `{"type": "fs_done"}` |
|
|
||||||
|
|
||||||
Any request can be answered with `{"type": "fs_error", "errno": "no-entry"}`.
|
|
||||||
`errno` is a `wasi:filesystem` error code: `no-entry`, `exist`,
|
|
||||||
`not-directory`, `is-directory`, `not-empty`, `access`, `not-permitted`,
|
|
||||||
`invalid`, `unsupported`, `busy`, or `io` (also for anything else).
|
|
||||||
|
|
||||||
- Opening a file sends `fs_read` once. Reads come from that copy.
|
|
||||||
- After a change, closing (or syncing) the file sends the whole file in one
|
|
||||||
`fs_write`.
|
|
||||||
- No reply within 10 seconds, or an actor that does not exist, makes the
|
|
||||||
operation fail (`io`, or `no-entry`). An actor that opens its own `fs/`
|
|
||||||
waits for itself and fails after the timeout.
|
|
||||||
- `fs` is reserved: no actor can have that ID. It also sends `fs_changed`
|
|
||||||
to watchers (see Watches).
|
|
||||||
|
|
||||||
## The broker
|
|
||||||
|
|
||||||
Requests to `broker`. The answer is a message whose `reply_to` is the ID of
|
|
||||||
the request: the request's `type` in past tense (`spawned`, `upgraded`,
|
|
||||||
`rolled_back`, `stopped`, `started`, `removed`, `subscribed`, `unsubscribed`,
|
|
||||||
`tapping`, `untapped`, `watching`, `unwatched`, `exiting`), or `{"type": "error", "reason": "..."}`.
|
|
||||||
|
|
||||||
| `type` | Fields | Effect |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `spawn` | `actor`, `wasm`, `source`?, `built_by`? | test, then start a new actor |
|
|
||||||
| `upgrade` | `actor`, `wasm`, `source`?, `built_by`? | test, then swap the version |
|
|
||||||
| `rollback` | `actor`, `version` | go back to an older version |
|
|
||||||
| `stop`, `start` | `actor` | stop or start an actor |
|
|
||||||
| `remove` | `actor` | stop the actor (normal shutdown), then delete its whole folder `/actor/<id>/` (all versions, `data/`, `public/`, logs); answer `{"type": "removed", "actor"}`. Messages to it are then undeliverable, and the ID can be spawned again. An unknown actor or a native one (`broker`, `console`, `cargo`, `fs`) is an `error` |
|
|
||||||
| `subscribe`, `unsubscribe` | `actor` | receive (or not) what `actor` publishes |
|
|
||||||
| `tap`, `untap` | | receive (or not) a copy of every message the broker routes |
|
|
||||||
| `watch`, `unwatch` | `path` | receive (or not) `fs_changed` for every change of `path` or anything under it; answer `{"type": "watching" \| "unwatched", "path"}` with the path as an `/actor/...` path |
|
|
||||||
| `exit` | | answer `exiting`, stop every actor, and end the `sausage` process (a wrapper like `nix run .#loop` starts it again, updated) |
|
|
||||||
|
|
||||||
Paths are `/actor/...`, or the sender's `/public/...` or `/data/...`.
|
|
||||||
|
|
||||||
**Taps.** After `tap`, the sender gets a copy of every message that the
|
|
||||||
broker routes (direct messages, replies, and publishes, between any actors,
|
|
||||||
including its own), for debugging, logging, or a live view of the world:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"type": "tapped", "message": {"id": "m-17", "from": "console", "to": "agent", "reply_to": null, "body": {...}}, "ts": 1792132579067}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `to` is `null` for a publish: one copy per publish, not per subscriber.
|
|
||||||
- `ts` is the time of routing, in milliseconds since the Unix epoch.
|
|
||||||
- Copies come from `broker`. They are not routed themselves, so they are
|
|
||||||
never tapped, and each message is tapped once per tapper.
|
|
||||||
- A copy that does not fit into the tapper's mailbox is dropped; routing
|
|
||||||
never waits for a tapper, and nobody gets `undeliverable` for a copy.
|
|
||||||
- Never answer a copy: the answer would be tapped again, forever.
|
|
||||||
- Like subscriptions, a tap is saved in `actor.json` (`"tapped": true`) and
|
|
||||||
survives restarts and upgrades. `untap` ends it.
|
|
||||||
|
|
||||||
**Watches.** The host watches the whole world folder with the OS file
|
|
||||||
watcher: every change inside it counts, whoever makes it (actors, a 9P mount,
|
|
||||||
a shell or `git` on the host). Actor-served files (`/actor/<id>/fs/`) are not
|
|
||||||
on disk and do not count. After `watch {path}`, the sender gets for each
|
|
||||||
change of `path` or a path under it (`/actor/a` covers `/actor/a/x`, not
|
|
||||||
`/actor/ab`):
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"type": "fs_changed", "path": "/actor/notes/public/list.md", "op": "write"}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `op` is `write`, `create`, `remove`, or `rename` (then `to` is the new
|
|
||||||
path; a watcher of either path gets it).
|
|
||||||
- Changes of one path within about 100 ms become one event (a `write` after
|
|
||||||
a `create` stays a `create`).
|
|
||||||
- The host's temporary files (names starting with `.sausage-tmp-`) are
|
|
||||||
skipped; renaming one onto a file is a `write` of that file.
|
|
||||||
- Events come from `fs`. Like tap copies, they are not routed, a full
|
|
||||||
mailbox drops them, and they never block the writer. Never answer them.
|
|
||||||
- If the OS drops events, every watcher gets one
|
|
||||||
`{"type": "fs_changed", "path": "/actor", "op": "rescan"}`: anything may
|
|
||||||
have changed.
|
|
||||||
- A watch is saved in `actor.json` (`"watches": [...]`) and survives
|
|
||||||
restarts and upgrades. `unwatch` with the same path ends it.
|
|
||||||
|
|
||||||
**Versions.** A new version (from `spawn`, `upgrade`, or a changed
|
|
||||||
`actor.wasm` on disk) is tested first, while the old version keeps running.
|
|
||||||
Only when its tests pass does the broker copy it to `versions/<n>/`
|
|
||||||
(`actor.wasm`, `source/`, `meta.json`) and switch to it. So `versions/` holds
|
|
||||||
exactly the versions that were installed, and instances always run from
|
|
||||||
there. If the tests fail, nothing changes, and the sender gets an `error`
|
|
||||||
with the reason. All versions share the actor's `data/` and `public/`: a
|
|
||||||
new version must read the data of the old one.
|
|
||||||
|
|
||||||
**Tests.** The broker runs the candidate on a copy of the actor's `data/`
|
|
||||||
and `public/`, without network. The tests are JSON files in
|
|
||||||
`<source>/tests/*.json`, one scenario per file:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"description": "add, then list",
|
|
||||||
"steps": [
|
|
||||||
{"send": {"type": "add", "text": "milk"}, "expect": {"type": "added"}},
|
|
||||||
{"send": {"type": "list"}, "expect": [{"type": "notes"}]}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Each step sends `send` (from `from`, default `test`), then reads the next
|
|
||||||
line that the candidate writes for each `expect` entry, in order, within 10
|
|
||||||
seconds. An entry matches when it is a subset of the message body. With
|
|
||||||
`to`, the message must also go to that recipient. Each file runs in a fresh
|
|
||||||
instance. A version without test files passes.
|
|
||||||
|
|
||||||
**Swap.** The broker holds the mailbox, stops the old instance, makes the
|
|
||||||
top-level `actor.wasm` and `source/` a copy of the new version, and starts
|
|
||||||
it. A rollback does the same with the older version.
|
|
||||||
|
|
||||||
**Crashes.** The broker restarts a crashed actor after 0.5 s, then doubles
|
|
||||||
the delay up to 30 s. A new version that crashes 3 times within 30 seconds
|
|
||||||
after an upgrade goes back to the previous version. The actor that asked for
|
|
||||||
the upgrade gets `{"type": "rolled_back", "from", "to", "reason", "last_exit"}`.
|
|
||||||
|
|
||||||
**The folder is the truth.** At start, the broker boots every
|
|
||||||
`actor/<id>/actor.wasm`. Afterwards, the file watcher reports a new or
|
|
||||||
changed `actor/<id>/actor.wasm` (or a new folder `actor/<id>/`) once it is
|
|
||||||
quiet for about 100 ms: a new file becomes a new actor, a changed one an
|
|
||||||
upgrade, tested first with the tests in `actor/<id>/source/`. A file equal
|
|
||||||
to the installed version changes nothing. If its tests fail, the file stays
|
|
||||||
where it is, the reason goes to `stderr.log`, and the next change of the
|
|
||||||
file tries again. When the OS drops events, the broker checks every actor
|
|
||||||
folder again, as at start.
|
|
||||||
|
|
||||||
## The `cargo` actor
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"type": "build", "source": "/actor/agent/public/drafts/notes"}
|
|
||||||
```
|
|
||||||
|
|
||||||
The build request may include `"skip_tests": true` to skip Rust unit tests.
|
|
||||||
By default, `cargo` first runs `cargo test` for the host target (not WASI),
|
|
||||||
so pure-logic unit tests in a `wasm32-wasip2` actor crate can run without a
|
|
||||||
WASI test runner. If tests fail or cannot compile, the build fails before the
|
|
||||||
WASI build. The reply carries `tests: {"passed": <count>, "failed": <count>,
|
|
||||||
"output": "<cargo test stdout and stderr>"}`; counts sum Cargo's test-result
|
|
||||||
summaries, and a test command/compilation failure without a summary counts as
|
|
||||||
one failure. With `skip_tests: true`, counts are zero and output says tests
|
|
||||||
were skipped.
|
|
||||||
|
|
||||||
The reply is `{"type": "built", "wasm": "/actor/cargo/public/builds/<n>/<name>.wasm", "built_by": "cargo", "tests": {...}, ...}`
|
|
||||||
or `{"type": "build_failed", "errors": "...", "tests": {...}}`. Pass
|
|
||||||
`wasm`, `source`, and `built_by` on to `spawn` or `upgrade`.
|
|
||||||
|
|
||||||
A Rust actor depends on the SDK with `sausage-actor = "0.1"`. `cargo` points
|
|
||||||
that dependency at the SDK of the host, which it publishes at
|
|
||||||
`/actor/cargo/public/sdk/`.
|
|
||||||
@@ -2,85 +2,64 @@
|
|||||||
|
|
||||||
You can see how the sausage is made.
|
You can see how the sausage is made.
|
||||||
|
|
||||||
Sausage is a world of small programs ("actors") that you grow by talking to
|
Sausage is a system of actors that you grow by talking to it. Every actor is a
|
||||||
it. Each actor is a WASI program with stdin and stdout. Actors send each other
|
WASI program (a `.wasm` file) with stdin and stdout. Actors send each other
|
||||||
JSON messages, one per line, through a broker. The agent is an actor too: it
|
newline-delimited JSON messages through a broker. The agent is an actor too:
|
||||||
writes new actors in Rust, builds them, tests them, starts them, and can
|
it does inference over HTTP, writes new actors, and can change its own code.
|
||||||
change its own code.
|
|
||||||
|
|
||||||
## Start
|
Inspired by [Jiti](https://ghuntley.com/lisp/), but with WASM instead of a
|
||||||
|
Lisp image.
|
||||||
|
|
||||||
You need [Nix](https://nixos.org/download) with flakes, and an API token for
|
## Quick start
|
||||||
an LLM (OpenAI Responses or Anthropic Messages API). No clone needed:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
nix run git+https://gitea.surma.technology/surma/sausage
|
nix develop
|
||||||
|
cargo build --release
|
||||||
|
export SAUSAGE_API_TOKEN=... # or type it at the prompt
|
||||||
|
./target/release/sausage init world --api openai-responses \
|
||||||
|
--base-url https://api.example.com --model some-model
|
||||||
|
./target/release/sausage run world
|
||||||
```
|
```
|
||||||
|
|
||||||
This creates a world in `./world` the first time (it builds the agent, which
|
`init` builds the agent (the first build takes a few minutes) and writes its
|
||||||
takes a few minutes), then starts it with a console. Later runs start the
|
configuration to `world/private/agent/data/config.json`. The source contains
|
||||||
existing world. The two steps on their own:
|
no endpoint and no token.
|
||||||
|
|
||||||
```sh
|
|
||||||
nix run git+https://gitea.surma.technology/surma/sausage -- init # creates ./world and builds the agent
|
|
||||||
nix run git+https://gitea.surma.technology/surma/sausage -- run # starts the world and a console
|
|
||||||
```
|
|
||||||
|
|
||||||
`init` asks for the API shape (`openai-responses` or `anthropic-messages`),
|
|
||||||
the base URL, the model, and the token. It reads `SAUSAGE_API`,
|
|
||||||
`SAUSAGE_BASE_URL`, `SAUSAGE_MODEL`, and `SAUSAGE_API_TOKEN` first. The
|
|
||||||
values go to `world/actor/agent/data/config.json`. Both commands take another
|
|
||||||
folder as an argument, for example `... -- run my-world`.
|
|
||||||
|
|
||||||
To keep a world on the newest version, run it in a loop:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
nix run git+https://gitea.surma.technology/surma/sausage#loop # ./world, created if needed
|
|
||||||
nix run git+https://gitea.surma.technology/surma/sausage#loop -- my-world # an existing world
|
|
||||||
```
|
|
||||||
|
|
||||||
The loop runs `nix run --refresh git+https://gitea.surma.technology/surma/sausage -- run my-world`.
|
|
||||||
When the host crashes or an actor sends `@broker {"type":"exit"}`, the loop
|
|
||||||
fetches the newest commit and starts it again. `/quit` ends the loop. Set
|
|
||||||
`SAUSAGE_FLAKE` to run another flake URL instead, for example your fork.
|
|
||||||
|
|
||||||
From a clone (`git clone https://gitea.surma.technology/surma/sausage`), the
|
|
||||||
same commands are `nix run`, `nix run -- init`, `nix run -- run` and
|
|
||||||
`nix run .#loop`.
|
|
||||||
|
|
||||||
In the console:
|
In the console:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
write an actor "notes" that stores notes, and start it → text to the agent
|
hello → text to the agent
|
||||||
@notes {"type":"add","text":"milk"} → a JSON message to an actor
|
@notes {"type":"add","text":"milk"} → a JSON message to an actor
|
||||||
@agent hello → text to an actor
|
@agent what actors exist? → text to an actor
|
||||||
|
/deploy notes examples/notes → build with cargo, then spawn or upgrade
|
||||||
|
/list → list actors
|
||||||
|
/await agent 300 → wait for the next message (scripts)
|
||||||
/quit
|
/quit
|
||||||
```
|
```
|
||||||
|
|
||||||
## The world folder
|
## Concepts
|
||||||
|
|
||||||
The folder is the source of truth. Every actor lives in `world/actor/<id>/`:
|
- **Actor**: a `wasi:cli/command` program. It reads messages on stdin and
|
||||||
|
writes messages on stdout. Logs go to stderr.
|
||||||
|
- **Message**: `{"id", "from", "reply_to"?, "body"}` on stdin;
|
||||||
|
`{"to"?, "id"?, "reply_to"?, "body"}` on stdout. No `to` = publish to
|
||||||
|
subscribers. There is no request/response: a reply is a new message with
|
||||||
|
`reply_to`. Delivery is at most once.
|
||||||
|
- **Broker**: routes messages, owns mailboxes, and runs actors. It accepts
|
||||||
|
only `.wasm` files: `spawn`, `upgrade`, `rollback`, `stop`, `start`,
|
||||||
|
`remove`, `subscribe`, `unsubscribe`, `list`.
|
||||||
|
- **cargo**: the built-in Rust compiler actor. `build` a source folder, get a
|
||||||
|
`.wasm` path back. Other actors can add other languages.
|
||||||
|
- **Filesystem** of each actor:
|
||||||
|
- `/data`: private state
|
||||||
|
- `/public`: its public files
|
||||||
|
- `/actor/<id>/{public,source,status.json,stderr.log}`: every actor, read-only
|
||||||
|
|
||||||
```text
|
A write becomes visible when the writer closes the file (commit on close).
|
||||||
actor.wasm the current version
|
- **Upgrade**: the broker tests the candidate on a copy of `/data` and
|
||||||
actor.json installed version, running, subscriptions
|
`/public` (transcripts in `source/tests/*.jsonl`). Then it holds new
|
||||||
source/ the source of the current version
|
messages, stops the old version, starts the new one, and delivers the held
|
||||||
public/ files that the actor publishes
|
messages. A new version that crashes 3 times in 30 seconds is rolled back.
|
||||||
data/ actor state; owner-writable, world-readable via /actor/<id>/data/ (not private)
|
|
||||||
stderr.log what the actor and the broker write about it
|
|
||||||
versions/<n>/ every installed version: actor.wasm, source/, meta.json
|
|
||||||
```
|
|
||||||
|
|
||||||
To add or replace an actor by hand, put an `actor.wasm` (and its `source/`,
|
See [DECISIONS.md](DECISIONS.md) for every design decision and
|
||||||
with the tests in `source/tests/`) into its folder; the host watches the
|
[SPIKE.md](SPIKE.md) for the spike results.
|
||||||
world folder and picks the file up once it stops changing. The broker tests
|
|
||||||
the file before it uses it.
|
|
||||||
If the tests fail, the old version keeps running, and `stderr.log` says why.
|
|
||||||
|
|
||||||
## More
|
|
||||||
|
|
||||||
- [PROTOCOL.md](PROTOCOL.md): messages, the filesystem, the broker, `cargo`.
|
|
||||||
Read this to write an actor in any language.
|
|
||||||
- [`sdk/`](sdk/): `sausage-actor`, the SDK for Rust actors.
|
|
||||||
- [`examples/notes`](examples/notes): a small actor with a test.
|
|
||||||
- [AGENTS.md](AGENTS.md): how to work on Sausage itself.
|
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
# Spike results
|
||||||
|
|
||||||
|
Date: 2026-10-15. Host: Scout container (8 cores), Rust 1.99, Wasmtime 49.0.3,
|
||||||
|
`wstd` 0.6.8. All numbers come from `sausage run` with release builds.
|
||||||
|
|
||||||
|
## 1. Host, broker, routing, filesystem — works
|
||||||
|
|
||||||
|
- `sausage init` builds the agent and registers it. `sausage run` starts the
|
||||||
|
broker, `cargo`, the console, and every registered actor.
|
||||||
|
- Routing, replies (`reply_to`), publish to subscribers, and `undeliverable`
|
||||||
|
errors work.
|
||||||
|
- **Read-only:** a write to `/actor/<other>/public/...` fails with "Operation
|
||||||
|
not permitted". A write to `/actor/<self>/public/...` fails too. A write to
|
||||||
|
`/public/...` works.
|
||||||
|
- **Commit on close:** while `fs-a` holds `/public/y.txt` open, `fs-b` gets
|
||||||
|
"No such file". `fs-a` itself also gets it through a second descriptor. The
|
||||||
|
temporary file does not appear in listings. After the close, `fs-b` reads
|
||||||
|
the content.
|
||||||
|
- **Append:** `append` to an existing file copies the old content. Readers
|
||||||
|
see the old version until the close.
|
||||||
|
- **Crash with an open file:** the write is discarded, and the old version
|
||||||
|
stays.
|
||||||
|
- **Escapes:** `/data/../../../../etc/hostname` and
|
||||||
|
`/actor/../private/agent/data/config.json` fail.
|
||||||
|
- **Hard kill of the host:** after `kill`, no `.sausage-tmp-*` files remained,
|
||||||
|
and the next `sausage run` started every actor again.
|
||||||
|
|
||||||
|
## 2. Async stdin + HTTPS in one actor — works
|
||||||
|
|
||||||
|
`examples/fetch` (`wstd`) handles `ping` at once while a 2-second HTTPS
|
||||||
|
request runs. HTTPS and plain HTTP both work through `wasi:http`, and the host
|
||||||
|
does TLS. Bodies go to `/public/fetch/<n>`, and the reply contains the
|
||||||
|
global path.
|
||||||
|
|
||||||
|
One server (`http://neverssl.com/`) never answered. `curl` also hangs on it,
|
||||||
|
so the server is the cause. But a request without a timeout keeps its task
|
||||||
|
alive, and the actor then did not exit on `shutdown`. The broker killed it
|
||||||
|
after 5 s, as designed. Actors should set HTTP timeouts.
|
||||||
|
|
||||||
|
## 3. Upgrade cycle — works
|
||||||
|
|
||||||
|
- **No loss during a swap:** `ticker` sent 200 messages to `notes` at 50 ms
|
||||||
|
intervals. `notes` was upgraded from v1 to v2 during the stream. The swap
|
||||||
|
happened after tick 68. The v1 snapshot has ticks 1–68, and v2 has ticks
|
||||||
|
1–200, in order, with no gaps.
|
||||||
|
- **Parked message:** `{"type":"crash"}` crashed v1. The broker restarted v1
|
||||||
|
and kept the message. After the upgrade to v2 (which handles `crash`), the
|
||||||
|
broker delivered it again, and v2 answered `not_crashing`.
|
||||||
|
- **Crash loop:** an upgrade to `examples/crasher` passed the smoke test (it
|
||||||
|
crashes after 3 s). It crashed 3 times in about 10 s. The broker rolled back
|
||||||
|
to v2 with its data and sent `rolled_back` to the console.
|
||||||
|
- **Explicit rollback** v2 → v1 restored the v1 snapshot.
|
||||||
|
- **Rejections:** a compiler error comes back as `build_failed` with the
|
||||||
|
rendered diagnostics. A failing transcript rejects the upgrade, and the old
|
||||||
|
version keeps running. Invalid IDs and `..` paths are rejected.
|
||||||
|
|
||||||
|
## 4. Agent — works with both API shapes
|
||||||
|
|
||||||
|
- **`gpt-6-luna`** (OpenAI Responses) through the vendors proxy:
|
||||||
|
- Asked "which actors exist", it called `broker` `list` and answered in
|
||||||
|
about 4 s (2 inferences).
|
||||||
|
- Asked for a `counter` actor, it took these steps without help:
|
||||||
|
1. It wrote `Cargo.toml`, `src/main.rs`, and `tests/basic.jsonl`.
|
||||||
|
2. It built the crate with `cargo` (1.9 s).
|
||||||
|
3. It spawned the actor (the tests passed).
|
||||||
|
4. It sent increment, increment, get (1, 2, 2).
|
||||||
|
5. It read `/actor/counter/public/value.txt`.
|
||||||
|
- **Self-upgrade:** asked to change its own prompt, it took these steps:
|
||||||
|
1. It copied its source and changed only `prompt.md` (a 3-line diff).
|
||||||
|
2. It built the change and upgraded `agent`.
|
||||||
|
3. The upgrade stopped it in the middle of the turn. The new version
|
||||||
|
resumed the turn from `/data/turn.json` and answered with the new
|
||||||
|
signature.
|
||||||
|
- **`claude-haiku-4-5`** (Anthropic Messages): the same conversation
|
||||||
|
continued, with tool calls.
|
||||||
|
|
||||||
|
## 5. Measurements
|
||||||
|
|
||||||
|
| What | Result |
|
||||||
|
| --- | --- |
|
||||||
|
| Round trip actor → actor → actor | p50 88 µs, p99 172 µs |
|
||||||
|
| Sequential round trips | ~10,400 per second |
|
||||||
|
| Warm `cargo` build, small actor | 0.5–1.9 s |
|
||||||
|
| Warm `cargo` build, agent | 5.5 s |
|
||||||
|
| First build of the agent (`init`) | 17 s |
|
||||||
|
| Component compile, small actor | ~70 ms |
|
||||||
|
| Component compile, agent (870 KB) | ~290 ms |
|
||||||
|
| Host start, 8 actors compiled in parallel | ~0.8 s each |
|
||||||
|
| Host RSS with 8 running actors | ~117 MB |
|
||||||
|
| First release build of `sausage` | ~9 min (Wasmtime) |
|
||||||
|
|
||||||
|
Caveat: `~/.cargo/config.toml` in this container sets `kache` as the
|
||||||
|
`rustc-wrapper`. `kache` caches compiler outputs across projects, so the
|
||||||
|
first build of the agent (17 s) is probably faster than on a clean machine.
|
||||||
|
|
||||||
|
## Next steps I suggest
|
||||||
|
|
||||||
|
1. A `matrix` actor (WASM, `wstd`, long-polled `/sync`). The agent could write
|
||||||
|
it, with you checking the result.
|
||||||
|
2. Cache compiled components (`Component::serialize`, `.cwasm` files) to
|
||||||
|
remove the compile time at host start.
|
||||||
|
3. A network allow-list per actor, for example in a manifest in the source
|
||||||
|
folder.
|
||||||
|
4. A sandbox for builds (for example `bwrap`), because build scripts run
|
||||||
|
natively.
|
||||||
|
5. Context compaction for the agent's conversation.
|
||||||
|
6. A better console (line editing, history).
|
||||||
+1
-1
@@ -5,7 +5,7 @@ edition = "2024"
|
|||||||
description = "The Sausage agent actor."
|
description = "The Sausage agent actor."
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
sausage-actor = { version = "0.1", path = "../sdk", features = ["async"] }
|
async-channel = "2"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
wstd = "0.6"
|
wstd = "0.6"
|
||||||
|
|||||||
@@ -1,81 +0,0 @@
|
|||||||
//! History window selection and context-size error detection.
|
|
||||||
|
|
||||||
use crate::llm::Entry;
|
|
||||||
|
|
||||||
pub fn is_too_large(error: &str) -> bool {
|
|
||||||
let Some((status, body)) = error
|
|
||||||
.strip_prefix("HTTP ")
|
|
||||||
.and_then(|error| error.split_once(':'))
|
|
||||||
else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
if !matches!(status.split_whitespace().next(), Some("400" | "413")) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let body = body.to_ascii_lowercase();
|
|
||||||
body.contains("context_length_exceeded")
|
|
||||||
|| body.contains("maximum context length")
|
|
||||||
|| body.contains("context length exceeded")
|
|
||||||
|| body.contains("too large")
|
|
||||||
|| (body.contains("too long")
|
|
||||||
&& ["context", "input", "prompt"]
|
|
||||||
.iter()
|
|
||||||
.any(|term| body.contains(term)))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn next_turn_start(history: &[Entry], from: usize, current: usize) -> Option<usize> {
|
|
||||||
let end = current.min(history.len().checked_sub(1)?);
|
|
||||||
(from..=end).find(|&index| matches!(&history[index], Entry::User { .. }))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn detects_context_size_errors_only_for_http_400_or_413() {
|
|
||||||
assert!(is_too_large("HTTP 400: context_length_exceeded"));
|
|
||||||
assert!(is_too_large("HTTP 413: prompt is too long"));
|
|
||||||
assert!(is_too_large("HTTP 400: maximum context length"));
|
|
||||||
assert!(is_too_large("HTTP 400: request is too large"));
|
|
||||||
assert!(!is_too_large("HTTP 500: prompt is too long"));
|
|
||||||
assert!(!is_too_large("HTTP 400: invalid token"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn history_skip_advances_by_whole_user_turns() {
|
|
||||||
let history = vec![
|
|
||||||
Entry::User {
|
|
||||||
text: "first".into(),
|
|
||||||
},
|
|
||||||
Entry::ToolCall {
|
|
||||||
id: "a".into(),
|
|
||||||
name: "tool".into(),
|
|
||||||
arguments: "{}".into(),
|
|
||||||
},
|
|
||||||
Entry::ToolResult {
|
|
||||||
id: "a".into(),
|
|
||||||
output: "done".into(),
|
|
||||||
},
|
|
||||||
Entry::User {
|
|
||||||
text: "second".into(),
|
|
||||||
},
|
|
||||||
Entry::ToolCall {
|
|
||||||
id: "b".into(),
|
|
||||||
name: "tool".into(),
|
|
||||||
arguments: "{}".into(),
|
|
||||||
},
|
|
||||||
Entry::ToolResult {
|
|
||||||
id: "b".into(),
|
|
||||||
output: "done".into(),
|
|
||||||
},
|
|
||||||
Entry::User {
|
|
||||||
text: "current".into(),
|
|
||||||
},
|
|
||||||
];
|
|
||||||
assert_eq!(next_turn_start(&history, 0, 6), Some(0));
|
|
||||||
assert_eq!(next_turn_start(&history, 1, 6), Some(3));
|
|
||||||
assert_eq!(next_turn_start(&history, 4, 6), Some(6));
|
|
||||||
assert_eq!(next_turn_start(&history, usize::MAX, 6).unwrap_or(6), 6);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+29
-109
@@ -6,6 +6,7 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use wstd::http::{Body, Client, Request};
|
use wstd::http::{Body, Client, Request};
|
||||||
|
use wstd::time::Duration;
|
||||||
|
|
||||||
#[derive(Deserialize, Clone)]
|
#[derive(Deserialize, Clone)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
@@ -14,63 +15,25 @@ pub struct Config {
|
|||||||
pub base_url: String,
|
pub base_url: String,
|
||||||
pub model: String,
|
pub model: String,
|
||||||
pub token: String,
|
pub token: String,
|
||||||
/// Optional reasoning effort for OpenAI models only (wins over `thinking` there).
|
/// Optional reasoning effort for OpenAI models ("low", "medium", ...).
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub reasoning_effort: Option<String>,
|
pub reasoning_effort: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default = "default_max_tokens")]
|
||||||
pub max_tokens: Option<u32>,
|
pub max_tokens: u32,
|
||||||
/// Thinking effort for both APIs: "low", "medium", "high", "xhigh", "max" (`LEVELS`), or "off"/unset for none.
|
|
||||||
/// Anthropic: adaptive thinking + output_config.effort; OpenAI: reasoning.effort. A level the model rejects is
|
|
||||||
/// replaced by the nearest accepted one (logged), never by "off".
|
|
||||||
#[serde(default)]
|
|
||||||
pub thinking: Option<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Thinking levels, lowest first.
|
fn default_max_tokens() -> u32 {
|
||||||
pub const LEVELS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"];
|
16000
|
||||||
|
|
||||||
/// The level nearest to `want` (lower on a tie) among those a 400 error lists in quotes (else all levels), not
|
|
||||||
/// yet `tried`. None: nothing left, so the request goes out without an effort field (model default).
|
|
||||||
fn fallback_effort(want: &str, error: &str, tried: &[String]) -> Option<String> {
|
|
||||||
let listed: Vec<&str> = error.split('\'').skip(1).step_by(2).filter(|w| LEVELS.contains(w)).collect();
|
|
||||||
let pool = if listed.iter().any(|l| *l != want) { listed } else { LEVELS.to_vec() };
|
|
||||||
let at = LEVELS.iter().position(|l| *l == want).unwrap_or(1) as i32;
|
|
||||||
pool.into_iter()
|
|
||||||
.filter(|l| *l != want && !tried.iter().any(|t| t == l))
|
|
||||||
.min_by_key(|l| {
|
|
||||||
let i = LEVELS.iter().position(|x| x == l).unwrap_or(1) as i32;
|
|
||||||
((i - at).abs(), i > at)
|
|
||||||
})
|
|
||||||
.map(str::to_string)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::fallback_effort;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nearest_accepted_level() {
|
|
||||||
let anthropic = "HTTP 400: output_config.effort: Input should be 'low', 'medium', 'high' or 'max'";
|
|
||||||
assert_eq!(fallback_effort("xhigh", anthropic, &["xhigh".into()]).as_deref(), Some("high"));
|
|
||||||
let openai = "HTTP 400: Invalid value: 'ultra'. Supported values are: 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', and 'max'.";
|
|
||||||
assert_eq!(fallback_effort("ultra", openai, &["ultra".into()]).as_deref(), Some("medium"));
|
|
||||||
// No list in the error: step to the neighbour, never to off.
|
|
||||||
assert_eq!(fallback_effort("max", "HTTP 400: effort max not supported", &["max".into()]).as_deref(), Some("xhigh"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
pub enum Entry {
|
pub enum Entry {
|
||||||
User { text: String },
|
User { text: String },
|
||||||
Assistant {
|
Assistant { text: String },
|
||||||
text: String,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
phase: Option<String>,
|
|
||||||
},
|
|
||||||
ToolCall { id: String, name: String, arguments: String },
|
ToolCall { id: String, name: String, arguments: String },
|
||||||
ToolResult { id: String, output: String },
|
ToolResult { id: String, output: String },
|
||||||
/// A provider item that only this API shape understands (OpenAI reasoning, Anthropic thinking).
|
/// A provider item that only this API shape understands (OpenAI reasoning).
|
||||||
Opaque { api: String, item: Value },
|
Opaque { api: String, item: Value },
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,28 +44,10 @@ pub struct Tool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn complete(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry]) -> Result<Vec<Entry>, String> {
|
pub async fn complete(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry]) -> Result<Vec<Entry>, String> {
|
||||||
let want = match cfg.api.as_str() {
|
match cfg.api.as_str() {
|
||||||
"openai-responses" => cfg.reasoning_effort.clone().or_else(|| cfg.thinking.clone()),
|
"openai-responses" => openai(cfg, system, tools, history).await,
|
||||||
_ => cfg.thinking.clone(),
|
"anthropic-messages" => anthropic(cfg, system, tools, history).await,
|
||||||
}
|
other => Err(format!("unknown API shape {other:?}")),
|
||||||
.filter(|t| t != "off");
|
|
||||||
let mut effort = want.clone();
|
|
||||||
let mut tried = Vec::new();
|
|
||||||
loop {
|
|
||||||
let result = match cfg.api.as_str() {
|
|
||||||
"openai-responses" => openai(cfg, system, tools, history, effort.as_deref()).await,
|
|
||||||
"anthropic-messages" => anthropic(cfg, system, tools, history, want.is_some(), effort.as_deref()).await,
|
|
||||||
other => return Err(format!("unknown API shape {other:?}")),
|
|
||||||
};
|
|
||||||
match (result, effort.clone()) {
|
|
||||||
// The model rejected the effort level: retry with the nearest accepted one (thinking stays on).
|
|
||||||
(Err(e), Some(current)) if e.starts_with("HTTP 400") && e.contains("effort") => {
|
|
||||||
tried.push(current.clone());
|
|
||||||
effort = fallback_effort(want.as_deref().unwrap_or(""), &e, &tried);
|
|
||||||
eprintln!("agent: {} rejected thinking effort {current:?}; retrying with {effort:?}: {e}", cfg.model);
|
|
||||||
}
|
|
||||||
(result, _) => return result,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,29 +57,27 @@ async fn post(url: &str, headers: &[(&str, String)], body: &Value) -> Result<Val
|
|||||||
req = req.header(*k, v.as_str());
|
req = req.header(*k, v.as_str());
|
||||||
}
|
}
|
||||||
let req = req.body(Body::from(body.to_string())).map_err(|e| format!("request: {e}"))?;
|
let req = req.body(Body::from(body.to_string())).map_err(|e| format!("request: {e}"))?;
|
||||||
let client = Client::new();
|
let mut client = Client::new();
|
||||||
|
client.set_first_byte_timeout(Duration::from_secs(600));
|
||||||
|
client.set_between_bytes_timeout(Duration::from_secs(600));
|
||||||
let resp = client.send(req).await.map_err(|e| format!("HTTP request failed: {e:?}"))?;
|
let resp = client.send(req).await.map_err(|e| format!("HTTP request failed: {e:?}"))?;
|
||||||
let status = resp.status();
|
let status = resp.status();
|
||||||
let mut body = resp.into_body();
|
let mut body = resp.into_body();
|
||||||
let text = body.str_contents().await.map_err(|e| format!("reading the response failed: {e:?}"))?.to_string();
|
let text = body.str_contents().await.map_err(|e| format!("reading the response failed: {e:?}"))?.to_string();
|
||||||
if !status.is_success() {
|
if !status.is_success() {
|
||||||
return Err(format!("HTTP {status}: {text}"));
|
let mut t = text;
|
||||||
|
t.truncate(2000);
|
||||||
|
return Err(format!("HTTP {status}: {t}"));
|
||||||
}
|
}
|
||||||
serde_json::from_str(&text).map_err(|e| format!("invalid JSON in response: {e}"))
|
serde_json::from_str(&text).map_err(|e| format!("invalid JSON in response: {e}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn openai(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], effort: Option<&str>) -> Result<Vec<Entry>, String> {
|
async fn openai(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry]) -> Result<Vec<Entry>, String> {
|
||||||
let mut input = Vec::new();
|
let mut input = Vec::new();
|
||||||
for e in history {
|
for e in history {
|
||||||
match e {
|
match e {
|
||||||
Entry::User { text } => input.push(json!({"role": "user", "content": text})),
|
Entry::User { text } => input.push(json!({"role": "user", "content": text})),
|
||||||
Entry::Assistant { text, phase } => {
|
Entry::Assistant { text } => input.push(json!({"role": "assistant", "content": text})),
|
||||||
let mut item = json!({"role": "assistant", "content": text});
|
|
||||||
if let Some(phase) = phase {
|
|
||||||
item["phase"] = json!(phase);
|
|
||||||
}
|
|
||||||
input.push(item);
|
|
||||||
}
|
|
||||||
Entry::ToolCall { id, name, arguments } => {
|
Entry::ToolCall { id, name, arguments } => {
|
||||||
input.push(json!({"type": "function_call", "call_id": id, "name": name, "arguments": arguments}))
|
input.push(json!({"type": "function_call", "call_id": id, "name": name, "arguments": arguments}))
|
||||||
}
|
}
|
||||||
@@ -156,11 +99,9 @@ async fn openai(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], e
|
|||||||
"tools": tools,
|
"tools": tools,
|
||||||
"store": false,
|
"store": false,
|
||||||
"include": ["reasoning.encrypted_content"],
|
"include": ["reasoning.encrypted_content"],
|
||||||
|
"max_output_tokens": cfg.max_tokens,
|
||||||
});
|
});
|
||||||
if let Some(max_tokens) = cfg.max_tokens {
|
if let Some(effort) = &cfg.reasoning_effort {
|
||||||
body["max_output_tokens"] = json!(max_tokens);
|
|
||||||
}
|
|
||||||
if let Some(effort) = effort {
|
|
||||||
body["reasoning"] = json!({"effort": effort});
|
body["reasoning"] = json!({"effort": effort});
|
||||||
}
|
}
|
||||||
let url = format!("{}/v1/responses", cfg.base_url);
|
let url = format!("{}/v1/responses", cfg.base_url);
|
||||||
@@ -179,9 +120,8 @@ async fn openai(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], e
|
|||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let phase = item["phase"].as_str().map(str::to_string);
|
if !text.is_empty() {
|
||||||
if !text.is_empty() || phase.is_some() {
|
out.push(Entry::Assistant { text });
|
||||||
out.push(Entry::Assistant { text, phase });
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some("function_call") => out.push(Entry::ToolCall {
|
Some("function_call") => out.push(Entry::ToolCall {
|
||||||
@@ -199,14 +139,7 @@ async fn openai(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry], e
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn anthropic(
|
async fn anthropic(cfg: &Config, system: &str, tools: &[Tool], history: &[Entry]) -> Result<Vec<Entry>, String> {
|
||||||
cfg: &Config,
|
|
||||||
system: &str,
|
|
||||||
tools: &[Tool],
|
|
||||||
history: &[Entry],
|
|
||||||
thinking: bool,
|
|
||||||
effort: Option<&str>,
|
|
||||||
) -> Result<Vec<Entry>, String> {
|
|
||||||
// Merge consecutive blocks of the same role into one message.
|
// Merge consecutive blocks of the same role into one message.
|
||||||
let mut messages: Vec<(String, Vec<Value>)> = Vec::new();
|
let mut messages: Vec<(String, Vec<Value>)> = Vec::new();
|
||||||
let mut push = |role: &str, block: Value| match messages.last_mut() {
|
let mut push = |role: &str, block: Value| match messages.last_mut() {
|
||||||
@@ -216,9 +149,7 @@ async fn anthropic(
|
|||||||
for e in history {
|
for e in history {
|
||||||
match e {
|
match e {
|
||||||
Entry::User { text } => push("user", json!({"type": "text", "text": text})),
|
Entry::User { text } => push("user", json!({"type": "text", "text": text})),
|
||||||
Entry::Assistant { text, .. } => {
|
Entry::Assistant { text } => push("assistant", json!({"type": "text", "text": text})),
|
||||||
push("assistant", json!({"type": "text", "text": text}))
|
|
||||||
}
|
|
||||||
Entry::ToolCall { id, name, arguments } => {
|
Entry::ToolCall { id, name, arguments } => {
|
||||||
let input: Value = serde_json::from_str(arguments).unwrap_or(json!({}));
|
let input: Value = serde_json::from_str(arguments).unwrap_or(json!({}));
|
||||||
push("assistant", json!({"type": "tool_use", "id": id, "name": name, "input": input}))
|
push("assistant", json!({"type": "tool_use", "id": id, "name": name, "input": input}))
|
||||||
@@ -226,8 +157,6 @@ async fn anthropic(
|
|||||||
Entry::ToolResult { id, output } => {
|
Entry::ToolResult { id, output } => {
|
||||||
push("user", json!({"type": "tool_result", "tool_use_id": id, "content": output}))
|
push("user", json!({"type": "tool_result", "tool_use_id": id, "content": output}))
|
||||||
}
|
}
|
||||||
// Thinking blocks go back unchanged (with their signature): required next to tool use.
|
|
||||||
Entry::Opaque { api, item } if api == "anthropic-messages" && thinking => push("assistant", item.clone()),
|
|
||||||
Entry::Opaque { .. } => {}
|
Entry::Opaque { .. } => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -236,19 +165,13 @@ async fn anthropic(
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|t| json!({"name": t.name, "description": t.description, "input_schema": t.parameters}))
|
.map(|t| json!({"name": t.name, "description": t.description, "input_schema": t.parameters}))
|
||||||
.collect();
|
.collect();
|
||||||
let mut body = json!({
|
let body = json!({
|
||||||
"model": cfg.model,
|
"model": cfg.model,
|
||||||
"system": system,
|
"system": system,
|
||||||
"messages": messages,
|
"messages": messages,
|
||||||
"tools": tools,
|
"tools": tools,
|
||||||
"max_tokens": cfg.max_tokens.unwrap_or(32000),
|
"max_tokens": cfg.max_tokens,
|
||||||
});
|
});
|
||||||
if thinking {
|
|
||||||
body["thinking"] = json!({"type": "adaptive"});
|
|
||||||
if let Some(effort) = effort {
|
|
||||||
body["output_config"] = json!({"effort": effort});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let url = format!("{}/v1/messages", cfg.base_url);
|
let url = format!("{}/v1/messages", cfg.base_url);
|
||||||
let resp = post(
|
let resp = post(
|
||||||
&url,
|
&url,
|
||||||
@@ -262,7 +185,7 @@ async fn anthropic(
|
|||||||
Some("text") => {
|
Some("text") => {
|
||||||
let text = block["text"].as_str().unwrap_or_default().to_string();
|
let text = block["text"].as_str().unwrap_or_default().to_string();
|
||||||
if !text.is_empty() {
|
if !text.is_empty() {
|
||||||
out.push(Entry::Assistant { text, phase: None });
|
out.push(Entry::Assistant { text });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some("tool_use") => out.push(Entry::ToolCall {
|
Some("tool_use") => out.push(Entry::ToolCall {
|
||||||
@@ -270,9 +193,6 @@ async fn anthropic(
|
|||||||
name: block["name"].as_str().unwrap_or_default().to_string(),
|
name: block["name"].as_str().unwrap_or_default().to_string(),
|
||||||
arguments: block["input"].to_string(),
|
arguments: block["input"].to_string(),
|
||||||
}),
|
}),
|
||||||
Some("thinking") | Some("redacted_thinking") => {
|
|
||||||
out.push(Entry::Opaque { api: "anthropic-messages".into(), item: block })
|
|
||||||
}
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+418
-237
@@ -1,92 +1,242 @@
|
|||||||
//! The Sausage agent actor.
|
//! The Sausage agent: a normal WASM actor that does inference over HTTP and
|
||||||
|
//! grows the world through messages to `cargo` and `broker`.
|
||||||
|
//!
|
||||||
|
//! Configuration: `/data/config.json` (see `llm::Config`).
|
||||||
|
//! Conversation: `/data/history.json` (one shared conversation).
|
||||||
|
|
||||||
mod history;
|
|
||||||
mod llm;
|
mod llm;
|
||||||
mod tools;
|
|
||||||
|
|
||||||
use history::{is_too_large, next_turn_start};
|
use std::collections::VecDeque;
|
||||||
use llm::{Config, Entry};
|
|
||||||
use sausage_actor::{Inbox, Message, Value, json, publish, send};
|
use async_channel::{Receiver, Sender};
|
||||||
use std::collections::{HashSet, VecDeque};
|
use llm::{Config, Entry, Tool};
|
||||||
use std::io::Write;
|
use serde_json::{Value, json};
|
||||||
use wstd::future::FutureExt;
|
use wstd::future::FutureExt;
|
||||||
|
use wstd::io::{AsyncRead, AsyncWrite};
|
||||||
use wstd::time::{Duration, Instant};
|
use wstd::time::{Duration, Instant};
|
||||||
|
|
||||||
const SYSTEM: &str = include_str!("prompt.md");
|
const SYSTEM: &str = include_str!("prompt.md");
|
||||||
const CONFIG: &str = "/data/config.json";
|
const CONFIG: &str = "/data/config.json";
|
||||||
const HISTORY: &str = "/data/history.jsonl";
|
const HISTORY: &str = "/data/history.json";
|
||||||
const HISTORY_SKIP: &str = "/data/history.skip";
|
/// An interrupted turn (sender and message ID), resumed at the next start.
|
||||||
const TURN: &str = "/data/turn.json";
|
const TURN: &str = "/data/turn.json";
|
||||||
|
const MAX_STEPS: usize = 40;
|
||||||
|
const MAX_HISTORY: usize = 600;
|
||||||
|
const MAX_TOOL_OUTPUT: usize = 30_000;
|
||||||
|
const MAX_NOTES: usize = 50;
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
wstd::runtime::block_on(async { Agent::new().run().await });
|
wstd::runtime::block_on(async {
|
||||||
|
let (tx, rx) = async_channel::unbounded();
|
||||||
|
wstd::runtime::spawn(read_stdin(tx)).detach();
|
||||||
|
Agent::new(rx).run().await;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read NDJSON messages from stdin into the channel.
|
||||||
|
async fn read_stdin(tx: Sender<Value>) {
|
||||||
|
let mut stdin = wstd::io::stdin();
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
let mut chunk = vec![0u8; 16 * 1024];
|
||||||
|
loop {
|
||||||
|
match stdin.read(&mut chunk).await {
|
||||||
|
Ok(0) | Err(_) => break,
|
||||||
|
Ok(n) => {
|
||||||
|
buf.extend_from_slice(&chunk[..n]);
|
||||||
|
while let Some(pos) = buf.iter().position(|b| *b == b'\n') {
|
||||||
|
let line: Vec<u8> = buf.drain(..=pos).collect();
|
||||||
|
if let Ok(v) = serde_json::from_slice::<Value>(&line)
|
||||||
|
&& tx.send(v).await.is_err()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tx.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate(mut s: String, max: usize) -> String {
|
||||||
|
if s.len() > max {
|
||||||
|
let mut end = max;
|
||||||
|
while !s.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
let total = s.len();
|
||||||
|
s.truncate(end);
|
||||||
|
s.push_str(&format!("\n… (truncated, {total} bytes in total)"));
|
||||||
|
}
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tools() -> Vec<Tool> {
|
||||||
|
vec![
|
||||||
|
Tool {
|
||||||
|
name: "send_message",
|
||||||
|
description: "Send a message to an actor. With wait_seconds > 0, wait for the reply (a message whose reply_to is the ID of this message). Returns the message ID, the reply (or null), and other messages that arrived meanwhile.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"to": {"type": "string", "description": "Actor ID"},
|
||||||
|
"body": {"type": "object", "description": "Message body. Must have a \"type\" field."},
|
||||||
|
"wait_seconds": {"type": "number", "description": "Seconds to wait for the reply. 0 = do not wait."}
|
||||||
|
},
|
||||||
|
"required": ["to", "body"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "wait_for_messages",
|
||||||
|
description: "Wait up to `seconds` for messages to arrive and return them.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"seconds": {"type": "number"}},
|
||||||
|
"required": ["seconds"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "read_file",
|
||||||
|
description: "Read a text file. Paths: /data/..., /public/..., /actor/<id>/...",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"path": {"type": "string"},
|
||||||
|
"offset": {"type": "integer", "description": "Byte offset (default 0)"},
|
||||||
|
"max_bytes": {"type": "integer", "description": "Default 30000"}
|
||||||
|
},
|
||||||
|
"required": ["path"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "list_dir",
|
||||||
|
description: "List a folder.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"path": {"type": "string"}},
|
||||||
|
"required": ["path"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "write_file",
|
||||||
|
description: "Write a text file (only under /data or /public). Creates parent folders.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
||||||
|
"required": ["path", "content"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "copy_dir",
|
||||||
|
description: "Copy a folder tree, for example /actor/<id>/source to /public/drafts/<id>. Skips target/ folders.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"from": {"type": "string"}, "to": {"type": "string"}},
|
||||||
|
"required": ["from", "to"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Tool {
|
||||||
|
name: "remove_path",
|
||||||
|
description: "Remove a file or folder under /data or /public.",
|
||||||
|
parameters: json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"path": {"type": "string"}},
|
||||||
|
"required": ["path"]
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_tree(from: &str, to: &str) -> std::io::Result<u64> {
|
||||||
|
std::fs::create_dir_all(to)?;
|
||||||
|
let mut n = 0;
|
||||||
|
for entry in std::fs::read_dir(from)? {
|
||||||
|
let entry = entry?;
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
let src = format!("{from}/{name}");
|
||||||
|
let dst = format!("{to}/{name}");
|
||||||
|
if entry.file_type()?.is_dir() {
|
||||||
|
if name != "target" {
|
||||||
|
n += copy_tree(&src, &dst)?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
std::fs::copy(&src, &dst)?;
|
||||||
|
n += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Agent {
|
struct Agent {
|
||||||
inbox: Inbox,
|
rx: Receiver<Value>,
|
||||||
|
stdout: wstd::io::Stdout,
|
||||||
history: Vec<Entry>,
|
history: Vec<Entry>,
|
||||||
history_skip: usize,
|
/// Text messages that start a new turn later.
|
||||||
active: bool,
|
queue: VecDeque<Value>,
|
||||||
queue: VecDeque<Message>,
|
/// Other messages that arrived while the agent was idle.
|
||||||
notes: Vec<Message>,
|
notes: Vec<Value>,
|
||||||
|
seq: u64,
|
||||||
|
shutdown: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Agent {
|
impl Agent {
|
||||||
fn new() -> Self {
|
fn new(rx: Receiver<Value>) -> Self {
|
||||||
let history = std::fs::read_to_string(HISTORY)
|
let history = std::fs::read_to_string(HISTORY)
|
||||||
.unwrap_or_default()
|
|
||||||
.lines()
|
|
||||||
.filter_map(|line| serde_json::from_str(line).ok())
|
|
||||||
.collect();
|
|
||||||
let history_skip = std::fs::read_to_string(HISTORY_SKIP)
|
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|skip| skip.trim().parse::<usize>().ok())
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
.unwrap_or(0);
|
.unwrap_or_default();
|
||||||
let active = std::fs::read_to_string(TURN)
|
|
||||||
.ok()
|
|
||||||
.and_then(|contents| serde_json::from_str::<bool>(&contents).ok())
|
|
||||||
.unwrap_or(false);
|
|
||||||
Self {
|
Self {
|
||||||
inbox: Inbox::start(),
|
rx,
|
||||||
|
stdout: wstd::io::stdout(),
|
||||||
history,
|
history,
|
||||||
history_skip,
|
|
||||||
active,
|
|
||||||
queue: VecDeque::new(),
|
queue: VecDeque::new(),
|
||||||
notes: Vec::new(),
|
notes: Vec::new(),
|
||||||
|
seq: 0,
|
||||||
|
shutdown: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn add_history(&mut self, entry: Entry) {
|
async fn run(mut self) {
|
||||||
let line = serde_json::to_string(&entry).unwrap();
|
eprintln!("agent: started with {} history entries", self.history.len());
|
||||||
match std::fs::OpenOptions::new()
|
self.resume().await;
|
||||||
.create(true)
|
while !self.shutdown {
|
||||||
.append(true)
|
let msg = match self.queue.pop_front() {
|
||||||
.open(HISTORY)
|
Some(m) => m,
|
||||||
{
|
None => match self.rx.recv().await {
|
||||||
Ok(mut file) => {
|
Ok(m) => m,
|
||||||
if let Err(e) = writeln!(file, "{line}") {
|
Err(_) => break,
|
||||||
eprintln!("agent: cannot append history: {e}");
|
},
|
||||||
|
};
|
||||||
|
if msg["body"]["type"] == "shutdown" {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if starts_turn(&msg) {
|
||||||
|
self.turn(msg).await;
|
||||||
|
} else {
|
||||||
|
self.notes.push(msg);
|
||||||
|
if self.notes.len() > MAX_NOTES {
|
||||||
|
self.notes.remove(0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => eprintln!("agent: cannot append history: {e}"),
|
|
||||||
}
|
}
|
||||||
self.history.push(entry);
|
self.save();
|
||||||
|
eprintln!("agent: stopped");
|
||||||
}
|
}
|
||||||
|
|
||||||
fn finish_turn(&mut self) {
|
/// Continue a turn that a restart (usually an upgrade of the agent itself)
|
||||||
self.active = false;
|
/// interrupted.
|
||||||
|
async fn resume(&mut self) {
|
||||||
|
let Some(turn) = std::fs::read_to_string(TURN).ok().and_then(|s| serde_json::from_str::<Value>(&s).ok()) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
let _ = std::fs::remove_file(TURN);
|
let _ = std::fs::remove_file(TURN);
|
||||||
}
|
let Ok(cfg) = std::fs::read_to_string(CONFIG).map_err(|_| ()).and_then(|s| serde_json::from_str::<Config>(&s).map_err(|_| ())) else {
|
||||||
|
return;
|
||||||
fn closing(&self) -> bool {
|
};
|
||||||
self.inbox.is_closed()
|
// Every tool call needs a result, or the APIs reject the history.
|
||||||
}
|
let answered: std::collections::HashSet<String> = self
|
||||||
|
|
||||||
fn repair_dangling_tools(&mut self) {
|
|
||||||
let answered: HashSet<String> = self
|
|
||||||
.history
|
.history
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|entry| match entry {
|
.filter_map(|e| match e {
|
||||||
Entry::ToolResult { id, .. } => Some(id.clone()),
|
Entry::ToolResult { id, .. } => Some(id.clone()),
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
@@ -94,246 +244,277 @@ impl Agent {
|
|||||||
let dangling: Vec<String> = self
|
let dangling: Vec<String> = self
|
||||||
.history
|
.history
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|entry| match entry {
|
.filter_map(|e| match e {
|
||||||
Entry::ToolCall { id, .. } if !answered.contains(id) => Some(id.clone()),
|
Entry::ToolCall { id, .. } if !answered.contains(id) => Some(id.clone()),
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
for id in dangling {
|
for id in dangling {
|
||||||
self.add_history(Entry::ToolResult {
|
self.history.push(Entry::ToolResult {
|
||||||
id,
|
id,
|
||||||
output:
|
output: json!({"error": "interrupted: the agent restarted before this tool finished"}).to_string(),
|
||||||
json!({"error": "interrupted: the agent restarted before this tool finished"})
|
|
||||||
.to_string(),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
fn turn_text(&self, message: &Message) -> String {
|
|
||||||
let mut text = format!(
|
|
||||||
"Message {} from {}:\n{}",
|
|
||||||
message.id,
|
|
||||||
message.from,
|
|
||||||
message.body["text"].as_str().unwrap_or("")
|
|
||||||
);
|
|
||||||
if !self.notes.is_empty() {
|
|
||||||
let notes = self
|
|
||||||
.notes
|
|
||||||
.iter()
|
|
||||||
.map(|note| serde_json::to_string(note).unwrap())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("\n");
|
|
||||||
text = format!("Messages that arrived while you were idle:\n{notes}\n\n{text}");
|
|
||||||
}
|
|
||||||
text
|
|
||||||
}
|
|
||||||
|
|
||||||
fn not_configured(&mut self, error: String) {
|
|
||||||
let _ = publish(json!({
|
|
||||||
"type": "text",
|
|
||||||
"text": format!("I am not configured: cannot read {CONFIG}: {error}")
|
|
||||||
}));
|
|
||||||
self.finish_turn();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run(mut self) {
|
|
||||||
self.repair_dangling_tools();
|
|
||||||
if self.active {
|
|
||||||
self.turn(None).await;
|
|
||||||
if self.closing() && self.active {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
loop {
|
|
||||||
let message = match self.queue.pop_front() {
|
|
||||||
Some(message) => message,
|
|
||||||
None => match self.inbox.recv().await {
|
|
||||||
Some(message) => message,
|
|
||||||
None => break,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
if starts_turn(&message) {
|
|
||||||
self.turn(Some(message)).await;
|
|
||||||
if self.closing() && self.active {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
self.notes.push(message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn turn(&mut self, message: Option<Message>) {
|
|
||||||
let current_start = if message.is_some() {
|
|
||||||
self.active = true;
|
|
||||||
let _ = std::fs::write(TURN, "true");
|
|
||||||
self.history.len()
|
|
||||||
} else {
|
|
||||||
let current_start = self
|
|
||||||
.history
|
|
||||||
.iter()
|
|
||||||
.rposition(|entry| matches!(entry, Entry::User { text } if !text.starts_with("[note from your runtime]")))
|
|
||||||
.unwrap_or(self.history.len());
|
|
||||||
let version = std::env::var("SAUSAGE_VERSION").unwrap_or_default();
|
let version = std::env::var("SAUSAGE_VERSION").unwrap_or_default();
|
||||||
self.add_history(Entry::User {
|
self.history.push(Entry::User {
|
||||||
text: format!(
|
text: format!(
|
||||||
"[note from your runtime] you restarted and now run version {version}. Continue the active turn."
|
"[note from your runtime] You restarted and now run version {version}. The restart interrupted your last turn (probably your own upgrade). Check the state you need, then finish the turn and answer the original message."
|
||||||
),
|
),
|
||||||
});
|
});
|
||||||
current_start
|
eprintln!("agent: resuming an interrupted turn");
|
||||||
};
|
self.steps(&cfg, turn["from"].clone(), turn["id"].clone()).await;
|
||||||
let cfg = match load_config() {
|
|
||||||
Ok(cfg) => cfg,
|
|
||||||
Err(error) => {
|
|
||||||
self.not_configured(error);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
if let Some(message) = message {
|
fn save(&mut self) {
|
||||||
let text = self.turn_text(&message);
|
if self.history.len() > MAX_HISTORY {
|
||||||
self.notes.clear();
|
let start = self.history.len() - MAX_HISTORY;
|
||||||
self.add_history(Entry::User { text });
|
let cut = (start..self.history.len())
|
||||||
|
.find(|i| matches!(self.history[*i], Entry::User { .. }))
|
||||||
|
.unwrap_or(start);
|
||||||
|
self.history.drain(..cut);
|
||||||
}
|
}
|
||||||
self.steps(&cfg, current_start).await;
|
if let Ok(s) = serde_json::to_string(&self.history)
|
||||||
if !self.closing() {
|
&& let Err(e) = std::fs::write(HISTORY, s)
|
||||||
self.finish_turn();
|
{
|
||||||
|
eprintln!("agent: cannot save history: {e}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn steps(&mut self, cfg: &Config, current_start: usize) {
|
async fn emit(&mut self, msg: Value) {
|
||||||
let tools = tools::definitions();
|
let mut line = msg.to_string();
|
||||||
loop {
|
line.push('\n');
|
||||||
if self.closing() {
|
if self.stdout.write_all(line.as_bytes()).await.is_err() || self.stdout.flush().await.is_err() {
|
||||||
return;
|
eprintln!("agent: cannot write to stdout");
|
||||||
}
|
}
|
||||||
let start = next_turn_start(&self.history, self.history_skip, current_start)
|
|
||||||
.unwrap_or(current_start);
|
|
||||||
if start != self.history_skip {
|
|
||||||
self.history_skip = start;
|
|
||||||
let _ = std::fs::write(HISTORY_SKIP, start.to_string());
|
|
||||||
}
|
}
|
||||||
let response = llm::complete(cfg, SYSTEM, &tools, &self.history[start..]).await;
|
|
||||||
let entries = match response {
|
async fn reply(&mut self, to: &Value, reply_to: &Value, text: String) {
|
||||||
Ok(entries) => entries,
|
self.emit(json!({"to": to, "reply_to": reply_to, "body": {"type": "text", "text": text}})).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sort a message that arrived during a turn.
|
||||||
|
fn stash(&mut self, m: Value, others: &mut Vec<Value>) {
|
||||||
|
if m["body"]["type"] == "shutdown" {
|
||||||
|
self.shutdown = true;
|
||||||
|
} else if starts_turn(&m) {
|
||||||
|
others.push(json!({"queued_for_a_later_turn": true, "message": m}));
|
||||||
|
self.queue.push_back(m);
|
||||||
|
} else {
|
||||||
|
others.push(m);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn turn(&mut self, msg: Value) {
|
||||||
|
let from = msg["from"].clone();
|
||||||
|
let id = msg["id"].clone();
|
||||||
|
let cfg: Config = match std::fs::read_to_string(CONFIG).map_err(|e| e.to_string()).and_then(|s| serde_json::from_str(&s).map_err(|e| e.to_string())) {
|
||||||
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if is_too_large(&e) {
|
self.reply(&from, &id, format!("I am not configured: cannot read {CONFIG}: {e}")).await;
|
||||||
if let Some(next) =
|
|
||||||
next_turn_start(&self.history, start.saturating_add(1), current_start)
|
|
||||||
{
|
|
||||||
self.history_skip = next;
|
|
||||||
let _ = std::fs::write(HISTORY_SKIP, next.to_string());
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let text = format!("Inference failed: {e}");
|
|
||||||
self.add_history(Entry::Assistant {
|
|
||||||
text: text.clone(),
|
|
||||||
phase: None,
|
|
||||||
});
|
|
||||||
let _ = publish(json!({"type": "text", "text": text}));
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let mut text = format!(
|
||||||
|
"Message {} from {}:\n{}",
|
||||||
|
id.as_str().unwrap_or("?"),
|
||||||
|
from.as_str().unwrap_or("?"),
|
||||||
|
msg["body"]["text"].as_str().unwrap_or("")
|
||||||
|
);
|
||||||
|
if !self.notes.is_empty() {
|
||||||
|
let notes: Vec<String> = self.notes.drain(..).map(|n| n.to_string()).collect();
|
||||||
|
text = format!("Messages that arrived while you were idle:\n{}\n\n{text}", notes.join("\n"));
|
||||||
|
}
|
||||||
|
self.history.push(Entry::User { text });
|
||||||
|
self.steps(&cfg, from, id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run inference and tools until the model answers. If a shutdown
|
||||||
|
/// interrupts the turn, remember it in TURN so the next start resumes it.
|
||||||
|
async fn steps(&mut self, cfg: &Config, from: Value, id: Value) {
|
||||||
|
let tools = tools();
|
||||||
|
let cfg = cfg.clone();
|
||||||
|
for _ in 0..MAX_STEPS {
|
||||||
|
let started = Instant::now();
|
||||||
|
let result = llm::complete(&cfg, SYSTEM, &tools, &self.history).await;
|
||||||
|
let entries = match result {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("agent: inference failed: {e}");
|
||||||
|
self.history.push(Entry::Assistant { text: format!("(inference failed: {e})") });
|
||||||
|
self.save();
|
||||||
|
self.reply(&from, &id, format!("Inference failed: {e}")).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
eprintln!("agent: inference took {:?}", started.elapsed());
|
||||||
let calls: Vec<(String, String, String)> = entries
|
let calls: Vec<(String, String, String)> = entries
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|entry| match entry {
|
.filter_map(|e| match e {
|
||||||
Entry::ToolCall {
|
Entry::ToolCall { id, name, arguments } => Some((id.clone(), name.clone(), arguments.clone())),
|
||||||
id,
|
|
||||||
name,
|
|
||||||
arguments,
|
|
||||||
} => Some((id.clone(), name.clone(), arguments.clone())),
|
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
for entry in entries {
|
let answer: Vec<String> = entries
|
||||||
if let Entry::Assistant { text, .. } = &entry {
|
.iter()
|
||||||
if !text.is_empty() {
|
.filter_map(|e| match e {
|
||||||
let _ = publish(json!({"type": "text", "text": text}));
|
Entry::Assistant { text } => Some(text.clone()),
|
||||||
}
|
_ => None,
|
||||||
}
|
})
|
||||||
self.add_history(entry);
|
.collect();
|
||||||
}
|
self.history.extend(entries);
|
||||||
if self.closing() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if calls.is_empty() {
|
if calls.is_empty() {
|
||||||
|
self.save();
|
||||||
|
let answer = if answer.is_empty() { "(no answer)".to_string() } else { answer.join("\n\n") };
|
||||||
|
self.reply(&from, &id, answer).await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (id, name, args) in calls {
|
for (call_id, name, args) in calls {
|
||||||
if self.closing() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let output = self.tool(&name, &args).await;
|
let output = self.tool(&name, &args).await;
|
||||||
self.add_history(Entry::ToolResult {
|
self.history.push(Entry::ToolResult { id: call_id, output: truncate(output.to_string(), MAX_TOOL_OUTPUT) });
|
||||||
id,
|
}
|
||||||
output: output.to_string(),
|
self.save();
|
||||||
});
|
if self.shutdown {
|
||||||
|
let _ = std::fs::write(TURN, json!({"from": from, "id": id}).to_string());
|
||||||
|
eprintln!("agent: turn interrupted by shutdown; it resumes after the restart");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
self.reply(&from, &id, format!("I stopped after {MAX_STEPS} steps. Tell me to continue if you want.")).await;
|
||||||
|
|
||||||
fn stash(&mut self, message: Message, others: &mut Vec<Value>) {
|
|
||||||
if starts_turn(&message) {
|
|
||||||
others.push(json!({"queued_for_a_later_turn": true, "message": message}));
|
|
||||||
self.queue.push_back(message);
|
|
||||||
} else {
|
|
||||||
others.push(serde_json::to_value(message).unwrap());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn drain_inbox(&mut self, others: &mut Vec<Value>) {
|
|
||||||
while let Some(message) = self.inbox.try_recv() {
|
|
||||||
self.stash(message, others);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn tool(&mut self, name: &str, args: &str) -> Value {
|
async fn tool(&mut self, name: &str, args: &str) -> Value {
|
||||||
let args: Value = match serde_json::from_str(args) {
|
let a: Value = match serde_json::from_str(args) {
|
||||||
Ok(args) => args,
|
Ok(v) => v,
|
||||||
Err(error) => return json!({"error": format!("arguments are not valid JSON: {error}")}),
|
Err(e) => return json!({"error": format!("arguments are not valid JSON: {e}")}),
|
||||||
};
|
};
|
||||||
if name == "send_message" {
|
let path = |k: &str| a[k].as_str().unwrap_or_default().to_string();
|
||||||
return self.send_message(&args).await;
|
let err = |e: std::io::Error| json!({"error": e.to_string()});
|
||||||
|
match name {
|
||||||
|
"send_message" => self.send_message(&a).await,
|
||||||
|
"wait_for_messages" => {
|
||||||
|
let secs = a["seconds"].as_f64().unwrap_or(10.0).clamp(0.0, 900.0);
|
||||||
|
let mut others = Vec::new();
|
||||||
|
match self.rx.recv().timeout(Duration::from_secs_f64(secs)).await {
|
||||||
|
Ok(Ok(m)) => {
|
||||||
|
self.stash(m, &mut others);
|
||||||
|
while let Ok(m) = self.rx.try_recv() {
|
||||||
|
self.stash(m, &mut others);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Err(_)) => self.shutdown = true,
|
||||||
|
Err(_) => {}
|
||||||
|
}
|
||||||
|
json!({"messages": others})
|
||||||
|
}
|
||||||
|
"read_file" => {
|
||||||
|
let p = path("path");
|
||||||
|
match std::fs::read(&p) {
|
||||||
|
Ok(bytes) => {
|
||||||
|
let offset = (a["offset"].as_u64().unwrap_or(0) as usize).min(bytes.len());
|
||||||
|
let max = a["max_bytes"].as_u64().unwrap_or(30_000) as usize;
|
||||||
|
let end = (offset + max).min(bytes.len());
|
||||||
|
json!({
|
||||||
|
"path": p, "size": bytes.len(), "offset": offset,
|
||||||
|
"content": String::from_utf8_lossy(&bytes[offset..end]),
|
||||||
|
"truncated": end < bytes.len(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(e) => err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"list_dir" => match std::fs::read_dir(path("path")) {
|
||||||
|
Ok(rd) => {
|
||||||
|
let mut entries: Vec<Value> = rd
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.take(500)
|
||||||
|
.map(|e| {
|
||||||
|
let meta = e.metadata().ok();
|
||||||
|
json!({
|
||||||
|
"name": e.file_name().to_string_lossy(),
|
||||||
|
"type": if meta.as_ref().is_some_and(|m| m.is_dir()) { "dir" } else { "file" },
|
||||||
|
"size": meta.map(|m| m.len()),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
entries.sort_by(|x, y| x["name"].as_str().cmp(&y["name"].as_str()));
|
||||||
|
json!({"entries": entries})
|
||||||
|
}
|
||||||
|
Err(e) => err(e),
|
||||||
|
},
|
||||||
|
"write_file" => {
|
||||||
|
let p = path("path");
|
||||||
|
if let Some(parent) = std::path::Path::new(&p).parent() {
|
||||||
|
let _ = std::fs::create_dir_all(parent);
|
||||||
|
}
|
||||||
|
match std::fs::write(&p, a["content"].as_str().unwrap_or_default()) {
|
||||||
|
Ok(()) => json!({"written": p}),
|
||||||
|
Err(e) => err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"copy_dir" => match copy_tree(&path("from"), &path("to")) {
|
||||||
|
Ok(n) => json!({"copied_files": n}),
|
||||||
|
Err(e) => err(e),
|
||||||
|
},
|
||||||
|
"remove_path" => {
|
||||||
|
let p = path("path");
|
||||||
|
let r = if std::path::Path::new(&p).is_dir() { std::fs::remove_dir_all(&p) } else { std::fs::remove_file(&p) };
|
||||||
|
match r {
|
||||||
|
Ok(()) => json!({"removed": p}),
|
||||||
|
Err(e) => err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other => json!({"error": format!("unknown tool {other}")}),
|
||||||
}
|
}
|
||||||
tools::run(name, &args).unwrap_or_else(|error| json!({"error": error.to_string()}))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn send_message(&mut self, args: &Value) -> Value {
|
async fn send_message(&mut self, a: &Value) -> Value {
|
||||||
let Some(to) = args["to"].as_str() else {
|
let Some(to) = a["to"].as_str() else { return json!({"error": "missing \"to\""}) };
|
||||||
return json!({"error": "missing \"to\""});
|
if !a["body"].is_object() {
|
||||||
};
|
|
||||||
if !args["body"].is_object() {
|
|
||||||
return json!({"error": "\"body\" must be a JSON object"});
|
return json!({"error": "\"body\" must be a JSON object"});
|
||||||
}
|
}
|
||||||
let id = send(to, args["body"].clone());
|
self.seq += 1;
|
||||||
let wait = args["wait_seconds"].as_f64().unwrap_or(0.0).max(0.0);
|
let own = format!("a{}-{}", wstd_now_ms(), self.seq);
|
||||||
|
let full = format!("agent/{own}");
|
||||||
|
self.emit(json!({"to": to, "id": own, "body": a["body"]})).await;
|
||||||
|
let wait = a["wait_seconds"].as_f64().unwrap_or(0.0).clamp(0.0, 900.0);
|
||||||
let mut reply = Value::Null;
|
let mut reply = Value::Null;
|
||||||
let mut others = Vec::new();
|
let mut others = Vec::new();
|
||||||
if wait > 0.0 {
|
if wait > 0.0 {
|
||||||
let deadline = Instant::now() + Duration::from_secs_f64(wait);
|
let deadline = Instant::now() + Duration::from_secs_f64(wait);
|
||||||
loop {
|
loop {
|
||||||
match self.inbox.recv().timeout(deadline).await {
|
match self.rx.recv().timeout(deadline).await {
|
||||||
Ok(Some(message)) if message.reply_to.as_deref() == Some(&id) => {
|
Ok(Ok(m)) => {
|
||||||
reply = serde_json::to_value(message).unwrap();
|
if m["reply_to"] == full.as_str() {
|
||||||
|
reply = m;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
self.stash(m, &mut others);
|
||||||
|
if self.shutdown {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Err(_)) => {
|
||||||
|
// stdin closed: the broker stops us.
|
||||||
|
self.shutdown = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Ok(Some(message)) => self.stash(message, &mut others),
|
|
||||||
Ok(None) => break,
|
|
||||||
Err(_) => break,
|
Err(_) => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.drain_inbox(&mut others);
|
json!({"id": full, "reply": reply, "waited": wait > 0.0, "other_messages": others})
|
||||||
json!({"id": id, "reply": reply, "waited": wait > 0.0, "other_messages": others})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_config() -> Result<Config, String> {
|
/// A text message that is not a reply starts a turn. Replies never do, so
|
||||||
std::fs::read_to_string(CONFIG)
|
/// two actors cannot keep the agent in an endless conversation.
|
||||||
.map_err(|e| e.to_string())
|
fn starts_turn(m: &Value) -> bool {
|
||||||
.and_then(|contents| serde_json::from_str(&contents).map_err(|e| e.to_string()))
|
m["body"]["type"] == "text" && m.get("reply_to").is_none_or(Value::is_null)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn starts_turn(message: &Message) -> bool {
|
fn wstd_now_ms() -> u64 {
|
||||||
message.kind() == "text" && message.reply_to.is_none()
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_millis() as u64)
|
||||||
|
.unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|||||||
+79
-6
@@ -1,7 +1,80 @@
|
|||||||
You are the `agent` actor in a Sausage world of actors.
|
You are the agent of a Sausage world. Your actor ID is `agent`.
|
||||||
People talk to you through other actors. Your text goes to your subscribers.
|
|
||||||
Before writing an actor, read `/actor/broker/public/PROTOCOL.md` for messages, the broker, `cargo`, and tests.
|
|
||||||
Read the SDK at `/actor/cargo/public/sdk/` before you write an actor.
|
|
||||||
You can change your own source at `/actor/agent/source`.
|
|
||||||
|
|
||||||
The global filesystem is readable by every actor, including every actor's `/data` at `/actor/<id>/data/`. An actor can write only its own `/data` and `/public`. `/data` is not private: any unprotected secret file there is readable by every actor.
|
Sausage is a system of actors. Each actor is a WASI program (a `.wasm` file). Actors talk only through messages. People talk to you through other actors, for example `console` (a terminal) or later `matrix`. You help them by using actors, and by writing, testing, and upgrading actors. You can also change your own code.
|
||||||
|
|
||||||
|
# Messages
|
||||||
|
|
||||||
|
Every message has a sender (`from`), an ID, an optional `reply_to` (the ID of the message it answers), and a JSON `body`. A body is an object with a `"type"` field. There is no request/response: a reply is a new message with `reply_to` set. Delivery is at most once.
|
||||||
|
|
||||||
|
A WASM actor reads one JSON object per line on stdin:
|
||||||
|
`{"id":"m-17","from":"console","body":{...}}`
|
||||||
|
and writes one JSON object per line on stdout:
|
||||||
|
`{"to":"console","reply_to":"m-17","body":{...}}`
|
||||||
|
Without `"to"`, the broker publishes the message to the subscribers of the sender. An actor may set its own `"id"`. The broker then uses `"<sender>/<id>"`. stdout is only for messages. Logs go to stderr. The broker sends `{"type":"shutdown"}` before it stops an actor. EOF on stdin means "stop now".
|
||||||
|
|
||||||
|
# Filesystem
|
||||||
|
|
||||||
|
Each actor sees:
|
||||||
|
- `/data`: private state, read-write. Only the actor itself sees it.
|
||||||
|
- `/public`: its public files, read-write.
|
||||||
|
- `/actor/<id>/public`: the public files of every actor, read-only.
|
||||||
|
- `/actor/<id>/source`: the source of the running version of every actor, read-only.
|
||||||
|
- `/actor/<id>/status.json` and `/actor/<id>/stderr.log`: state and stderr log of every actor.
|
||||||
|
|
||||||
|
`ls /actor` lists all actors. A write becomes visible to other actors only when the writer closes the file. Send large or binary data as a file path, not in a message.
|
||||||
|
|
||||||
|
# Built-in actors
|
||||||
|
|
||||||
|
`broker` (only accepts `.wasm` files, knows no source language):
|
||||||
|
- `{"type":"spawn","actor":"<new-id>","wasm":"<path>","source":"<folder>","built_by":"cargo"}`
|
||||||
|
- `{"type":"upgrade","actor":"<id>","wasm":"<path>","source":"<folder>","built_by":"cargo"}`
|
||||||
|
- `{"type":"rollback","actor":"<id>"}` (optional `"version"`)
|
||||||
|
- `{"type":"stop","actor":"<id>"}`, `{"type":"start","actor":"<id>"}`, `{"type":"remove","actor":"<id>"}` (archives, never deletes)
|
||||||
|
- `{"type":"subscribe","actor":"<id>"}`, `{"type":"unsubscribe","actor":"<id>"}`, `{"type":"list"}`
|
||||||
|
|
||||||
|
Before an upgrade or spawn, the broker tests the candidate on a copy of the actor's `/data` and `/public`, with no HTTP access. The tests are message transcripts in `<source>/tests/*.jsonl`. Each line is `{"send": <body>}` or `{"expect": <subset of the reply body>}` (optional `"to"`). Without test files, the candidate must start and stay up for 2 seconds. If a new version crashes 3 times within 30 seconds, the broker rolls it back automatically. The broker keeps the message that crashed an actor and delivers it again after the next upgrade.
|
||||||
|
|
||||||
|
`cargo` (built-in Rust compiler):
|
||||||
|
- `{"type":"build","source":"<folder with Cargo.toml>"}` → `{"type":"built","wasm":"/actor/cargo/public/builds/<n>/<name>.wasm",...}` or `{"type":"build_failed","errors":"..."}`. A build can take a minute. Wait up to 600 seconds for the reply.
|
||||||
|
|
||||||
|
# Writing an actor in Rust
|
||||||
|
|
||||||
|
1. Write the crate to `/public/drafts/<name>/`: `Cargo.toml`, `src/main.rs`, and `tests/*.jsonl`. Use `edition = "2024"`. Use `serde_json` for messages.
|
||||||
|
2. Send `build` to `cargo` with the folder path `/actor/agent/public/drafts/<name>` and wait for the reply. Fix compiler errors and build again.
|
||||||
|
3. Send `spawn` (new actor) or `upgrade` (existing actor) to `broker` with the `wasm` path from the build reply and the same `source` folder. Wait for the reply.
|
||||||
|
4. Send a message to the new actor to check it.
|
||||||
|
|
||||||
|
A simple actor that only reacts to messages can block on stdin:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut out = std::io::stdout().lock();
|
||||||
|
for line in std::io::stdin().lock().lines() {
|
||||||
|
let Ok(line) = line else { break };
|
||||||
|
let Ok(msg) = serde_json::from_str::<Value>(&line) else { continue };
|
||||||
|
let body = &msg["body"];
|
||||||
|
let reply = match body["type"].as_str() {
|
||||||
|
Some("shutdown") => break,
|
||||||
|
Some("ping") => json!({"type": "pong"}),
|
||||||
|
_ => json!({"type": "error", "reason": "unknown message"}),
|
||||||
|
};
|
||||||
|
writeln!(out, "{}", json!({"to": msg["from"], "reply_to": msg["id"], "body": reply})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
An actor that needs HTTP or timers and must also read messages uses the async `wstd` crate (`wstd = "0.6"`): `#[wstd::main] async fn main()`, `wstd::io::stdin()` (`AsyncRead`), `wstd::io::stdout()` (`AsyncWrite`, `write_all`, `flush`), `wstd::http::{Client, Request, Body}` (`Client::new().send(Request::get(url).body(Body::empty())?).await?`, then `response.into_body().contents().await?`), `wstd::task::sleep(wstd::time::Duration::from_secs(n)).await`, and `wstd::runtime::spawn(future)`. Actors have outgoing HTTP and HTTPS only, no raw sockets. Threads are not available. Store state in `/data` so it survives restarts and upgrades. A new version must read the data format of the old version.
|
||||||
|
|
||||||
|
# Changing yourself
|
||||||
|
|
||||||
|
Your own source is at `/actor/agent/source`. To change yourself, copy it to `/public/drafts/agent`, edit the copy, build it, and upgrade `agent`. The upgrade restarts you, and the conversation continues from `/data`. Be careful: if the new version cannot answer, the person must roll you back through the console. Change yourself only when the person asks for it.
|
||||||
|
|
||||||
|
# How to work
|
||||||
|
|
||||||
|
- Use tools to look at the real state before you claim anything. Read files instead of guessing.
|
||||||
|
- Keep answers short and concrete. Say what you changed and whether it worked.
|
||||||
|
- Your final text in a turn goes back to the person who wrote to you.
|
||||||
|
|||||||
@@ -1,138 +0,0 @@
|
|||||||
//! Tool descriptions and file operations for the agent.
|
|
||||||
|
|
||||||
use crate::llm::Tool;
|
|
||||||
use sausage_actor::{Value, json};
|
|
||||||
use std::io;
|
|
||||||
|
|
||||||
pub fn definitions() -> Vec<Tool> {
|
|
||||||
vec![
|
|
||||||
Tool {
|
|
||||||
name: "send_message",
|
|
||||||
description: "Send an actor a JSON body with a `type`. `broker` and `cargo` are built-in actors; see `/actor/broker/public/PROTOCOL.md`. With `wait_seconds`, the reply's `reply_to` matches the returned ID. Returns the ID, reply, and other messages received.",
|
|
||||||
parameters: schema(
|
|
||||||
json!({
|
|
||||||
"to": {"type": "string", "description": "Actor ID"},
|
|
||||||
"body": {"type": "object", "description": "JSON object with a type field"},
|
|
||||||
"wait_seconds": {"type": "number", "description": "Seconds to wait for a reply. Default 0."}
|
|
||||||
}),
|
|
||||||
&["to", "body"],
|
|
||||||
),
|
|
||||||
},
|
|
||||||
Tool {
|
|
||||||
name: "read_file",
|
|
||||||
description: "Read a text file under `/data`, `/public`, or `/actor/<id>`. Reads all bytes by default; `offset` and `max_bytes` are optional.",
|
|
||||||
parameters: schema(
|
|
||||||
json!({
|
|
||||||
"path": {"type": "string"},
|
|
||||||
"offset": {"type": "integer", "description": "Byte offset (default 0)"},
|
|
||||||
"max_bytes": {"type": "integer", "description": "Optional maximum number of bytes to read"}
|
|
||||||
}),
|
|
||||||
&["path"],
|
|
||||||
),
|
|
||||||
},
|
|
||||||
Tool {
|
|
||||||
name: "write_file",
|
|
||||||
description: "Write text under `/data` or `/public`. Parent folders are created.",
|
|
||||||
parameters: schema(
|
|
||||||
json!({"path": {"type": "string"}, "content": {"type": "string"}}),
|
|
||||||
&["path", "content"],
|
|
||||||
),
|
|
||||||
},
|
|
||||||
Tool {
|
|
||||||
name: "list_dir",
|
|
||||||
description: "List entries in a directory under `/data`, `/public`, or `/actor/<id>`. Returns each name, type, and size.",
|
|
||||||
parameters: schema(json!({"path": {"type": "string"}}), &["path"]),
|
|
||||||
},
|
|
||||||
Tool {
|
|
||||||
name: "copy_dir",
|
|
||||||
description: "Copy a directory tree, for example `/actor/<id>/source` to `/public/drafts/<id>`. Skips `target/` folders.",
|
|
||||||
parameters: schema(
|
|
||||||
json!({"from": {"type": "string"}, "to": {"type": "string"}}),
|
|
||||||
&["from", "to"],
|
|
||||||
),
|
|
||||||
},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
fn schema(properties: Value, required: &[&str]) -> Value {
|
|
||||||
json!({"type": "object", "properties": properties, "required": required})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn run(name: &str, args: &Value) -> io::Result<Value> {
|
|
||||||
match name {
|
|
||||||
"read_file" => read_file(args),
|
|
||||||
"write_file" => write_file(args),
|
|
||||||
"list_dir" => list_dir(args),
|
|
||||||
"copy_dir" => copy_dir(args),
|
|
||||||
other => Ok(json!({"error": format!("unknown tool {other}")})),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_file(args: &Value) -> io::Result<Value> {
|
|
||||||
let path = args["path"].as_str().unwrap_or_default();
|
|
||||||
let bytes = std::fs::read(path)?;
|
|
||||||
let offset = (args["offset"].as_u64().unwrap_or(0) as usize).min(bytes.len());
|
|
||||||
let end = args["max_bytes"].as_u64().map_or(bytes.len(), |max| {
|
|
||||||
offset.saturating_add(max as usize).min(bytes.len())
|
|
||||||
});
|
|
||||||
Ok(json!({
|
|
||||||
"path": path,
|
|
||||||
"size": bytes.len(),
|
|
||||||
"offset": offset,
|
|
||||||
"content": String::from_utf8_lossy(&bytes[offset..end]),
|
|
||||||
"truncated": end < bytes.len(),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_file(args: &Value) -> io::Result<Value> {
|
|
||||||
let path = args["path"].as_str().unwrap_or_default();
|
|
||||||
if let Some(parent) = std::path::Path::new(path).parent() {
|
|
||||||
let _ = std::fs::create_dir_all(parent);
|
|
||||||
}
|
|
||||||
std::fs::write(path, args["content"].as_str().unwrap_or_default())?;
|
|
||||||
Ok(json!({"written": path}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn list_dir(args: &Value) -> io::Result<Value> {
|
|
||||||
let mut entries: Vec<Value> = std::fs::read_dir(args["path"].as_str().unwrap_or_default())?
|
|
||||||
.filter_map(|entry| entry.ok())
|
|
||||||
.map(|entry| {
|
|
||||||
let metadata = entry.metadata().ok();
|
|
||||||
json!({
|
|
||||||
"name": entry.file_name().to_string_lossy(),
|
|
||||||
"type": if metadata.as_ref().is_some_and(|meta| meta.is_dir()) { "dir" } else { "file" },
|
|
||||||
"size": metadata.map(|meta| meta.len()),
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
entries.sort_by(|left, right| left["name"].as_str().cmp(&right["name"].as_str()));
|
|
||||||
Ok(json!({"entries": entries}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn copy_dir(args: &Value) -> io::Result<Value> {
|
|
||||||
let count = copy_tree(
|
|
||||||
args["from"].as_str().unwrap_or_default(),
|
|
||||||
args["to"].as_str().unwrap_or_default(),
|
|
||||||
)?;
|
|
||||||
Ok(json!({"copied_files": count}))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn copy_tree(from: &str, to: &str) -> io::Result<u64> {
|
|
||||||
std::fs::create_dir_all(to)?;
|
|
||||||
let mut count = 0;
|
|
||||||
for entry in std::fs::read_dir(from)? {
|
|
||||||
let entry = entry?;
|
|
||||||
let name = entry.file_name().to_string_lossy().to_string();
|
|
||||||
let src = format!("{from}/{name}");
|
|
||||||
let dst = format!("{to}/{name}");
|
|
||||||
if entry.file_type()?.is_dir() {
|
|
||||||
if name != "target" {
|
|
||||||
count += copy_tree(&src, &dst)?;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
std::fs::copy(src, dst)?;
|
|
||||||
count += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(count)
|
|
||||||
}
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
{
|
|
||||||
"description": "the agent answers text with text, even without config or network",
|
|
||||||
"steps": [
|
|
||||||
{"send": {"type": "text", "text": "Hello"}, "expect": {"type": "text"}}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// The agent answers a text message with text, even without config or network.
|
||||||
|
{"send": {"type": "text", "text": "Hello"}}
|
||||||
|
{"expect": {"type": "text"}}
|
||||||
@@ -15,11 +15,9 @@ clap = { version = "4", features = ["derive"] }
|
|||||||
http = "1"
|
http = "1"
|
||||||
http-body-util = "0.1"
|
http-body-util = "0.1"
|
||||||
include_dir = "0.7"
|
include_dir = "0.7"
|
||||||
notify = "8"
|
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
toml_edit = "0.25"
|
|
||||||
wasmtime = "49"
|
wasmtime = "49"
|
||||||
wasmtime-wasi = "49"
|
wasmtime-wasi = "49"
|
||||||
wasmtime-wasi-http = "49"
|
wasmtime-wasi-http = "49"
|
||||||
|
|||||||
+621
-429
File diff suppressed because it is too large
Load Diff
+109
-77
@@ -1,25 +1,18 @@
|
|||||||
//! Tests for a candidate version, before the broker installs it.
|
//! Tests for a candidate version, before the broker installs it.
|
||||||
//!
|
//!
|
||||||
//! A test is a JSON file in `source/tests/*.json`, one scenario per file:
|
//! A test is a message transcript in `source/tests/*.jsonl`. Each line is one
|
||||||
|
//! of:
|
||||||
//!
|
//!
|
||||||
//! ```json
|
//! ```text
|
||||||
//! {
|
//! {"send": {...body...}} send a message to the candidate
|
||||||
//! "description": "add, then list",
|
//! {"send": {...}, "from": "someone"} with an explicit sender
|
||||||
//! "steps": [
|
//! {"expect": {...subset of body...}} the next message from the candidate
|
||||||
//! {"send": {"type": "add", "text": "milk"}, "expect": {"type": "added"}},
|
//! {"expect": {...}, "to": "someone"} and its recipient
|
||||||
//! {"send": {"type": "list"}, "from": "someone", "expect": [{"type": "notes"}], "to": "someone"}
|
|
||||||
//! ]
|
|
||||||
//! }
|
|
||||||
//! ```
|
//! ```
|
||||||
//!
|
//!
|
||||||
//! Each step sends `send` (optional, from `from` or "test"), then reads one
|
|
||||||
//! message for each entry of `expect` (an object or a list). An expected body
|
|
||||||
//! matches if it is a subset of the actual body. With `to`, each message must
|
|
||||||
//! also go to that recipient.
|
|
||||||
//!
|
|
||||||
//! Each file runs in a fresh instance on a copy of the actor's `/data` and
|
//! Each file runs in a fresh instance on a copy of the actor's `/data` and
|
||||||
//! `/public` (empty folders for a new actor), without network. A version
|
//! `/public`. The candidate has no HTTP access. Without test files, the
|
||||||
//! without test files passes.
|
//! candidate must start and stay up for `SMOKE_TIME`.
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -29,9 +22,11 @@ use serde_json::{Value, json};
|
|||||||
use wasmtime::component::Component;
|
use wasmtime::component::Component;
|
||||||
|
|
||||||
use crate::broker::{Broker, SHUTDOWN_DEADLINE, shutdown_line, truncate};
|
use crate::broker::{Broker, SHUTDOWN_DEADLINE, shutdown_line, truncate};
|
||||||
use crate::runtime::Instance;
|
use crate::fs::Mount;
|
||||||
|
use crate::runtime::{Instance, InstanceConfig};
|
||||||
use crate::world;
|
use crate::world;
|
||||||
|
|
||||||
|
const SMOKE_TIME: Duration = Duration::from_secs(2);
|
||||||
const EXPECT_TIMEOUT: Duration = Duration::from_secs(10);
|
const EXPECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
/// `expected` matches `actual` if every key of an expected object exists in
|
/// `expected` matches `actual` if every key of an expected object exists in
|
||||||
@@ -64,93 +59,130 @@ fn stderr_tail(path: &Path) -> String {
|
|||||||
text[s..].to_string()
|
text[s..].to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The version under test, which starts from a copy of the actor's state.
|
pub async fn test(
|
||||||
struct Candidate<'a> {
|
broker: &Broker,
|
||||||
broker: &'a Broker,
|
id: &str,
|
||||||
id: &'a str,
|
component: &Component,
|
||||||
version: u64,
|
source: &Path,
|
||||||
component: &'a Component,
|
data: Option<&Path>,
|
||||||
}
|
public: Option<&Path>,
|
||||||
|
) -> Result<Value> {
|
||||||
/// Run every test file of `<source>/tests/`.
|
|
||||||
pub async fn test(broker: &Broker, id: &str, version: u64, component: &Component, source: Option<&Path>) -> Result<Value> {
|
|
||||||
let candidate = Candidate { broker, id, version, component };
|
|
||||||
let Some(source) = source else { return Ok(json!({"passed": []})) };
|
|
||||||
let mut files: Vec<_> = std::fs::read_dir(source.join("tests"))
|
let mut files: Vec<_> = std::fs::read_dir(source.join("tests"))
|
||||||
.map(|rd| rd.filter_map(|e| Some(e.ok()?.path())).filter(|p| p.extension().is_some_and(|x| x == "json")).collect())
|
.map(|rd| {
|
||||||
|
rd.filter_map(|e| e.ok())
|
||||||
|
.map(|e| e.path())
|
||||||
|
.filter(|p| p.extension().is_some_and(|x| x == "jsonl"))
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
files.sort();
|
files.sort();
|
||||||
|
|
||||||
|
if files.is_empty() {
|
||||||
|
let (mut inst, scratch) = start(broker, id, component, data, public)?;
|
||||||
|
tokio::time::sleep(SMOKE_TIME).await;
|
||||||
|
if inst.is_finished() {
|
||||||
|
let exit = inst.wait().await;
|
||||||
|
bail!(
|
||||||
|
"candidate ended during the smoke test ({exit}). stderr:\n{}",
|
||||||
|
stderr_tail(&scratch.dir.join("stderr.log"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
|
||||||
|
return Ok(json!({"smoke_test": "passed", "files": 0}));
|
||||||
|
}
|
||||||
|
|
||||||
let mut passed = Vec::new();
|
let mut passed = Vec::new();
|
||||||
for file in &files {
|
for file in &files {
|
||||||
let name = file.file_name().unwrap().to_string_lossy().to_string();
|
let name = file.file_name().unwrap().to_string_lossy().to_string();
|
||||||
candidate.run_file(file).await.map_err(|e| anyhow!("test {name} failed: {e:#}"))?;
|
run_file(broker, id, component, data, public, file)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow!("test {name} failed: {e:#}"))?;
|
||||||
passed.push(name);
|
passed.push(name);
|
||||||
}
|
}
|
||||||
Ok(json!({"passed": passed}))
|
Ok(json!({"passed": passed}))
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Candidate<'_> {
|
fn start(broker: &Broker, id: &str, component: &Component, data: Option<&Path>, public: Option<&Path>) -> Result<(Instance, Scratch)> {
|
||||||
/// Start an instance on a fresh copy of the state, without network.
|
let dir = broker.world.scratch_dir().join(format!("{id}-{}-{}", world::now(), rand_suffix()));
|
||||||
fn start(&self) -> Result<(Instance, Scratch)> {
|
|
||||||
static N: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
|
||||||
let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
let dir = self.broker.world.scratch_dir().join(format!("{}-{}-{n}", self.id, std::process::id()));
|
|
||||||
let scratch = Scratch { dir: dir.clone() };
|
let scratch = Scratch { dir: dir.clone() };
|
||||||
let world = &self.broker.world;
|
std::fs::create_dir_all(dir.join("data"))?;
|
||||||
for (from, name) in [(world.data_dir(self.id), "data"), (world.public_dir(self.id), "public")] {
|
std::fs::create_dir_all(dir.join("public"))?;
|
||||||
std::fs::create_dir_all(dir.join(name))?;
|
if let Some(d) = data.filter(|d| d.is_dir()) {
|
||||||
if from.is_dir() {
|
world::copy_tree(d, &dir.join("data"))?;
|
||||||
world::copy_tree(&from, &dir.join(name))?;
|
|
||||||
}
|
}
|
||||||
|
if let Some(p) = public.filter(|p| p.is_dir()) {
|
||||||
|
world::copy_tree(p, &dir.join("public"))?;
|
||||||
}
|
}
|
||||||
let cfg = self.broker.instance_config(self.id, self.version, Some(&dir));
|
let cfg = InstanceConfig {
|
||||||
Ok((self.broker.runtime.start(self.component, cfg)?, scratch))
|
mounts: vec![
|
||||||
|
Mount { guest: "/actor".into(), host: broker.world.actors_root(), writable: false },
|
||||||
|
Mount { guest: "/public".into(), host: dir.join("public"), writable: true },
|
||||||
|
Mount { guest: "/data".into(), host: dir.join("data"), writable: true },
|
||||||
|
],
|
||||||
|
allow_http: false,
|
||||||
|
stderr_log: dir.join("stderr.log"),
|
||||||
|
env: vec![("SAUSAGE_ACTOR".into(), id.to_string()), ("SAUSAGE_CANDIDATE".into(), "1".into())],
|
||||||
|
};
|
||||||
|
Ok((broker.runtime.start(component, cfg)?, scratch))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_file(&self, file: &Path) -> Result<()> {
|
fn rand_suffix() -> u32 {
|
||||||
let test: Value = serde_json::from_str(&std::fs::read_to_string(file)?).map_err(|e| anyhow!("invalid JSON: {e}"))?;
|
use std::hash::{BuildHasher, Hasher};
|
||||||
let steps = test["steps"].as_array().ok_or_else(|| anyhow!("missing \"steps\" list"))?;
|
let mut h = std::collections::hash_map::RandomState::new().build_hasher();
|
||||||
let (mut inst, scratch) = self.start()?;
|
h.write_u64(std::process::id() as u64);
|
||||||
let stderr = scratch.dir.join("stderr.log");
|
h.finish() as u32
|
||||||
// Lines that the candidate wrote while a step was sent.
|
|
||||||
let mut early = std::collections::VecDeque::new();
|
|
||||||
for (n, step) in steps.iter().enumerate() {
|
|
||||||
let n = n + 1;
|
|
||||||
if let Some(body) = step.get("send") {
|
|
||||||
let from = step["from"].as_str().unwrap_or("test");
|
|
||||||
let env = json!({"id": format!("test/{n}"), "from": from, "body": body});
|
|
||||||
inst.send_line(&env.to_string(), |line| early.push_back(line))
|
|
||||||
.await
|
|
||||||
.map_err(|e| anyhow!("step {n}: candidate does not read stdin: {e}"))?;
|
|
||||||
}
|
}
|
||||||
let expected = match step.get("expect") {
|
|
||||||
None => vec![],
|
async fn run_file(
|
||||||
Some(Value::Array(list)) => list.clone(),
|
broker: &Broker,
|
||||||
Some(one) => vec![one.clone()],
|
id: &str,
|
||||||
};
|
component: &Component,
|
||||||
for expected in expected {
|
data: Option<&Path>,
|
||||||
let next = async { if early.is_empty() { inst.stdout.recv().await } else { early.pop_front() } };
|
public: Option<&Path>,
|
||||||
let line = match tokio::time::timeout(EXPECT_TIMEOUT, next).await {
|
file: &Path,
|
||||||
Err(_) => bail!("step {n}: no message within {EXPECT_TIMEOUT:?}; expected {expected}"),
|
) -> Result<()> {
|
||||||
|
let text = std::fs::read_to_string(file)?;
|
||||||
|
let (mut inst, scratch) = start(broker, id, component, data, public)?;
|
||||||
|
let stderr = scratch.dir.join("stderr.log");
|
||||||
|
let mut sent = 0u32;
|
||||||
|
for (n, line) in text.lines().enumerate() {
|
||||||
|
let line = line.trim();
|
||||||
|
if line.is_empty() || line.starts_with("//") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let step: Value = serde_json::from_str(line).map_err(|e| anyhow!("line {}: invalid JSON: {e}", n + 1))?;
|
||||||
|
if let Some(body) = step.get("send") {
|
||||||
|
sent += 1;
|
||||||
|
let from = step.get("from").and_then(Value::as_str).unwrap_or("test");
|
||||||
|
let env = json!({"id": format!("test/{sent}"), "from": from, "body": body});
|
||||||
|
inst.send_line(&env.to_string())
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow!("line {}: candidate does not read stdin: {e}", n + 1))?;
|
||||||
|
} else if let Some(expected) = step.get("expect") {
|
||||||
|
let got = tokio::time::timeout(EXPECT_TIMEOUT, inst.stdout.recv()).await;
|
||||||
|
let got = match got {
|
||||||
|
Err(_) => bail!("line {}: no message within {EXPECT_TIMEOUT:?}; expected {expected}", n + 1),
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
let exit = inst.wait().await;
|
let exit = inst.wait().await;
|
||||||
bail!("step {n}: candidate ended ({exit}); expected {expected}. stderr:\n{}", stderr_tail(&stderr))
|
bail!("line {}: candidate ended ({exit}); expected {expected}. stderr:\n{}", n + 1, stderr_tail(&stderr))
|
||||||
}
|
}
|
||||||
Ok(Some(l)) => l,
|
Ok(Some(l)) => l,
|
||||||
};
|
};
|
||||||
let msg: Value = serde_json::from_str(&line)
|
let msg: Value = serde_json::from_str(&got)
|
||||||
.map_err(|_| anyhow!("step {n}: candidate wrote a non-JSON line: {}", truncate(&line, 300)))?;
|
.map_err(|_| anyhow!("line {}: candidate wrote a non-JSON line: {}", n + 1, truncate(&got, 300)))?;
|
||||||
if !subset_match(&expected, &msg["body"]) {
|
let body = msg.get("body").cloned().unwrap_or(Value::Null);
|
||||||
bail!("step {n}: expected body {expected}, got {}", truncate(&msg["body"].to_string(), 1000));
|
if !subset_match(expected, &body) {
|
||||||
|
bail!("line {}: expected body {expected}, got {}", n + 1, truncate(&body.to_string(), 1000));
|
||||||
}
|
}
|
||||||
if let Some(to) = step.get("to")
|
if let Some(to) = step.get("to")
|
||||||
&& msg.get("to") != Some(to)
|
&& msg.get("to") != Some(to)
|
||||||
{
|
{
|
||||||
bail!("step {n}: expected recipient {to}, got {}", msg.get("to").unwrap_or(&Value::Null));
|
bail!("line {}: expected recipient {to}, got {}", n + 1, msg.get("to").cloned().unwrap_or(Value::Null));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
bail!("line {}: use \"send\" or \"expect\"", n + 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE).await;
|
||||||
inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |_| {}).await;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|||||||
+48
-207
@@ -1,133 +1,65 @@
|
|||||||
//! The native `cargo` actor: builds Rust source folders into `.wasm` files.
|
//! The native `cargo` actor: builds Rust source folders into `.wasm` files.
|
||||||
//!
|
//!
|
||||||
//! Request: `{"type": "build", "source": "/actor/<id>/public/<folder>"}`
|
//! Request: `{"type": "build", "source": "/actor/<id>/public/<folder>"}`
|
||||||
//! optional `"skip_tests": true` skips native `cargo test` (tests run by default).
|
//! Replies: `{"type": "built", "wasm": "/actor/cargo/public/builds/<n>/<name>.wasm", ...}`
|
||||||
//! Replies: `{"type": "built", "wasm": "/actor/cargo/public/builds/<n>/<name>.wasm", "tests": {"passed": 1, "failed": 0, "output": "..."}, ...}`
|
//! `{"type": "build_failed", "errors": "..."}`
|
||||||
//! `{"type": "build_failed", "errors": "...", "tests": {"passed": 0, "failed": 1, "output": "..."}}`
|
|
||||||
//!
|
//!
|
||||||
//! The broker knows nothing about Rust. This actor is one way to get a
|
//! The broker knows nothing about Rust. This actor is one way to get a
|
||||||
//! `.wasm` file. Other actors can add other languages.
|
//! `.wasm` file. Other actors can add other languages.
|
||||||
//!
|
|
||||||
//! Actors depend on the SDK with `sausage-actor = "0.1"`. The SDK is not
|
|
||||||
//! published: this actor copies the SDK of this host to `public/sdk/` (so
|
|
||||||
//! that every actor can read it), and points the dependency there in the
|
|
||||||
//! build copy of `Cargo.toml`.
|
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use anyhow::{Context, Result, anyhow, bail};
|
use anyhow::{Context, Result, anyhow, bail};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::{Mutex, mpsc};
|
||||||
|
|
||||||
use include_dir::{Dir, include_dir};
|
use crate::broker::{Broker, Envelope, truncate};
|
||||||
|
|
||||||
use crate::broker::{Broker, Envelope};
|
|
||||||
use crate::world::{self, World};
|
use crate::world::{self, World};
|
||||||
|
|
||||||
pub const ID: &str = "cargo";
|
pub const ID: &str = "cargo";
|
||||||
pub const TARGET: &str = "wasm32-wasip2";
|
pub const TARGET: &str = "wasm32-wasip2";
|
||||||
|
const BUILD_TIMEOUT: Duration = Duration::from_secs(900);
|
||||||
/// The source of the `sausage-actor` SDK.
|
|
||||||
static SDK: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../sdk");
|
|
||||||
|
|
||||||
#[derive(Clone, Default, Debug)]
|
|
||||||
pub struct TestResults {
|
|
||||||
pub passed: u64,
|
|
||||||
pub failed: u64,
|
|
||||||
pub output: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TestResults {
|
|
||||||
fn value(&self) -> Value {
|
|
||||||
json!({"passed": self.passed, "failed": self.failed, "output": self.output})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct BuildOutput {
|
pub struct BuildOutput {
|
||||||
/// Host path of the published `.wasm` file.
|
/// Host path of the published `.wasm` file.
|
||||||
pub wasm: PathBuf,
|
pub wasm: PathBuf,
|
||||||
pub seconds: f64,
|
pub seconds: f64,
|
||||||
pub warnings: String,
|
pub warnings: String,
|
||||||
pub tests: TestResults,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub struct BuildFailure {
|
|
||||||
pub reason: String,
|
|
||||||
pub tests: TestResults,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for BuildFailure {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
if self.tests.failed > 0 && !self.tests.output.is_empty() {
|
|
||||||
write!(f, "{}:\n{}", self.reason, self.tests.output)
|
|
||||||
} else {
|
|
||||||
f.write_str(&self.reason)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::error::Error for BuildFailure {}
|
|
||||||
|
|
||||||
impl From<anyhow::Error> for BuildFailure {
|
|
||||||
fn from(error: anyhow::Error) -> Self {
|
|
||||||
Self { reason: format!("{error:#}"), tests: TestResults::default() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// All builds share one target folder, so they run one at a time.
|
|
||||||
pub struct Cargo {
|
pub struct Cargo {
|
||||||
world: World,
|
world: World,
|
||||||
|
/// One build at a time: all builds share one target folder.
|
||||||
|
lock: Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Cargo {
|
impl Cargo {
|
||||||
/// Create the actor and install the SDK of this host into `public/sdk/`.
|
pub fn new(world: World) -> Self {
|
||||||
pub fn new(world: World) -> Result<Self> {
|
Self { world, lock: Mutex::new(()) }
|
||||||
write_dir(&SDK, &world.public_dir(ID).join("sdk"))?;
|
|
||||||
Ok(Self { world })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run native `cargo test` unless opted out, then build and publish the
|
/// Build the crate in `source` (a host path) and publish the `.wasm` file
|
||||||
/// crate's `wasm32-wasip2` binary under `/actor/cargo/public/builds/<n>/`.
|
/// under `/actor/cargo/public/builds/<n>/`.
|
||||||
pub async fn build(&self, source: &Path, skip_tests: bool) -> std::result::Result<BuildOutput, BuildFailure> {
|
pub async fn build(&self, source: &Path) -> Result<BuildOutput> {
|
||||||
if !source.join("Cargo.toml").is_file() {
|
if !source.join("Cargo.toml").is_file() {
|
||||||
return Err(BuildFailure {
|
bail!("{} has no Cargo.toml", source.display());
|
||||||
reason: format!("{} has no Cargo.toml", source.display()),
|
|
||||||
tests: TestResults::default(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
let _guard = self.lock.lock().await;
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
let data = self.world.data_dir(ID);
|
let data = self.world.data_dir(ID);
|
||||||
let build_dir = data.join("work");
|
let key = format!("{:016x}", fxhash(source.to_string_lossy().as_bytes()));
|
||||||
world::replace_source(source, &build_dir)?;
|
let build_dir = data.join("work").join(&key);
|
||||||
|
world::replace_tree(source, &build_dir)?;
|
||||||
|
// The crate must not join a parent workspace by accident.
|
||||||
let manifest = build_dir.join("Cargo.toml");
|
let manifest = build_dir.join("Cargo.toml");
|
||||||
let original = std::fs::read_to_string(&manifest).context("read build manifest")?;
|
let text = std::fs::read_to_string(&manifest)?;
|
||||||
let prepared = prepare_manifest(&original, Some(&self.world.public_dir(ID).join("sdk")))?;
|
if !text.contains("[workspace]") {
|
||||||
std::fs::write(&manifest, prepared).context("write prepared manifest")?;
|
std::fs::write(&manifest, format!("{text}\n[workspace]\n"))?;
|
||||||
|
|
||||||
let target_dir = data.join("target");
|
|
||||||
let tests = if skip_tests {
|
|
||||||
TestResults { output: "skipped (skip_tests=true)".into(), ..TestResults::default() }
|
|
||||||
} else {
|
|
||||||
match run_tests(&build_dir, &target_dir).await {
|
|
||||||
Ok(tests) => tests,
|
|
||||||
Err(error) => {
|
|
||||||
let output = format!("cannot run cargo tests: {error:#}");
|
|
||||||
return Err(BuildFailure {
|
|
||||||
reason: output.clone(),
|
|
||||||
tests: TestResults { failed: 1, output, ..TestResults::default() },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if tests.failed > 0 {
|
|
||||||
return Err(BuildFailure { reason: "unit tests failed".into(), tests });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let build_result: Result<BuildOutput> = async {
|
|
||||||
let mut cmd = Command::new("cargo");
|
let mut cmd = Command::new("cargo");
|
||||||
cmd.arg("build")
|
cmd.arg("build")
|
||||||
.arg("--release")
|
.arg("--release")
|
||||||
@@ -135,13 +67,16 @@ impl Cargo {
|
|||||||
.arg(TARGET)
|
.arg(TARGET)
|
||||||
.arg("--message-format=json-render-diagnostics")
|
.arg("--message-format=json-render-diagnostics")
|
||||||
.arg("--target-dir")
|
.arg("--target-dir")
|
||||||
.arg(&target_dir)
|
.arg(data.join("target"))
|
||||||
.current_dir(&build_dir)
|
.current_dir(&build_dir)
|
||||||
.kill_on_drop(true);
|
.kill_on_drop(true);
|
||||||
let out = cmd.output().await.context("cannot run cargo (is it on PATH, with the wasm32-wasip2 target?)")?;
|
let out = tokio::time::timeout(BUILD_TIMEOUT, cmd.output())
|
||||||
|
.await
|
||||||
|
.map_err(|_| anyhow!("cargo build took longer than {BUILD_TIMEOUT:?}"))?
|
||||||
|
.context("cannot run cargo (is it on PATH, with the wasm32-wasip2 target?)")?;
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
|
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
|
||||||
if !out.status.success() {
|
if !out.status.success() {
|
||||||
bail!("{}", strip_progress(&stderr));
|
bail!("{}", truncate(&strip_progress(&stderr), 20_000));
|
||||||
}
|
}
|
||||||
let mut artifact = None;
|
let mut artifact = None;
|
||||||
for line in String::from_utf8_lossy(&out.stdout).lines() {
|
for line in String::from_utf8_lossy(&out.stdout).lines() {
|
||||||
@@ -159,28 +94,19 @@ impl Cargo {
|
|||||||
let name = artifact.file_name().context("artifact has no file name")?;
|
let name = artifact.file_name().context("artifact has no file name")?;
|
||||||
let dest = dest_dir.join(name);
|
let dest = dest_dir.join(name);
|
||||||
world::write_atomic(&dest, &std::fs::read(&artifact)?)?;
|
world::write_atomic(&dest, &std::fs::read(&artifact)?)?;
|
||||||
Ok(BuildOutput {
|
Ok(BuildOutput { wasm: dest, seconds: started.elapsed().as_secs_f64(), warnings: strip_progress(&stderr) })
|
||||||
wasm: dest,
|
|
||||||
seconds: started.elapsed().as_secs_f64(),
|
|
||||||
warnings: strip_progress(&stderr),
|
|
||||||
tests: tests.clone(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
.await;
|
|
||||||
build_result.map_err(|error| BuildFailure { reason: format!("{error:#}"), tests })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The native `cargo` actor. Builds run one at a time, in order.
|
pub async fn run_actor(self: Arc<Self>, broker: Arc<Broker>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||||
/// Replies (messages with `reply_to`) get no answer.
|
|
||||||
pub async fn run_actor(self, broker: Arc<Broker>, mut mailbox: mpsc::Receiver<Envelope>) {
|
|
||||||
while let Some(env) = mailbox.recv().await {
|
while let Some(env) = mailbox.recv().await {
|
||||||
if env.reply_to.is_some() {
|
let this = self.clone();
|
||||||
continue;
|
let broker = broker.clone();
|
||||||
}
|
tokio::spawn(async move {
|
||||||
let body = self.handle(&broker, &env).await.unwrap_or_else(|e| {
|
let body = this.handle(&broker, &env).await.unwrap_or_else(|e| {
|
||||||
json!({"type": "build_failed", "source": env.body.get("source"), "errors": format!("{e:#}"), "tests": TestResults::default().value()})
|
json!({"type": "build_failed", "source": env.body.get("source"), "errors": format!("{e:#}")})
|
||||||
});
|
});
|
||||||
broker.send(ID, Some(&env.from), None, Some(&env.id), body);
|
broker.send(ID, Some(&env.from), None, Some(&env.id), body);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,9 +115,7 @@ impl Cargo {
|
|||||||
Some("build") => {
|
Some("build") => {
|
||||||
let source_s = env.body.get("source").and_then(Value::as_str).ok_or_else(|| anyhow!("missing \"source\""))?;
|
let source_s = env.body.get("source").and_then(Value::as_str).ok_or_else(|| anyhow!("missing \"source\""))?;
|
||||||
let source = broker.world.resolve(&env.from, source_s)?;
|
let source = broker.world.resolve(&env.from, source_s)?;
|
||||||
let skip_tests = env.body.get("skip_tests").and_then(Value::as_bool).unwrap_or(false);
|
let out = self.build(&source).await?;
|
||||||
match self.build(&source, skip_tests).await {
|
|
||||||
Ok(out) => {
|
|
||||||
let wasm = self.world.global_path(&out.wasm).unwrap_or_else(|| out.wasm.display().to_string());
|
let wasm = self.world.global_path(&out.wasm).unwrap_or_else(|| out.wasm.display().to_string());
|
||||||
Ok(json!({
|
Ok(json!({
|
||||||
"type": "built",
|
"type": "built",
|
||||||
@@ -199,107 +123,15 @@ impl Cargo {
|
|||||||
"source": source_s,
|
"source": source_s,
|
||||||
"built_by": ID,
|
"built_by": ID,
|
||||||
"seconds": (out.seconds * 10.0).round() / 10.0,
|
"seconds": (out.seconds * 10.0).round() / 10.0,
|
||||||
"warnings": out.warnings,
|
"warnings": truncate(&out.warnings, 4000),
|
||||||
"tests": out.tests.value(),
|
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
Err(failure) => Ok(json!({
|
Some("shutdown") => Ok(json!({"type": "ignored"})),
|
||||||
"type": "build_failed",
|
|
||||||
"source": source_s,
|
|
||||||
"errors": failure.reason,
|
|
||||||
"tests": failure.tests.value(),
|
|
||||||
})),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
other => bail!("unknown request {other:?}. Known: build"),
|
other => bail!("unknown request {other:?}. Known: build"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run tests for the host triple: WASI crates' pure-logic tests can use the
|
|
||||||
/// normal Rust test harness without needing a WASI test runner.
|
|
||||||
async fn run_tests(build_dir: &Path, target_dir: &Path) -> Result<TestResults> {
|
|
||||||
let mut cmd = Command::new("cargo");
|
|
||||||
cmd.arg("test")
|
|
||||||
.arg("--target-dir")
|
|
||||||
.arg(target_dir)
|
|
||||||
.current_dir(build_dir)
|
|
||||||
.kill_on_drop(true);
|
|
||||||
let out = cmd.output().await.context("cannot run cargo test")?;
|
|
||||||
let stdout = String::from_utf8_lossy(&out.stdout).to_string();
|
|
||||||
let stderr = String::from_utf8_lossy(&out.stderr).to_string();
|
|
||||||
let combined = match (stdout.is_empty(), stderr.is_empty()) {
|
|
||||||
(true, true) => String::new(),
|
|
||||||
(false, true) => stdout,
|
|
||||||
(true, false) => stderr,
|
|
||||||
(false, false) => format!("{stdout}\n{stderr}"),
|
|
||||||
};
|
|
||||||
let output = strip_progress(&combined);
|
|
||||||
let passed = test_count(&output, "passed");
|
|
||||||
let mut failed = test_count(&output, "failed");
|
|
||||||
// Compilation failures and runner errors have no `test result` summary,
|
|
||||||
// but still make the default test phase fail.
|
|
||||||
if !out.status.success() && failed == 0 {
|
|
||||||
failed = 1;
|
|
||||||
}
|
|
||||||
Ok(TestResults { passed, failed, output })
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Count the `passed` or `failed` totals in cargo's per-target test summaries.
|
|
||||||
fn test_count(output: &str, field: &str) -> u64 {
|
|
||||||
output
|
|
||||||
.lines()
|
|
||||||
.filter_map(|line| line.split_once("test result:").map(|(_, rest)| rest))
|
|
||||||
.flat_map(|summary| summary.split(';'))
|
|
||||||
.filter_map(|part| {
|
|
||||||
let words: Vec<_> = part.split_whitespace().collect();
|
|
||||||
let index = words.iter().position(|word| *word == field)?;
|
|
||||||
words.get(index.checked_sub(1)?)?.parse::<u64>().ok()
|
|
||||||
})
|
|
||||||
.sum()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Point a `sausage-actor` dependency at `sdk` (keeping its features), and
|
|
||||||
/// add an empty `[workspace]`, so the crate never joins a parent workspace.
|
|
||||||
fn prepare_manifest(text: &str, sdk: Option<&Path>) -> Result<String> {
|
|
||||||
let mut doc: toml_edit::DocumentMut = text.parse()?;
|
|
||||||
if let Some(sdk) = sdk {
|
|
||||||
for section in ["dependencies", "dev-dependencies", "build-dependencies"] {
|
|
||||||
let Some(dep) = doc.get_mut(section).and_then(|t| t.get_mut("sausage-actor")) else { continue };
|
|
||||||
let mut table = dep.as_inline_table().cloned().unwrap_or_default();
|
|
||||||
for key in ["version", "path", "git", "branch", "tag", "rev", "registry"] {
|
|
||||||
table.remove(key);
|
|
||||||
}
|
|
||||||
table.insert("path", sdk.display().to_string().into());
|
|
||||||
*dep = toml_edit::value(table);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if doc.get("workspace").is_none() {
|
|
||||||
doc["workspace"] = toml_edit::table();
|
|
||||||
}
|
|
||||||
Ok(doc.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Write the files of `dir` below `to`. Skip files that did not change, so
|
|
||||||
/// that cargo does not rebuild the SDK at every start.
|
|
||||||
fn write_dir(dir: &Dir<'_>, to: &Path) -> Result<()> {
|
|
||||||
for file in dir.files() {
|
|
||||||
let dest = to.join(file.path());
|
|
||||||
let contents = match file.path().file_name() {
|
|
||||||
Some(n) if n == "Cargo.toml" => prepare_manifest(file.contents_utf8().context("Cargo.toml")?, None)?.into_bytes(),
|
|
||||||
_ => file.contents().to_vec(),
|
|
||||||
};
|
|
||||||
if std::fs::read(&dest).ok() != Some(contents.clone()) {
|
|
||||||
std::fs::create_dir_all(dest.parent().context("no parent")?)?;
|
|
||||||
std::fs::write(&dest, contents)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for sub in dir.dirs() {
|
|
||||||
write_dir(sub, to)?;
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn next_build_number(dir: &Path) -> Result<u64> {
|
fn next_build_number(dir: &Path) -> Result<u64> {
|
||||||
std::fs::create_dir_all(dir)?;
|
std::fs::create_dir_all(dir)?;
|
||||||
let max = std::fs::read_dir(dir)?
|
let max = std::fs::read_dir(dir)?
|
||||||
@@ -326,3 +158,12 @@ fn strip_progress(stderr: &str) -> String {
|
|||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join("\n")
|
.join("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn fxhash(bytes: &[u8]) -> u64 {
|
||||||
|
let mut h: u64 = 0xcbf29ce484222325;
|
||||||
|
for b in bytes {
|
||||||
|
h ^= *b as u64;
|
||||||
|
h = h.wrapping_mul(0x100000001b3);
|
||||||
|
}
|
||||||
|
h
|
||||||
|
}
|
||||||
|
|||||||
+164
-24
@@ -3,13 +3,19 @@
|
|||||||
//! - `@<actor> <json>` sends a JSON body. `@<actor> <text>` sends
|
//! - `@<actor> <json>` sends a JSON body. `@<actor> <text>` sends
|
||||||
//! `{"type": "text", "text": ...}`.
|
//! `{"type": "text", "text": ...}`.
|
||||||
//! - A line without `@` goes to `agent` as text.
|
//! - A line without `@` goes to `agent` as text.
|
||||||
//! - `/help`, `/quit`. EOF on stdin also quits.
|
//! - `/deploy <actor> <folder>` builds a Rust folder with `cargo`, then spawns
|
||||||
|
//! or upgrades the actor.
|
||||||
|
//! - `/list`, `/sleep <seconds>`, `/help`, `/quit`.
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::collections::HashMap;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Result, anyhow, bail};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::{mpsc, oneshot};
|
||||||
|
|
||||||
use crate::broker::{Broker, Envelope};
|
use crate::broker::{Broker, Envelope};
|
||||||
|
|
||||||
@@ -19,49 +25,183 @@ const HELP: &str = "\
|
|||||||
@<actor> <json> send a JSON body to an actor
|
@<actor> <json> send a JSON body to an actor
|
||||||
@<actor> <text> send {\"type\":\"text\",\"text\":...}
|
@<actor> <text> send {\"type\":\"text\",\"text\":...}
|
||||||
<text> send text to the agent
|
<text> send text to the agent
|
||||||
|
/deploy <actor> <dir> build a Rust folder with cargo, then spawn or upgrade
|
||||||
|
/list list actors
|
||||||
|
/sleep <seconds> wait (for scripts)
|
||||||
|
/await <actor> <secs> wait for the next message from an actor (for scripts)
|
||||||
/quit stop all actors and exit";
|
/quit stop all actors and exit";
|
||||||
|
|
||||||
/// Print a message: text bodies as text, everything else as JSON.
|
pub struct Console {
|
||||||
|
broker: Arc<Broker>,
|
||||||
|
waiters: Mutex<HashMap<String, oneshot::Sender<Envelope>>>,
|
||||||
|
seq: AtomicU64,
|
||||||
|
/// Number of messages received per sender, for `/await`.
|
||||||
|
seen: Mutex<HashMap<String, u64>>,
|
||||||
|
arrived: tokio::sync::Notify,
|
||||||
|
}
|
||||||
|
|
||||||
fn show(env: &Envelope) {
|
fn show(env: &Envelope) {
|
||||||
let reply = if env.reply_to.is_some() { " ↩" } else { "" };
|
let reply = if env.reply_to.is_some() { " ↩" } else { "" };
|
||||||
match env.body["text"].as_str() {
|
let b = &env.body;
|
||||||
Some(text) if env.body["type"] == "text" => println!("[{}{reply}] {text}", env.from),
|
if b["type"] == "text"
|
||||||
_ => println!("[{}{reply}] {}", env.from, env.body),
|
&& let Some(t) = b["text"].as_str()
|
||||||
|
{
|
||||||
|
println!("[{}{reply}] {t}", env.from);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut compact = b.clone();
|
||||||
|
let mut extra = Vec::new();
|
||||||
|
if let Some(obj) = compact.as_object_mut() {
|
||||||
|
for key in ["errors", "warnings", "reason"] {
|
||||||
|
if let Some(Value::String(s)) = obj.get(key)
|
||||||
|
&& s.contains('\n')
|
||||||
|
{
|
||||||
|
extra.push((key, s.clone()));
|
||||||
|
obj.remove(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("[{}{reply}] {compact}", env.from);
|
||||||
|
for (k, s) in extra {
|
||||||
|
println!(" {k}:\n{}", s.lines().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Print every message that arrives for the console.
|
impl Console {
|
||||||
pub async fn run_mailbox(mut mailbox: mpsc::Receiver<Envelope>) {
|
pub fn new(broker: Arc<Broker>) -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
broker,
|
||||||
|
waiters: Mutex::new(HashMap::new()),
|
||||||
|
seq: AtomicU64::new(1),
|
||||||
|
seen: Mutex::new(HashMap::new()),
|
||||||
|
arrived: tokio::sync::Notify::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run_mailbox(self: Arc<Self>, mut mailbox: mpsc::Receiver<Envelope>) {
|
||||||
while let Some(env) = mailbox.recv().await {
|
while let Some(env) = mailbox.recv().await {
|
||||||
show(&env);
|
show(&env);
|
||||||
|
*self.seen.lock().unwrap().entry(env.from.clone()).or_default() += 1;
|
||||||
|
self.arrived.notify_waiters();
|
||||||
|
if let Some(r) = &env.reply_to
|
||||||
|
&& let Some(tx) = self.waiters.lock().unwrap().remove(r)
|
||||||
|
{
|
||||||
|
let _ = tx.send(env);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Read lines until EOF or `/quit`, and send them as messages.
|
fn send(&self, to: &str, body: Value) -> String {
|
||||||
pub async fn run_input(broker: Arc<Broker>) {
|
let own = format!("c{}", self.seq.fetch_add(1, Ordering::Relaxed));
|
||||||
|
self.broker.send(ID, Some(to), Some(&own), None, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn request(&self, to: &str, body: Value, timeout: Duration) -> Result<Envelope> {
|
||||||
|
let own = format!("c{}", self.seq.fetch_add(1, Ordering::Relaxed));
|
||||||
|
let (tx, rx) = oneshot::channel();
|
||||||
|
self.waiters.lock().unwrap().insert(format!("{ID}/{own}"), tx);
|
||||||
|
self.broker.send(ID, Some(to), Some(&own), None, body);
|
||||||
|
tokio::time::timeout(timeout, rx)
|
||||||
|
.await
|
||||||
|
.map_err(|_| anyhow!("no reply from {to} within {timeout:?}"))?
|
||||||
|
.map_err(|_| anyhow!("reply channel closed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn deploy(&self, actor: &str, dir: &str) -> Result<()> {
|
||||||
|
let path = std::path::absolute(dir)?;
|
||||||
|
let source = path.display().to_string();
|
||||||
|
println!("· building {source} …");
|
||||||
|
let built = self.request(crate::cargo_actor::ID, json!({"type": "build", "source": source}), Duration::from_secs(900)).await?;
|
||||||
|
if built.body["type"] != "built" {
|
||||||
|
bail!("build failed");
|
||||||
|
}
|
||||||
|
let exists = self.broker.registry().actors.contains_key(actor);
|
||||||
|
let ty = if exists { "upgrade" } else { "spawn" };
|
||||||
|
println!("· {ty} {actor} …");
|
||||||
|
let body = json!({
|
||||||
|
"type": ty,
|
||||||
|
"actor": actor,
|
||||||
|
"wasm": built.body["wasm"],
|
||||||
|
"source": source,
|
||||||
|
"built_by": built.body["built_by"],
|
||||||
|
});
|
||||||
|
let r = self.request("broker", body, Duration::from_secs(300)).await?;
|
||||||
|
if r.body["type"] == "error" {
|
||||||
|
bail!("{ty} failed");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read commands until EOF or `/quit`.
|
||||||
|
pub async fn run_input(self: Arc<Self>) {
|
||||||
println!("Sausage console. Type /help for commands.");
|
println!("Sausage console. Type /help for commands.");
|
||||||
let mut lines = BufReader::new(tokio::io::stdin()).lines();
|
let mut lines = BufReader::new(tokio::io::stdin()).lines();
|
||||||
while let Ok(Some(line)) = lines.next_line().await {
|
while let Ok(Some(line)) = lines.next_line().await {
|
||||||
let line = line.trim();
|
let line = line.trim();
|
||||||
let (to, body) = match line {
|
if line.is_empty() || line.starts_with('#') {
|
||||||
"" => continue,
|
|
||||||
"/quit" => break,
|
|
||||||
"/help" => {
|
|
||||||
println!("{HELP}");
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
_ => match line.strip_prefix('@') {
|
if let Err(e) = self.command(line).await {
|
||||||
Some(rest) => {
|
if e.to_string() == "quit" {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
println!("! {e:#}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command(&self, line: &str) -> Result<()> {
|
||||||
|
if let Some(rest) = line.strip_prefix('@') {
|
||||||
let (to, text) = rest.split_once(char::is_whitespace).unwrap_or((rest, ""));
|
let (to, text) = rest.split_once(char::is_whitespace).unwrap_or((rest, ""));
|
||||||
let body = match serde_json::from_str::<Value>(text.trim()) {
|
let text = text.trim();
|
||||||
|
let body = match serde_json::from_str::<Value>(text) {
|
||||||
Ok(v @ Value::Object(_)) => v,
|
Ok(v @ Value::Object(_)) => v,
|
||||||
_ => json!({"type": "text", "text": text.trim()}),
|
_ => json!({"type": "text", "text": text}),
|
||||||
};
|
};
|
||||||
(to, body)
|
self.send(to, body);
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
None => ("agent", json!({"type": "text", "text": line})),
|
if let Some(cmd) = line.strip_prefix('/') {
|
||||||
},
|
let mut parts = cmd.split_whitespace();
|
||||||
|
match parts.next() {
|
||||||
|
Some("help") => println!("{HELP}"),
|
||||||
|
Some("quit") => bail!("quit"),
|
||||||
|
Some("list") => {
|
||||||
|
self.send("broker", json!({"type": "list"}));
|
||||||
|
}
|
||||||
|
Some("sleep") => {
|
||||||
|
let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(1.0);
|
||||||
|
tokio::time::sleep(Duration::from_secs_f64(secs)).await;
|
||||||
|
}
|
||||||
|
Some("await") => {
|
||||||
|
let from = parts.next().unwrap_or("agent").to_string();
|
||||||
|
let secs: f64 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(60.0);
|
||||||
|
let count = |c: &Self| c.seen.lock().unwrap().get(&from).copied().unwrap_or(0);
|
||||||
|
let before = count(self);
|
||||||
|
let deadline = tokio::time::Instant::now() + Duration::from_secs_f64(secs);
|
||||||
|
loop {
|
||||||
|
let notified = self.arrived.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
if count(self) != before {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if tokio::time::timeout_at(deadline, notified).await.is_err() {
|
||||||
|
bail!("no message from {from} within {secs} s");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("deploy") => {
|
||||||
|
let (Some(actor), Some(dir)) = (parts.next(), parts.next()) else {
|
||||||
|
bail!("usage: /deploy <actor> <dir>");
|
||||||
};
|
};
|
||||||
broker.send(ID, Some(to), None, None, body);
|
self.deploy(actor, dir).await?;
|
||||||
|
}
|
||||||
|
_ => bail!("unknown command. Type /help."),
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
self.send("agent", json!({"type": "text", "text": line}));
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+434
@@ -0,0 +1,434 @@
|
|||||||
|
//! Filesystem layer with commit-on-close semantics.
|
||||||
|
//!
|
||||||
|
//! The layer wraps the wasmtime-wasi `wasi:filesystem` implementation. For
|
||||||
|
//! mounts with `commit_on_close`, an open for write goes to a hidden temporary
|
||||||
|
//! file next to the target. When the guest drops the descriptor (close), the
|
||||||
|
//! layer renames the temporary file into place. Until then, no other reader
|
||||||
|
//! sees the change. If the instance traps or is killed, the changes are lost.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Component, Path, PathBuf};
|
||||||
|
|
||||||
|
use wasmtime::component::{HasData, Resource};
|
||||||
|
use wasmtime_wasi::filesystem::{Descriptor, WasiFilesystemCtxView};
|
||||||
|
use wasmtime_wasi::p2::bindings::filesystem::types::{
|
||||||
|
self, ErrorCode, HostDescriptor, HostDirectoryEntryStream,
|
||||||
|
};
|
||||||
|
use wasmtime_wasi::p2::bindings::filesystem::preopens;
|
||||||
|
use wasmtime_wasi::p2::{DynInputStream, DynOutputStream, FsError, FsResult};
|
||||||
|
|
||||||
|
/// Name prefix of uncommitted files. The layer hides these names from guests.
|
||||||
|
pub const TMP_PREFIX: &str = ".sausage-tmp-";
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct Mount {
|
||||||
|
pub guest: String,
|
||||||
|
pub host: PathBuf,
|
||||||
|
pub writable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct DirInfo {
|
||||||
|
host: PathBuf,
|
||||||
|
root: PathBuf,
|
||||||
|
commit: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Pending {
|
||||||
|
tmp: PathBuf,
|
||||||
|
target: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-instance state of the layer.
|
||||||
|
pub struct FsExt {
|
||||||
|
mounts: Vec<Mount>,
|
||||||
|
dirs: HashMap<u32, DirInfo>,
|
||||||
|
pending: HashMap<u32, Pending>,
|
||||||
|
counter: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FsExt {
|
||||||
|
pub fn new(mounts: Vec<Mount>) -> Self {
|
||||||
|
Self { mounts, dirs: HashMap::new(), pending: HashMap::new(), counter: 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Commit every open write. Use this when the guest exits normally,
|
||||||
|
/// because process exit closes all files.
|
||||||
|
pub fn commit_all(&mut self) {
|
||||||
|
for (_, p) in self.pending.drain() {
|
||||||
|
commit(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for FsExt {
|
||||||
|
/// Discard writes that were not committed (trap, kill).
|
||||||
|
fn drop(&mut self) {
|
||||||
|
for (_, p) in self.pending.drain() {
|
||||||
|
let _ = std::fs::remove_file(&p.tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit(p: Pending) {
|
||||||
|
if let Err(e) = std::fs::rename(&p.tmp, &p.target) {
|
||||||
|
eprintln!("sausage: commit of {} failed: {e}", p.target.display());
|
||||||
|
let _ = std::fs::remove_file(&p.tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_tmp_name(path: &str) -> bool {
|
||||||
|
path.rsplit('/').next().is_some_and(|n| n.starts_with(TMP_PREFIX))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Join `rel` to the directory `dir` lexically. Return `None` if the result
|
||||||
|
/// leaves the mount root.
|
||||||
|
fn resolve(dir: &DirInfo, rel: &str) -> Option<PathBuf> {
|
||||||
|
let mut out = dir.host.clone();
|
||||||
|
for c in Path::new(rel).components() {
|
||||||
|
match c {
|
||||||
|
Component::Normal(n) => out.push(n),
|
||||||
|
Component::CurDir => {}
|
||||||
|
Component::ParentDir => {
|
||||||
|
if !out.pop() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Component::RootDir | Component::Prefix(_) => return None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.starts_with(&dir.root).then_some(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn io_err(e: std::io::Error) -> FsError {
|
||||||
|
ErrorCode::from(e).into()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ActorFs<'a> {
|
||||||
|
pub inner: WasiFilesystemCtxView<'a>,
|
||||||
|
pub ext: &'a mut FsExt,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ActorFsData;
|
||||||
|
|
||||||
|
impl HasData for ActorFsData {
|
||||||
|
type Data<'a> = ActorFs<'a>;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl preopens::Host for ActorFs<'_> {
|
||||||
|
fn get_directories(&mut self) -> wasmtime::Result<Vec<(Resource<Descriptor>, String)>> {
|
||||||
|
let dirs = preopens::Host::get_directories(&mut self.inner)?;
|
||||||
|
for (r, name) in &dirs {
|
||||||
|
if let Some(m) = self.ext.mounts.iter().find(|m| &m.guest == name) {
|
||||||
|
self.ext.dirs.insert(
|
||||||
|
r.rep(),
|
||||||
|
DirInfo { host: m.host.clone(), root: m.host.clone(), commit: m.writable },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(dirs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl types::Host for ActorFs<'_> {
|
||||||
|
fn convert_error_code(&mut self, err: FsError) -> wasmtime::Result<ErrorCode> {
|
||||||
|
types::Host::convert_error_code(&mut self.inner, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn filesystem_error_code(
|
||||||
|
&mut self,
|
||||||
|
err: Resource<wasmtime::Error>,
|
||||||
|
) -> wasmtime::Result<Option<ErrorCode>> {
|
||||||
|
types::Host::filesystem_error_code(&mut self.inner, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HostDescriptor for ActorFs<'_> {
|
||||||
|
async fn advise(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
offset: types::Filesize,
|
||||||
|
len: types::Filesize,
|
||||||
|
advice: types::Advice,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
HostDescriptor::advise(&mut self.inner, fd, offset, len, advice).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sync_data(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
||||||
|
HostDescriptor::sync_data(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_flags(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorFlags> {
|
||||||
|
HostDescriptor::get_flags(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_type(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorType> {
|
||||||
|
HostDescriptor::get_type(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_size(&mut self, fd: Resource<types::Descriptor>, size: types::Filesize) -> FsResult<()> {
|
||||||
|
HostDescriptor::set_size(&mut self.inner, fd, size).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_times(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
atim: types::NewTimestamp,
|
||||||
|
mtim: types::NewTimestamp,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
HostDescriptor::set_times(&mut self.inner, fd, atim, mtim).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
len: types::Filesize,
|
||||||
|
offset: types::Filesize,
|
||||||
|
) -> FsResult<(Vec<u8>, bool)> {
|
||||||
|
HostDescriptor::read(&mut self.inner, fd, len, offset).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
buf: Vec<u8>,
|
||||||
|
offset: types::Filesize,
|
||||||
|
) -> FsResult<types::Filesize> {
|
||||||
|
HostDescriptor::write(&mut self.inner, fd, buf, offset).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_directory(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
) -> FsResult<Resource<types::DirectoryEntryStream>> {
|
||||||
|
HostDescriptor::read_directory(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sync(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
||||||
|
HostDescriptor::sync(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||||
|
HostDescriptor::create_directory_at(&mut self.inner, fd, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorStat> {
|
||||||
|
HostDescriptor::stat(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
path_flags: types::PathFlags,
|
||||||
|
path: String,
|
||||||
|
) -> FsResult<types::DescriptorStat> {
|
||||||
|
if is_tmp_name(&path) {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
HostDescriptor::stat_at(&mut self.inner, fd, path_flags, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_times_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
path_flags: types::PathFlags,
|
||||||
|
path: String,
|
||||||
|
atim: types::NewTimestamp,
|
||||||
|
mtim: types::NewTimestamp,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
HostDescriptor::set_times_at(&mut self.inner, fd, path_flags, path, atim, mtim).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn link_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
old_path_flags: types::PathFlags,
|
||||||
|
old_path: String,
|
||||||
|
new_descriptor: Resource<types::Descriptor>,
|
||||||
|
new_path: String,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
HostDescriptor::link_at(&mut self.inner, fd, old_path_flags, old_path, new_descriptor, new_path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn open_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
path_flags: types::PathFlags,
|
||||||
|
path: String,
|
||||||
|
oflags: types::OpenFlags,
|
||||||
|
flags: types::DescriptorFlags,
|
||||||
|
) -> FsResult<Resource<types::Descriptor>> {
|
||||||
|
if is_tmp_name(&path) {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
let parent = self.ext.dirs.get(&fd.rep()).cloned();
|
||||||
|
|
||||||
|
// Commit-on-close: redirect writes to a hidden temporary file.
|
||||||
|
if let Some(dir) = parent.as_ref().filter(|d| d.commit)
|
||||||
|
&& flags.contains(types::DescriptorFlags::WRITE)
|
||||||
|
&& !oflags.contains(types::OpenFlags::DIRECTORY)
|
||||||
|
{
|
||||||
|
let target = resolve(dir, &path).ok_or(FsError::from(ErrorCode::NotPermitted))?;
|
||||||
|
let meta = std::fs::metadata(&target).ok();
|
||||||
|
if meta.as_ref().is_some_and(|m| m.is_dir()) {
|
||||||
|
return Err(ErrorCode::IsDirectory.into());
|
||||||
|
}
|
||||||
|
let exists = meta.is_some();
|
||||||
|
if exists && oflags.contains(types::OpenFlags::EXCLUSIVE) {
|
||||||
|
return Err(ErrorCode::Exist.into());
|
||||||
|
}
|
||||||
|
if !exists && !oflags.contains(types::OpenFlags::CREATE) {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
let target_dir = target.parent().ok_or(FsError::from(ErrorCode::NotPermitted))?;
|
||||||
|
if !target_dir.is_dir() {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
self.ext.counter += 1;
|
||||||
|
let tmp_name = format!("{TMP_PREFIX}{}-{}", std::process::id(), self.ext.counter);
|
||||||
|
let tmp_rel = match path.rsplit_once('/') {
|
||||||
|
Some((d, _)) => format!("{d}/{tmp_name}"),
|
||||||
|
None => tmp_name.clone(),
|
||||||
|
};
|
||||||
|
let tmp = target_dir.join(&tmp_name);
|
||||||
|
let truncate = oflags.contains(types::OpenFlags::TRUNCATE);
|
||||||
|
if exists && !truncate {
|
||||||
|
std::fs::copy(&target, &tmp).map_err(io_err)?;
|
||||||
|
}
|
||||||
|
let oflags = (oflags | types::OpenFlags::CREATE)
|
||||||
|
& !(types::OpenFlags::EXCLUSIVE | types::OpenFlags::TRUNCATE);
|
||||||
|
return match HostDescriptor::open_at(&mut self.inner, fd, path_flags, tmp_rel, oflags, flags).await {
|
||||||
|
Ok(r) => {
|
||||||
|
self.ext.pending.insert(r.rep(), Pending { tmp, target });
|
||||||
|
Ok(r)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
let _ = std::fs::remove_file(&tmp);
|
||||||
|
Err(e)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let r = HostDescriptor::open_at(&mut self.inner, fd, path_flags, path.clone(), oflags, flags).await?;
|
||||||
|
if let Some(dir) = parent
|
||||||
|
&& matches!(self.inner.table.get(&r)?, Descriptor::Dir(_))
|
||||||
|
&& let Some(host) = resolve(&dir, &path)
|
||||||
|
{
|
||||||
|
self.ext.dirs.insert(r.rep(), DirInfo { host, root: dir.root, commit: dir.commit });
|
||||||
|
}
|
||||||
|
Ok(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drop(&mut self, fd: Resource<types::Descriptor>) -> wasmtime::Result<()> {
|
||||||
|
let rep = fd.rep();
|
||||||
|
HostDescriptor::drop(&mut self.inner, fd)?;
|
||||||
|
self.ext.dirs.remove(&rep);
|
||||||
|
if let Some(p) = self.ext.pending.remove(&rep) {
|
||||||
|
commit(p);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn readlink_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<String> {
|
||||||
|
HostDescriptor::readlink_at(&mut self.inner, fd, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||||
|
HostDescriptor::remove_directory_at(&mut self.inner, fd, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn rename_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
old_path: String,
|
||||||
|
new_fd: Resource<types::Descriptor>,
|
||||||
|
new_path: String,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
if is_tmp_name(&old_path) || is_tmp_name(&new_path) {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
HostDescriptor::rename_at(&mut self.inner, fd, old_path, new_fd, new_path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn symlink_at(
|
||||||
|
&mut self,
|
||||||
|
_fd: Resource<types::Descriptor>,
|
||||||
|
_src_path: String,
|
||||||
|
_dest_path: String,
|
||||||
|
) -> FsResult<()> {
|
||||||
|
// Symlinks would break the lexical path tracking of commit-on-close.
|
||||||
|
Err(ErrorCode::NotPermitted.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn unlink_file_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
||||||
|
if is_tmp_name(&path) {
|
||||||
|
return Err(ErrorCode::NoEntry.into());
|
||||||
|
}
|
||||||
|
HostDescriptor::unlink_file_at(&mut self.inner, fd, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_via_stream(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
offset: types::Filesize,
|
||||||
|
) -> FsResult<Resource<DynInputStream>> {
|
||||||
|
HostDescriptor::read_via_stream(&mut self.inner, fd, offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_via_stream(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
offset: types::Filesize,
|
||||||
|
) -> FsResult<Resource<DynOutputStream>> {
|
||||||
|
HostDescriptor::write_via_stream(&mut self.inner, fd, offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_via_stream(&mut self, fd: Resource<types::Descriptor>) -> FsResult<Resource<DynOutputStream>> {
|
||||||
|
HostDescriptor::append_via_stream(&mut self.inner, fd)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn is_same_object(
|
||||||
|
&mut self,
|
||||||
|
a: Resource<types::Descriptor>,
|
||||||
|
b: Resource<types::Descriptor>,
|
||||||
|
) -> wasmtime::Result<bool> {
|
||||||
|
HostDescriptor::is_same_object(&mut self.inner, a, b).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn metadata_hash(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::MetadataHashValue> {
|
||||||
|
HostDescriptor::metadata_hash(&mut self.inner, fd).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn metadata_hash_at(
|
||||||
|
&mut self,
|
||||||
|
fd: Resource<types::Descriptor>,
|
||||||
|
path_flags: types::PathFlags,
|
||||||
|
path: String,
|
||||||
|
) -> FsResult<types::MetadataHashValue> {
|
||||||
|
HostDescriptor::metadata_hash_at(&mut self.inner, fd, path_flags, path).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HostDirectoryEntryStream for ActorFs<'_> {
|
||||||
|
async fn read_directory_entry(
|
||||||
|
&mut self,
|
||||||
|
stream: Resource<types::DirectoryEntryStream>,
|
||||||
|
) -> FsResult<Option<types::DirectoryEntry>> {
|
||||||
|
let rep = stream.rep();
|
||||||
|
loop {
|
||||||
|
let entry = HostDirectoryEntryStream::read_directory_entry(
|
||||||
|
&mut self.inner,
|
||||||
|
Resource::new_borrow(rep),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
match entry {
|
||||||
|
Some(e) if e.name.starts_with(TMP_PREFIX) => continue,
|
||||||
|
other => return Ok(other),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drop(&mut self, stream: Resource<types::DirectoryEntryStream>) -> wasmtime::Result<()> {
|
||||||
|
HostDirectoryEntryStream::drop(&mut self.inner, stream)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
//! Sausage: grow a system of WASI actors by talking to it.
|
|
||||||
//!
|
|
||||||
//! The host runs the broker, the native actors (`broker`, `cargo`, and the
|
|
||||||
//! console), one supervisor per WASM actor, and the file watcher of the
|
|
||||||
//! world folder (`watch.rs`). See `PROTOCOL.md`.
|
|
||||||
|
|
||||||
pub mod broker;
|
|
||||||
pub mod candidate;
|
|
||||||
pub mod cargo_actor;
|
|
||||||
pub mod console;
|
|
||||||
pub mod runtime;
|
|
||||||
pub mod supervisor;
|
|
||||||
pub mod vfs;
|
|
||||||
pub mod watch;
|
|
||||||
pub mod world;
|
|
||||||
|
|
||||||
use std::path::Path;
|
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use anyhow::{Result, bail};
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
|
|
||||||
pub use broker::{Broker, Envelope};
|
|
||||||
pub use world::World;
|
|
||||||
|
|
||||||
/// The contract between the host and the actors. The host publishes it at
|
|
||||||
/// `/actor/broker/public/PROTOCOL.md`, so that every actor can read it.
|
|
||||||
const PROTOCOL: &str = include_str!("../../PROTOCOL.md");
|
|
||||||
|
|
||||||
/// A running world.
|
|
||||||
pub struct Host {
|
|
||||||
pub world: World,
|
|
||||||
pub broker: Arc<Broker>,
|
|
||||||
native: Vec<tokio::task::JoinHandle<()>>,
|
|
||||||
watcher: Option<watch::Watcher>,
|
|
||||||
_lock: std::fs::File,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Host {
|
|
||||||
/// Start the broker, `cargo`, the file watcher, and every registered
|
|
||||||
/// actor of the world at `root`. Only one host can run a world at a time.
|
|
||||||
pub async fn start(root: &Path) -> Result<Host> {
|
|
||||||
let world = World::new(std::path::absolute(root)?);
|
|
||||||
if !world.exists() {
|
|
||||||
bail!("{} is not a Sausage world. Create one with: sausage init", root.display());
|
|
||||||
}
|
|
||||||
let lock = std::fs::File::create(world.root.join(".lock"))?;
|
|
||||||
if lock.try_lock().is_err() {
|
|
||||||
bail!("another sausage process runs {}", root.display());
|
|
||||||
}
|
|
||||||
let broker = Broker::new(world.clone(), runtime::Runtime::new()?)?;
|
|
||||||
world::write_atomic(&world.public_dir("broker").join("PROTOCOL.md"), PROTOCOL.as_bytes())?;
|
|
||||||
|
|
||||||
let cargo = cargo_actor::Cargo::new(world.clone())?;
|
|
||||||
let native = vec![
|
|
||||||
tokio::spawn(broker.clone().run_actor(broker.open_mailbox("broker"))),
|
|
||||||
tokio::spawn(cargo.run_actor(broker.clone(), broker.open_mailbox(cargo_actor::ID))),
|
|
||||||
];
|
|
||||||
// Watch first, so that no change during the boot pass is lost.
|
|
||||||
let watcher = watch::start(broker.clone())
|
|
||||||
.inspect_err(|e| broker.host_error(&format!("no file watching: {e}")))
|
|
||||||
.ok();
|
|
||||||
for change in broker.scan().await {
|
|
||||||
eprintln!("sausage: {change}");
|
|
||||||
}
|
|
||||||
Ok(Host { world, broker, native, watcher, _lock: lock })
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Register a native actor with the ID `id` and return its mailbox.
|
|
||||||
pub fn mailbox(&self, id: &str) -> mpsc::Receiver<Envelope> {
|
|
||||||
self.broker.open_mailbox(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Stop the file watcher, the broker and `cargo` (with the requests they
|
|
||||||
/// still run), then every actor, gracefully.
|
|
||||||
pub async fn stop(mut self) {
|
|
||||||
drop(self.watcher.take());
|
|
||||||
for task in &self.native {
|
|
||||||
task.abort();
|
|
||||||
}
|
|
||||||
for task in self.native {
|
|
||||||
let _ = task.await;
|
|
||||||
}
|
|
||||||
self.broker.shutdown_all().await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+116
-60
@@ -1,36 +1,51 @@
|
|||||||
|
//! Sausage: grow a system of WASI actors by talking to it.
|
||||||
|
|
||||||
|
mod broker;
|
||||||
|
mod candidate;
|
||||||
|
mod cargo_actor;
|
||||||
|
mod console;
|
||||||
|
mod fs;
|
||||||
|
mod runtime;
|
||||||
|
mod world;
|
||||||
|
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use include_dir::{Dir, include_dir};
|
use include_dir::{Dir, include_dir};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use sausage::world::{ActorMeta, World};
|
use crate::broker::Broker;
|
||||||
use sausage::{Host, cargo_actor, console};
|
use crate::world::{NATIVE, Registry, VersionMeta, World};
|
||||||
|
|
||||||
/// The source of the agent actor. `init` builds it like any other actor.
|
/// The source of the agent actor. `init` builds it like any other actor.
|
||||||
static AGENT_SOURCE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../agent");
|
static AGENT_SOURCE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../agent");
|
||||||
|
|
||||||
// Without a command, `sausage` runs the world in `./world` and creates it
|
|
||||||
// first (like `init`) if it does not exist yet.
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(version, about)]
|
#[command(version, about)]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
#[command(subcommand)]
|
#[command(subcommand)]
|
||||||
command: Option<Cmd>,
|
command: Cmd,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Subcommand)]
|
#[derive(Subcommand)]
|
||||||
enum Cmd {
|
enum Cmd {
|
||||||
/// Create a new world and build the agent actor.
|
/// Create a new world and build the agent actor.
|
||||||
///
|
|
||||||
/// Reads SAUSAGE_API, SAUSAGE_BASE_URL, SAUSAGE_MODEL and
|
|
||||||
/// SAUSAGE_API_TOKEN, and asks for every value that is missing.
|
|
||||||
Init {
|
Init {
|
||||||
/// Folder of the new world.
|
/// Folder of the new world.
|
||||||
#[arg(default_value = "world")]
|
#[arg(default_value = "world")]
|
||||||
dir: PathBuf,
|
dir: PathBuf,
|
||||||
|
/// API shape: `openai-responses` or `anthropic-messages`.
|
||||||
|
#[arg(long)]
|
||||||
|
api: Option<String>,
|
||||||
|
/// Base URL of the inference API (for example `https://api.openai.com`).
|
||||||
|
#[arg(long)]
|
||||||
|
base_url: Option<String>,
|
||||||
|
/// Model ID.
|
||||||
|
#[arg(long)]
|
||||||
|
model: Option<String>,
|
||||||
},
|
},
|
||||||
/// Start the broker, all actors, and the console.
|
/// Start the broker, all actors, and the console.
|
||||||
Run {
|
Run {
|
||||||
@@ -39,13 +54,7 @@ enum Cmd {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The value of `var`, or the answer to `question`.
|
fn prompt(question: &str, default: Option<&str>) -> Result<String> {
|
||||||
fn setting(var: &str, question: &str, default: Option<&str>) -> Result<String> {
|
|
||||||
if let Ok(v) = std::env::var(var)
|
|
||||||
&& !v.is_empty()
|
|
||||||
{
|
|
||||||
return Ok(v);
|
|
||||||
}
|
|
||||||
match default {
|
match default {
|
||||||
Some(d) => print!("{question} [{d}]: "),
|
Some(d) => print!("{question} [{d}]: "),
|
||||||
None => print!("{question}: "),
|
None => print!("{question}: "),
|
||||||
@@ -53,76 +62,123 @@ fn setting(var: &str, question: &str, default: Option<&str>) -> Result<String> {
|
|||||||
std::io::stdout().flush()?;
|
std::io::stdout().flush()?;
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
std::io::stdin().read_line(&mut line)?;
|
std::io::stdin().read_line(&mut line)?;
|
||||||
match (line.trim(), default) {
|
let v = line.trim().to_string();
|
||||||
("", Some(d)) => Ok(d.to_string()),
|
match (v.is_empty(), default) {
|
||||||
("", None) => bail!("{question} is required (or set {var})"),
|
(true, Some(d)) => Ok(d.to_string()),
|
||||||
(v, _) => Ok(v.to_string()),
|
(true, None) => bail!("{question} is required"),
|
||||||
|
_ => Ok(v),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn init(dir: PathBuf) -> Result<()> {
|
async fn init(dir: PathBuf, api: Option<String>, base_url: Option<String>, model: Option<String>) -> Result<()> {
|
||||||
let api = setting("SAUSAGE_API", "API shape (openai-responses or anthropic-messages)", Some("openai-responses"))?;
|
let world = World::new(std::path::absolute(&dir)?);
|
||||||
|
if world.exists() {
|
||||||
|
bail!("{} is already a Sausage world", dir.display());
|
||||||
|
}
|
||||||
|
let api = match api {
|
||||||
|
Some(a) => a,
|
||||||
|
None => prompt("API shape (openai-responses or anthropic-messages)", Some("openai-responses"))?,
|
||||||
|
};
|
||||||
if api != "openai-responses" && api != "anthropic-messages" {
|
if api != "openai-responses" && api != "anthropic-messages" {
|
||||||
bail!("unknown API shape {api:?}");
|
bail!("unknown API shape {api:?}");
|
||||||
}
|
}
|
||||||
let base_url = setting("SAUSAGE_BASE_URL", "API base URL", None)?;
|
let base_url = match base_url {
|
||||||
let model = setting("SAUSAGE_MODEL", "Model", None)?;
|
Some(u) => u,
|
||||||
let token = setting("SAUSAGE_API_TOKEN", "API token", None)?;
|
None => prompt("API base URL", None)?,
|
||||||
|
};
|
||||||
|
let model = match model {
|
||||||
|
Some(m) => m,
|
||||||
|
None => prompt("Model", None)?,
|
||||||
|
};
|
||||||
|
let token = match std::env::var("SAUSAGE_API_TOKEN") {
|
||||||
|
Ok(t) if !t.is_empty() => t,
|
||||||
|
_ => prompt("API token", None)?,
|
||||||
|
};
|
||||||
|
|
||||||
let world = World::create(&dir)?;
|
std::fs::create_dir_all(&world.root)?;
|
||||||
let console = ActorMeta { subscriptions: ["agent".to_string()].into(), ..world.read_meta(console::ID) };
|
let mut reg = Registry::default();
|
||||||
world.write_meta(console::ID, &console)?;
|
reg.subscriptions.entry("agent".into()).or_default().insert("console".into());
|
||||||
|
world.save_registry(®)?;
|
||||||
|
for id in NATIVE {
|
||||||
|
world.ensure_actor_dirs(id)?;
|
||||||
|
world::write_atomic(
|
||||||
|
&world.actor_dir(id).join("status.json"),
|
||||||
|
serde_json::to_string_pretty(&json!({"id": id, "kind": "native", "state": "built-in"}))?.as_bytes(),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
// The agent's configuration, in its /data.
|
// The agent's private configuration. Only the agent can read /data.
|
||||||
world.ensure_actor_dirs("agent")?;
|
world.ensure_actor_dirs("agent")?;
|
||||||
let config = json!({"api": api, "base_url": base_url.trim_end_matches('/'), "model": model, "token": token});
|
let config = json!({"api": api, "base_url": base_url.trim_end_matches('/'), "model": model, "token": token});
|
||||||
let config_path = world.data_dir("agent").join("config.json");
|
let config_path = world.data_dir("agent").join("config.json");
|
||||||
std::fs::write(&config_path, serde_json::to_string_pretty(&config)?)?;
|
std::fs::write(&config_path, serde_json::to_string_pretty(&config)?)?;
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
std::fs::set_permissions(&config_path, std::fs::Permissions::from_mode(0o600))?;
|
||||||
|
}
|
||||||
|
|
||||||
// The agent is an ordinary actor folder: `run` finds its `actor.wasm`,
|
// Build the agent with the same path as every other Rust actor.
|
||||||
// tests it, and installs it as version 1.
|
let source = world.root.join("broker").join("seed").join("agent");
|
||||||
let source = world.source_dir("agent");
|
|
||||||
std::fs::create_dir_all(&source)?;
|
std::fs::create_dir_all(&source)?;
|
||||||
AGENT_SOURCE.extract(&source).context("extract agent source")?;
|
AGENT_SOURCE.extract(&source).context("extract agent source")?;
|
||||||
println!("Building the agent actor (the first build takes a few minutes) …");
|
println!("Building the agent actor (the first build takes a few minutes) …");
|
||||||
let out = cargo_actor::Cargo::new(world.clone())?.build(&source, false).await.context("build agent")?;
|
let cargo = cargo_actor::Cargo::new(world.clone());
|
||||||
std::fs::copy(&out.wasm, world.wasm_path("agent"))?;
|
let out = cargo.build(&source).await.context("build agent")?;
|
||||||
println!("Built in {:.1} s. Start the world with: sausage run {}", out.seconds, dir.display());
|
println!("Built in {:.1} s.", out.seconds);
|
||||||
|
|
||||||
|
let runtime = runtime::Runtime::new()?;
|
||||||
|
let broker = Broker::new(world.clone(), runtime)?;
|
||||||
|
let meta = VersionMeta {
|
||||||
|
version: 1,
|
||||||
|
wasm: world.global_path(&out.wasm).unwrap_or_default(),
|
||||||
|
source: Some(source.display().to_string()),
|
||||||
|
built_by: Some(cargo_actor::ID.into()),
|
||||||
|
requested_by: "init".into(),
|
||||||
|
created_at: world::now(),
|
||||||
|
};
|
||||||
|
broker.create_actor("agent", &out.wasm, Some(&source), meta, false).await?;
|
||||||
|
println!("World ready at {}. Start it with: sausage run {}", world.root.display(), dir.display());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run(dir: PathBuf) -> Result<()> {
|
async fn run(dir: PathBuf) -> Result<()> {
|
||||||
let host = Host::start(&dir).await?;
|
let world = World::new(std::path::absolute(&dir)?);
|
||||||
tokio::spawn(console::run_mailbox(host.mailbox(console::ID)));
|
if !world.exists() {
|
||||||
let broker = host.broker.clone();
|
bail!("{} is not a Sausage world. Create one with: sausage init {}", dir.display(), dir.display());
|
||||||
let exit = tokio::select! {
|
|
||||||
_ = console::run_input(host.broker.clone()) => false,
|
|
||||||
_ = broker.exit_requested() => true,
|
|
||||||
};
|
|
||||||
println!("Stopping actors …");
|
|
||||||
host.stop().await;
|
|
||||||
if exit {
|
|
||||||
// The console still blocks on a stdin read that cannot be cancelled,
|
|
||||||
// so the runtime would not shut down on its own. Exit code 75 tells
|
|
||||||
// `nix run .#loop` to start the host again (0 = /quit = stop the loop).
|
|
||||||
println!("Exiting, as an actor asked the broker to.");
|
|
||||||
std::process::exit(75);
|
|
||||||
}
|
}
|
||||||
|
// One host process per world.
|
||||||
|
let lock = std::fs::File::create(world.root.join(".lock"))?;
|
||||||
|
if lock.try_lock().is_err() {
|
||||||
|
bail!("another sausage process runs {}", dir.display());
|
||||||
|
}
|
||||||
|
let runtime = runtime::Runtime::new()?;
|
||||||
|
let broker = Broker::new(world.clone(), runtime)?;
|
||||||
|
|
||||||
|
let rx = broker.open_mailbox("broker");
|
||||||
|
tokio::spawn(broker.clone().run_actor(rx));
|
||||||
|
|
||||||
|
let rx = broker.open_mailbox(cargo_actor::ID);
|
||||||
|
let cargo = Arc::new(cargo_actor::Cargo::new(world.clone()));
|
||||||
|
tokio::spawn(cargo.run_actor(broker.clone(), rx));
|
||||||
|
|
||||||
|
let rx = broker.open_mailbox(console::ID);
|
||||||
|
let console = console::Console::new(broker.clone());
|
||||||
|
tokio::spawn(console.clone().run_mailbox(rx));
|
||||||
|
|
||||||
|
broker.start_all();
|
||||||
|
console.run_input().await;
|
||||||
|
println!("Stopping actors …");
|
||||||
|
broker.shutdown_all().await;
|
||||||
|
drop(lock);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
match Cli::parse().command {
|
let cli = Cli::parse();
|
||||||
Some(Cmd::Init { dir }) => init(dir).await,
|
match cli.command {
|
||||||
Some(Cmd::Run { dir }) => run(dir).await,
|
Cmd::Init { dir, api, base_url, model } => init(dir, api, base_url, model).await,
|
||||||
None => {
|
Cmd::Run { dir } => run(dir).await,
|
||||||
let dir = PathBuf::from("world");
|
|
||||||
if !dir.join("actor").is_dir() {
|
|
||||||
println!("No world in ./world yet, creating it (same as `sausage init`).");
|
|
||||||
init(dir.clone()).await?;
|
|
||||||
}
|
|
||||||
run(dir).await
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+60
-94
@@ -4,7 +4,7 @@
|
|||||||
//! NDJSON lines to its stdin and reads NDJSON lines from its stdout.
|
//! NDJSON lines to its stdin and reads NDJSON lines from its stdout.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, Weak};
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
@@ -14,21 +14,22 @@ use tokio::task::JoinHandle;
|
|||||||
use wasmtime::component::{Component, Linker, ResourceTable};
|
use wasmtime::component::{Component, Linker, ResourceTable};
|
||||||
use wasmtime::{Config, Engine, Store};
|
use wasmtime::{Config, Engine, Store};
|
||||||
use wasmtime_wasi::cli::{AsyncStdinStream, AsyncStdoutStream};
|
use wasmtime_wasi::cli::{AsyncStdinStream, AsyncStdoutStream};
|
||||||
use wasmtime_wasi::filesystem::WasiFilesystemCtxView;
|
|
||||||
use wasmtime_wasi::p2::bindings::Command;
|
use wasmtime_wasi::p2::bindings::Command;
|
||||||
use wasmtime_wasi::p2::bindings::filesystem;
|
use wasmtime_wasi::p2::bindings::filesystem::{preopens, types};
|
||||||
use wasmtime_wasi::{FsPerms, I32Exit, WasiCtx, WasiCtxView, WasiView};
|
use wasmtime_wasi::{FsPerms, I32Exit, WasiCtx, WasiCtxView, WasiView};
|
||||||
use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, WasiHttpView};
|
use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpCtxView, WasiHttpHooks, WasiHttpView};
|
||||||
|
|
||||||
use crate::broker::Broker;
|
use crate::fs::{ActorFs, ActorFsData, FsExt, Mount};
|
||||||
use crate::vfs::{Vfs, VfsState, VfsView};
|
|
||||||
|
|
||||||
/// A host folder that the guest sees at `guest`.
|
pub struct WasiPart {
|
||||||
#[derive(Clone, Debug)]
|
ctx: WasiCtx,
|
||||||
pub struct Mount {
|
table: ResourceTable,
|
||||||
pub guest: String,
|
}
|
||||||
pub host: PathBuf,
|
|
||||||
pub writable: bool,
|
impl WasiView for WasiPart {
|
||||||
|
fn ctx(&mut self) -> WasiCtxView<'_> {
|
||||||
|
WasiCtxView { ctx: &mut self.ctx, table: &mut self.table }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct HttpHooks {
|
pub struct HttpHooks {
|
||||||
@@ -66,26 +67,28 @@ impl WasiHttpHooks for HttpHooks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct ActorState {
|
pub struct ActorState {
|
||||||
ctx: WasiCtx,
|
wasi: WasiPart,
|
||||||
table: ResourceTable,
|
fsx: FsExt,
|
||||||
http: WasiHttpCtx,
|
http: WasiHttpCtx,
|
||||||
hooks: HttpHooks,
|
hooks: HttpHooks,
|
||||||
vfs: VfsState,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn vfs_view(s: &mut ActorState) -> VfsView<'_> {
|
|
||||||
VfsView { inner: WasiFilesystemCtxView { ctx: s.ctx.filesystem(), table: &mut s.table }, state: &mut s.vfs }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WasiView for ActorState {
|
impl WasiView for ActorState {
|
||||||
fn ctx(&mut self) -> WasiCtxView<'_> {
|
fn ctx(&mut self) -> WasiCtxView<'_> {
|
||||||
WasiCtxView { ctx: &mut self.ctx, table: &mut self.table }
|
self.wasi.ctx()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WasiHttpView for ActorState {
|
impl WasiHttpView for ActorState {
|
||||||
fn http(&mut self) -> WasiHttpCtxView<'_> {
|
fn http(&mut self) -> WasiHttpCtxView<'_> {
|
||||||
WasiHttpCtxView { ctx: &mut self.http, table: &mut self.table, hooks: &mut self.hooks }
|
WasiHttpCtxView { ctx: &mut self.http, table: &mut self.wasi.table, hooks: &mut self.hooks }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn actor_fs(s: &mut ActorState) -> ActorFs<'_> {
|
||||||
|
ActorFs {
|
||||||
|
inner: wasmtime_wasi::filesystem::WasiFilesystemView::filesystem(&mut s.wasi),
|
||||||
|
ext: &mut s.fsx,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,6 +108,12 @@ pub enum Exit {
|
|||||||
Trap(String),
|
Trap(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Exit {
|
||||||
|
pub fn is_clean(&self) -> bool {
|
||||||
|
matches!(self, Exit::Clean)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for Exit {
|
impl std::fmt::Display for Exit {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
match self {
|
match self {
|
||||||
@@ -117,52 +126,24 @@ impl std::fmt::Display for Exit {
|
|||||||
|
|
||||||
pub struct InstanceConfig {
|
pub struct InstanceConfig {
|
||||||
pub mounts: Vec<Mount>,
|
pub mounts: Vec<Mount>,
|
||||||
/// Outgoing HTTP, and TCP, UDP, and DNS through `wasi:sockets`.
|
pub allow_http: bool,
|
||||||
pub network: bool,
|
|
||||||
pub stderr_log: PathBuf,
|
pub stderr_log: PathBuf,
|
||||||
pub env: Vec<(String, String)>,
|
pub env: Vec<(String, String)>,
|
||||||
/// The actor that runs in the instance.
|
|
||||||
pub actor: String,
|
|
||||||
/// Asks the actors that serve `/actor/<id>/fs/` (see `vfs.rs`).
|
|
||||||
pub broker: Weak<Broker>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A running guest. Dropping it kills the guest.
|
|
||||||
pub struct Instance {
|
pub struct Instance {
|
||||||
stdin: Option<DuplexStream>,
|
stdin: DuplexStream,
|
||||||
/// Lines that the guest writes to stdout. Closed when the guest ends.
|
/// Lines that the guest writes to stdout. Closed when the guest ends.
|
||||||
pub stdout: mpsc::Receiver<String>,
|
pub stdout: mpsc::Receiver<String>,
|
||||||
task: JoinHandle<Exit>,
|
task: JoinHandle<Exit>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for Instance {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.task.abort();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Instance {
|
impl Instance {
|
||||||
/// Write one line to the guest's stdin, in one write. Meanwhile, pass the
|
/// Write one line to the guest's stdin.
|
||||||
/// lines that the guest writes to `route`: a guest that blocks on a full
|
pub async fn send_line(&mut self, line: &str) -> std::io::Result<()> {
|
||||||
/// stdout would never read its stdin.
|
self.stdin.write_all(line.as_bytes()).await?;
|
||||||
pub async fn send_line(&mut self, line: &str, mut route: impl FnMut(String)) -> std::io::Result<()> {
|
self.stdin.write_all(b"\n").await?;
|
||||||
let stdin = self.stdin.as_mut().ok_or(std::io::ErrorKind::BrokenPipe)?;
|
self.stdin.flush().await
|
||||||
let bytes = format!("{line}\n");
|
|
||||||
let write = async {
|
|
||||||
stdin.write_all(bytes.as_bytes()).await?;
|
|
||||||
stdin.flush().await
|
|
||||||
};
|
|
||||||
tokio::pin!(write);
|
|
||||||
let mut open = true;
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
done = &mut write => return done,
|
|
||||||
out = self.stdout.recv(), if open => match out {
|
|
||||||
Some(out) => route(out),
|
|
||||||
None => open = false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Wait until the guest ends.
|
/// Wait until the guest ends.
|
||||||
@@ -178,35 +159,19 @@ impl Instance {
|
|||||||
self.task.is_finished()
|
self.task.is_finished()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kill the guest. Its stdout closes when it has ended.
|
/// Send `shutdown`, close stdin, and wait. Kill the guest after `deadline`.
|
||||||
pub fn kill(&self) {
|
pub async fn stop(mut self, shutdown_line: &str, deadline: Duration) -> Exit {
|
||||||
|
let _ = self.send_line(shutdown_line).await;
|
||||||
|
let _ = self.stdin.shutdown().await;
|
||||||
|
drop(self.stdin);
|
||||||
|
match tokio::time::timeout(deadline, &mut self.task).await {
|
||||||
|
Ok(Ok(exit)) => exit,
|
||||||
|
Ok(Err(e)) => Exit::Trap(format!("host task failed: {e}")),
|
||||||
|
Err(_) => {
|
||||||
self.task.abort();
|
self.task.abort();
|
||||||
|
let _ = (&mut self.task).await;
|
||||||
|
Exit::Trap("killed after shutdown deadline".into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send `shutdown_line`, close stdin, and wait until the guest ends.
|
|
||||||
/// Pass every line that the guest writes meanwhile to `route`. Kill the
|
|
||||||
/// guest when `deadline` has passed.
|
|
||||||
pub async fn stop(mut self, shutdown_line: &str, deadline: Duration, mut route: impl FnMut(String)) -> Exit {
|
|
||||||
let stdin = self.stdin.take();
|
|
||||||
let stdout = &mut self.stdout;
|
|
||||||
let finished = tokio::time::timeout(deadline, async {
|
|
||||||
let close = async {
|
|
||||||
if let Some(mut stdin) = stdin {
|
|
||||||
let _ = stdin.write_all(format!("{shutdown_line}\n").as_bytes()).await;
|
|
||||||
let _ = stdin.shutdown().await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let drain = async {
|
|
||||||
while let Some(line) = stdout.recv().await {
|
|
||||||
route(line);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
tokio::join!(close, drain);
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
match finished {
|
|
||||||
Ok(()) => self.wait().await,
|
|
||||||
Err(_) => Exit::Trap("killed after shutdown deadline".into()),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,10 +203,10 @@ impl Runtime {
|
|||||||
let mut linker: Linker<ActorState> = Linker::new(&engine);
|
let mut linker: Linker<ActorState> = Linker::new(&engine);
|
||||||
wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
|
wasmtime_wasi::p2::add_to_linker_async(&mut linker)?;
|
||||||
wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?;
|
wasmtime_wasi_http::p2::add_only_http_to_linker_async(&mut linker)?;
|
||||||
// Replace `wasi:filesystem` with the wrapper that serves `/actor/<id>/fs/`.
|
// Replace the filesystem with the commit-on-close layer.
|
||||||
linker.allow_shadowing(true);
|
linker.allow_shadowing(true);
|
||||||
filesystem::types::add_to_linker::<ActorState, Vfs>(&mut linker, vfs_view)?;
|
types::add_to_linker::<ActorState, ActorFsData>(&mut linker, actor_fs)?;
|
||||||
filesystem::preopens::add_to_linker::<ActorState, Vfs>(&mut linker, vfs_view)?;
|
preopens::add_to_linker::<ActorState, ActorFsData>(&mut linker, actor_fs)?;
|
||||||
linker.allow_shadowing(false);
|
linker.allow_shadowing(false);
|
||||||
|
|
||||||
// Time slices: every 10 ms each running guest yields to the scheduler.
|
// Time slices: every 10 ms each running guest yields to the scheduler.
|
||||||
@@ -273,9 +238,6 @@ impl Runtime {
|
|||||||
.with_context(|| format!("open {}", cfg.stderr_log.display()))?;
|
.with_context(|| format!("open {}", cfg.stderr_log.display()))?;
|
||||||
|
|
||||||
let mut b = WasiCtx::builder();
|
let mut b = WasiCtx::builder();
|
||||||
if cfg.network {
|
|
||||||
b.inherit_network().allow_tcp(true).allow_udp(true).allow_ip_name_lookup(true);
|
|
||||||
}
|
|
||||||
b.stdin(AsyncStdinStream::new(guest_in));
|
b.stdin(AsyncStdinStream::new(guest_in));
|
||||||
b.stdout(AsyncStdoutStream::new(8192, guest_out));
|
b.stdout(AsyncStdoutStream::new(8192, guest_out));
|
||||||
b.stderr(AsyncStdoutStream::new(8192, tokio::fs::File::from_std(stderr)));
|
b.stderr(AsyncStdoutStream::new(8192, tokio::fs::File::from_std(stderr)));
|
||||||
@@ -289,11 +251,10 @@ impl Runtime {
|
|||||||
.with_context(|| format!("mount {} at {}", m.host.display(), m.guest))?;
|
.with_context(|| format!("mount {} at {}", m.host.display(), m.guest))?;
|
||||||
}
|
}
|
||||||
let state = ActorState {
|
let state = ActorState {
|
||||||
ctx: b.build(),
|
wasi: WasiPart { ctx: b.build(), table: ResourceTable::new() },
|
||||||
table: ResourceTable::new(),
|
fsx: FsExt::new(cfg.mounts),
|
||||||
http: WasiHttpCtx::new(),
|
http: WasiHttpCtx::new(),
|
||||||
hooks: HttpHooks { allow: cfg.network },
|
hooks: HttpHooks { allow: cfg.allow_http },
|
||||||
vfs: VfsState::new(cfg.broker.clone(), cfg.actor.clone()),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let (line_tx, line_rx) = mpsc::channel(1024);
|
let (line_tx, line_rx) = mpsc::channel(1024);
|
||||||
@@ -317,7 +278,7 @@ impl Runtime {
|
|||||||
cmd.wasi_cli_run().call_run(&mut store).await
|
cmd.wasi_cli_run().call_run(&mut store).await
|
||||||
}
|
}
|
||||||
.await;
|
.await;
|
||||||
match result {
|
let exit = match result {
|
||||||
Ok(Ok(())) => Exit::Clean,
|
Ok(Ok(())) => Exit::Clean,
|
||||||
Ok(Err(())) => Exit::Code(1),
|
Ok(Err(())) => Exit::Code(1),
|
||||||
Err(e) => match e.downcast_ref::<I32Exit>() {
|
Err(e) => match e.downcast_ref::<I32Exit>() {
|
||||||
@@ -330,9 +291,14 @@ impl Runtime {
|
|||||||
Exit::Trap(short_trap(&full))
|
Exit::Trap(short_trap(&full))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
};
|
||||||
|
// Process exit closes every file, so a clean exit commits.
|
||||||
|
if exit.is_clean() {
|
||||||
|
store.data_mut().fsx.commit_all();
|
||||||
}
|
}
|
||||||
|
exit
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(Instance { stdin: Some(host_in), stdout: line_rx, task })
|
Ok(Instance { stdin: host_in, stdout: line_rx, task })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,287 +0,0 @@
|
|||||||
//! The supervisor of one WASM actor: it runs the instance, feeds it the
|
|
||||||
//! mailbox, routes its output, restarts it after a crash, and swaps versions.
|
|
||||||
//!
|
|
||||||
//! The broker talks to a supervisor only through `Control` requests.
|
|
||||||
|
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
|
|
||||||
use anyhow::{Result, bail};
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use tokio::sync::{mpsc, oneshot};
|
|
||||||
use wasmtime::component::Component;
|
|
||||||
|
|
||||||
use crate::broker::{Broker, Envelope, SHUTDOWN_DEADLINE, shutdown_line};
|
|
||||||
use crate::runtime::{Exit, Instance};
|
|
||||||
|
|
||||||
/// A new version that crashes this often within `CRASH_WINDOW` after an
|
|
||||||
/// upgrade is rolled back automatically.
|
|
||||||
const CRASH_LIMIT: usize = 3;
|
|
||||||
const CRASH_WINDOW: Duration = Duration::from_secs(30);
|
|
||||||
|
|
||||||
pub type Reply = oneshot::Sender<Result<Value>>;
|
|
||||||
|
|
||||||
pub enum Control {
|
|
||||||
/// Swap to `version` (already in `versions/`, compiled as `component`).
|
|
||||||
Upgrade { version: u64, component: Component, requester: String, reply: Reply },
|
|
||||||
Rollback { version: u64, reply: Reply },
|
|
||||||
Stop { reply: Reply },
|
|
||||||
Start { reply: Reply },
|
|
||||||
Shutdown { reply: Reply },
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The time after an upgrade in which crashes count toward a rollback.
|
|
||||||
struct Probation {
|
|
||||||
previous: u64,
|
|
||||||
since: Instant,
|
|
||||||
crashes: usize,
|
|
||||||
requester: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct Supervisor {
|
|
||||||
broker: Arc<Broker>,
|
|
||||||
id: String,
|
|
||||||
version: u64,
|
|
||||||
running: bool,
|
|
||||||
mailbox: mpsc::Receiver<Envelope>,
|
|
||||||
control: mpsc::Receiver<Control>,
|
|
||||||
component: Option<Component>,
|
|
||||||
instance: Option<Instance>,
|
|
||||||
started_at: Instant,
|
|
||||||
restarts: u32,
|
|
||||||
restart_at: Option<Instant>,
|
|
||||||
probation: Option<Probation>,
|
|
||||||
/// A control request that arrived during a blocked delivery.
|
|
||||||
next_control: Option<Control>,
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn next_line(instance: &mut Option<Instance>) -> Option<String> {
|
|
||||||
match instance {
|
|
||||||
Some(i) => i.stdout.recv().await,
|
|
||||||
None => std::future::pending().await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sleep_until(t: Option<Instant>) {
|
|
||||||
match t {
|
|
||||||
Some(t) => tokio::time::sleep_until(t.into()).await,
|
|
||||||
None => std::future::pending().await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Supervisor {
|
|
||||||
pub fn new(
|
|
||||||
broker: Arc<Broker>,
|
|
||||||
id: String,
|
|
||||||
version: u64,
|
|
||||||
running: bool,
|
|
||||||
mailbox: mpsc::Receiver<Envelope>,
|
|
||||||
control: mpsc::Receiver<Control>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
broker,
|
|
||||||
id,
|
|
||||||
version,
|
|
||||||
running,
|
|
||||||
mailbox,
|
|
||||||
control,
|
|
||||||
component: None,
|
|
||||||
instance: None,
|
|
||||||
started_at: Instant::now(),
|
|
||||||
restarts: 0,
|
|
||||||
restart_at: None,
|
|
||||||
probation: None,
|
|
||||||
next_control: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn run(mut self) {
|
|
||||||
loop {
|
|
||||||
if let Some(ctl) = self.next_control.take() {
|
|
||||||
if self.handle(ctl).await {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if self.running && self.instance.is_none() && self.restart_at.is_none_or(|t| Instant::now() >= t) {
|
|
||||||
self.restart_at = None;
|
|
||||||
let _ = self.start().await;
|
|
||||||
}
|
|
||||||
let has_instance = self.instance.is_some();
|
|
||||||
tokio::select! {
|
|
||||||
ctl = self.control.recv() => {
|
|
||||||
let Some(ctl) = ctl else { break };
|
|
||||||
if self.handle(ctl).await {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
msg = self.mailbox.recv(), if has_instance => {
|
|
||||||
let Some(msg) = msg else { break };
|
|
||||||
self.deliver(msg).await;
|
|
||||||
}
|
|
||||||
line = next_line(&mut self.instance) => match line {
|
|
||||||
Some(line) => self.broker.route_line(&self.id, &line),
|
|
||||||
None => {
|
|
||||||
let exit = self.instance.as_mut().unwrap().wait().await;
|
|
||||||
self.instance = None;
|
|
||||||
self.on_exit(exit).await;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
_ = sleep_until(self.restart_at), if !has_instance && self.running => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Write a message to the instance's stdin, and route its output
|
|
||||||
/// meanwhile. If a control request arrives before the actor reads the
|
|
||||||
/// message, the instance is killed (a partly written line would corrupt
|
|
||||||
/// the next message) and the request runs.
|
|
||||||
async fn deliver(&mut self, msg: Envelope) {
|
|
||||||
let Some(inst) = self.instance.as_mut() else { return };
|
|
||||||
let (broker, id, line) = (self.broker.clone(), self.id.clone(), msg.to_line());
|
|
||||||
let stuck = tokio::select! {
|
|
||||||
written = inst.send_line(&line, |l| broker.route_line(&id, &l)) => match written {
|
|
||||||
Ok(()) => return,
|
|
||||||
// The instance is ending. The main loop sees its closed stdout.
|
|
||||||
Err(_) => false,
|
|
||||||
},
|
|
||||||
Some(ctl) = self.control.recv() => {
|
|
||||||
self.next_control = Some(ctl);
|
|
||||||
true
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if msg.from != "broker" {
|
|
||||||
let body = json!({"type": "undeliverable", "reason": "the actor stopped", "to": self.id});
|
|
||||||
self.broker.send("broker", Some(&msg.from), None, Some(&msg.id), body);
|
|
||||||
}
|
|
||||||
if stuck && let Some(inst) = &self.instance {
|
|
||||||
self.broker.log(&self.id, "killed: it did not read stdin when the broker needed it");
|
|
||||||
inst.kill();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Start an instance of the current version. If that fails, the actor
|
|
||||||
/// stays stopped until the next start, upgrade, or rollback.
|
|
||||||
async fn start(&mut self) -> Result<()> {
|
|
||||||
let started = self.try_start();
|
|
||||||
match &started {
|
|
||||||
Ok(()) => self.started_at = Instant::now(),
|
|
||||||
Err(e) => {
|
|
||||||
self.broker.log(&self.id, &format!("cannot start version {}: {e:#}", self.version));
|
|
||||||
self.running = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
started
|
|
||||||
}
|
|
||||||
|
|
||||||
fn try_start(&mut self) -> Result<()> {
|
|
||||||
if self.component.is_none() {
|
|
||||||
self.component = Some(self.broker.load_version(&self.id, self.version)?);
|
|
||||||
}
|
|
||||||
let cfg = self.broker.instance_config(&self.id, self.version, None);
|
|
||||||
self.instance = Some(self.broker.runtime.start(self.component.as_ref().unwrap(), cfg)?);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Stop the instance gracefully. Its last messages still get routed.
|
|
||||||
async fn stop(&mut self) {
|
|
||||||
let Some(inst) = self.instance.take() else { return };
|
|
||||||
let (broker, id) = (self.broker.clone(), self.id.clone());
|
|
||||||
let exit = inst.stop(&shutdown_line(), SHUTDOWN_DEADLINE, |line| broker.route_line(&id, &line)).await;
|
|
||||||
self.broker.log(&self.id, &format!("stopped: {exit}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn on_exit(&mut self, exit: Exit) {
|
|
||||||
self.broker.log(&self.id, &format!("instance ended unexpectedly: {exit}"));
|
|
||||||
if let Some(p) = &mut self.probation
|
|
||||||
&& p.since.elapsed() < CRASH_WINDOW
|
|
||||||
{
|
|
||||||
p.crashes += 1;
|
|
||||||
if p.crashes >= CRASH_LIMIT {
|
|
||||||
let (from, to, requester) = (self.version, p.previous, p.requester.clone());
|
|
||||||
self.broker.log(&self.id, &format!("version {from} crashed {CRASH_LIMIT} times after the upgrade; rolling back to {to}"));
|
|
||||||
let body = match self.rollback(to).await {
|
|
||||||
Ok(_) => json!({"type": "rolled_back", "actor": self.id, "from": from, "to": to,
|
|
||||||
"reason": "crash loop after upgrade", "last_exit": exit.to_string()}),
|
|
||||||
Err(e) => json!({"type": "error", "actor": self.id, "reason": format!("automatic rollback failed: {e:#}")}),
|
|
||||||
};
|
|
||||||
self.broker.send("broker", Some(&requester), None, None, body);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Restart after 0.5 s, doubling up to 30 s; a minute of uptime resets it.
|
|
||||||
if self.started_at.elapsed() > Duration::from_secs(60) {
|
|
||||||
self.restarts = 0;
|
|
||||||
}
|
|
||||||
let delay = Duration::from_millis(500 << self.restarts.min(6)).min(Duration::from_secs(30));
|
|
||||||
self.restarts += 1;
|
|
||||||
self.restart_at = Some(Instant::now() + delay);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Make `version` current. It is compiled first (unless `component` is
|
|
||||||
/// given): if that fails, nothing changes. `data/` and `public/` stay as
|
|
||||||
/// they are.
|
|
||||||
async fn install(&mut self, version: u64, component: Option<Component>) -> Result<()> {
|
|
||||||
let component = match component {
|
|
||||||
Some(c) => c,
|
|
||||||
None => self.broker.load_version(&self.id, version)?,
|
|
||||||
};
|
|
||||||
self.stop().await;
|
|
||||||
self.broker.install(&self.id, version)?;
|
|
||||||
self.version = version;
|
|
||||||
self.component = Some(component);
|
|
||||||
self.running = true;
|
|
||||||
self.restarts = 0;
|
|
||||||
self.restart_at = None;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn upgrade(&mut self, version: u64, component: Component, requester: String) -> Result<Value> {
|
|
||||||
let previous = self.version;
|
|
||||||
self.install(version, Some(component)).await?;
|
|
||||||
self.probation = Some(Probation { previous, since: Instant::now(), crashes: 0, requester });
|
|
||||||
if let Err(e) = self.start().await {
|
|
||||||
self.rollback(previous).await?;
|
|
||||||
bail!("version {version} did not start ({e:#}); rolled back to {previous}");
|
|
||||||
}
|
|
||||||
Ok(json!({"type": "upgraded", "actor": self.id, "version": version, "previous": previous}))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn rollback(&mut self, version: u64) -> Result<Value> {
|
|
||||||
let from = self.version;
|
|
||||||
self.install(version, None).await?;
|
|
||||||
self.probation = None;
|
|
||||||
self.start().await?;
|
|
||||||
Ok(json!({"type": "rolled_back", "actor": self.id, "from": from, "to": version}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Handle a control request. Returns `true` when the supervisor ends.
|
|
||||||
async fn handle(&mut self, ctl: Control) -> bool {
|
|
||||||
match ctl {
|
|
||||||
Control::Upgrade { version, component, requester, reply } => {
|
|
||||||
let _ = reply.send(self.upgrade(version, component, requester).await);
|
|
||||||
}
|
|
||||||
Control::Rollback { version, reply } => {
|
|
||||||
let _ = reply.send(self.rollback(version).await);
|
|
||||||
}
|
|
||||||
Control::Stop { reply } => {
|
|
||||||
self.running = false;
|
|
||||||
self.stop().await;
|
|
||||||
let r = self.broker.set_running(&self.id, false);
|
|
||||||
let _ = reply.send(r.map(|_| json!({"type": "stopped", "actor": self.id})));
|
|
||||||
}
|
|
||||||
Control::Start { reply } => {
|
|
||||||
self.running = true;
|
|
||||||
self.restart_at = None;
|
|
||||||
let r = self.broker.set_running(&self.id, true);
|
|
||||||
let _ = reply.send(r.map(|_| json!({"type": "started", "actor": self.id})));
|
|
||||||
}
|
|
||||||
Control::Shutdown { reply } => {
|
|
||||||
self.stop().await;
|
|
||||||
let _ = reply.send(Ok(json!({"type": "shut_down"})));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-796
@@ -1,796 +0,0 @@
|
|||||||
//! Actor-served files: `/actor/<id>/fs/...` (see PROTOCOL.md, "Actor-served
|
|
||||||
//! files").
|
|
||||||
//!
|
|
||||||
//! This module wraps the `wasi:filesystem` implementation of `wasmtime-wasi`.
|
|
||||||
//! A path under `/actor/<id>/fs/` does not reach the disk: the host asks the
|
|
||||||
//! actor `<id>` with a message from `fs` and waits up to `ASK_TIMEOUT` for
|
|
||||||
//! the reply. Other paths go to `wasmtime-wasi` unchanged except that an
|
|
||||||
//! actor directory listing advertises `fs/` when its source docs declare it.
|
|
||||||
//!
|
|
||||||
//! - `fs_read {path}` -> `fs_file {text | base64}`, `fs_dir {entries: [{name, type}]}`
|
|
||||||
//! - `fs_write {path, text | base64}`, `fs_mkdir {path}`, `fs_remove {path}`,
|
|
||||||
//! `fs_rename {path, to}` -> `fs_done`
|
|
||||||
//! - any of them -> `fs_error {errno}` (a `wasi:filesystem` error-code name)
|
|
||||||
//!
|
|
||||||
//! A file is read once when it is opened. After a change, it is written back
|
|
||||||
//! in one `fs_write` when it is synced or closed.
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::hash::{Hash, Hasher};
|
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::{Arc, Mutex, Weak};
|
|
||||||
|
|
||||||
use bytes::Bytes;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use wasmtime::component::{HasData, Resource};
|
|
||||||
use wasmtime_wasi::filesystem::WasiFilesystemCtxView;
|
|
||||||
use wasmtime_wasi::p2::bindings::filesystem::types::{
|
|
||||||
self, DescriptorFlags, DescriptorType, DirectoryEntry, ErrorCode, HostDescriptor, HostDirectoryEntryStream, OpenFlags,
|
|
||||||
};
|
|
||||||
use wasmtime_wasi::p2::bindings::filesystem::preopens;
|
|
||||||
use wasmtime_wasi::p2::pipe::MemoryInputPipe;
|
|
||||||
use wasmtime_wasi::p2::{DynInputStream, DynOutputStream, FsError, FsResult, OutputStream, Pollable, StreamResult};
|
|
||||||
|
|
||||||
use crate::broker::Broker;
|
|
||||||
|
|
||||||
/// The `from` of the host's requests. Replies to it go back to the waiting call.
|
|
||||||
pub const ID: &str = "fs";
|
|
||||||
pub const ASK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
|
||||||
|
|
||||||
/// Per-instance state.
|
|
||||||
pub struct VfsState {
|
|
||||||
broker: Weak<Broker>,
|
|
||||||
/// The actor of this instance, for notes in its `stderr.log`.
|
|
||||||
me: String,
|
|
||||||
/// Native directory descriptors and their absolute paths. Keeping every
|
|
||||||
/// `/actor` child (not only the preopen) lets `fs/` work after traversal.
|
|
||||||
dirs: HashMap<u32, String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl VfsState {
|
|
||||||
pub fn new(broker: Weak<Broker>, me: String) -> Self {
|
|
||||||
Self { broker, me, dirs: HashMap::new() }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// [`HasData`] for the linker: the filesystem functions get a [`VfsView`].
|
|
||||||
pub struct Vfs;
|
|
||||||
|
|
||||||
impl HasData for Vfs {
|
|
||||||
type Data<'a> = VfsView<'a>;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct VfsView<'a> {
|
|
||||||
pub inner: WasiFilesystemCtxView<'a>,
|
|
||||||
pub state: &'a mut VfsState,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A descriptor of an actor-served file or folder (in the resource table in
|
|
||||||
/// place of a `wasmtime-wasi` descriptor).
|
|
||||||
#[derive(Clone)]
|
|
||||||
enum Virt {
|
|
||||||
Dir { actor: String, path: String },
|
|
||||||
File(VFile),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
struct VFile {
|
|
||||||
actor: String,
|
|
||||||
path: String,
|
|
||||||
data: Arc<Mutex<Vec<u8>>>,
|
|
||||||
dirty: Arc<AtomicBool>,
|
|
||||||
write: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A directory-entry-stream of an actor-served folder.
|
|
||||||
struct VDir(std::vec::IntoIter<DirectoryEntry>);
|
|
||||||
|
|
||||||
enum Node {
|
|
||||||
File(Vec<u8>),
|
|
||||||
Dir(Vec<DirectoryEntry>),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sends the requests. Owned, so that no borrow of the view lives across an await.
|
|
||||||
struct Asker {
|
|
||||||
broker: Option<Arc<Broker>>,
|
|
||||||
me: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Asker {
|
|
||||||
async fn ask(&self, actor: &str, body: Value) -> Result<Value, ErrorCode> {
|
|
||||||
let broker = self.broker.as_ref().ok_or(ErrorCode::Io)?;
|
|
||||||
let what = format!("{} {} to {actor}", body["type"].as_str().unwrap_or(""), body["path"].as_str().unwrap_or(""));
|
|
||||||
match broker.ask(actor, body).await {
|
|
||||||
None => {
|
|
||||||
broker.log(&self.me, &format!("{what}: no reply within {} s", ASK_TIMEOUT.as_secs()));
|
|
||||||
Err(ErrorCode::Io)
|
|
||||||
}
|
|
||||||
Some(reply) => match reply["type"].as_str() {
|
|
||||||
Some("fs_error") => Err(errno(&reply["errno"])),
|
|
||||||
Some("undeliverable") => Err(ErrorCode::NoEntry),
|
|
||||||
_ => Ok(reply),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read(&self, actor: &str, path: &str) -> Result<Node, ErrorCode> {
|
|
||||||
let reply = self.ask(actor, json!({"type": "fs_read", "path": path})).await?;
|
|
||||||
match reply["type"].as_str() {
|
|
||||||
Some("fs_file") => content(&reply).map(Node::File),
|
|
||||||
Some("fs_dir") => Ok(Node::Dir(
|
|
||||||
reply["entries"]
|
|
||||||
.as_array()
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.filter_map(|e| {
|
|
||||||
let type_ = if e["type"] == "dir" { DescriptorType::Directory } else { DescriptorType::RegularFile };
|
|
||||||
Some(DirectoryEntry { type_, name: e["name"].as_str()?.to_string() })
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
)),
|
|
||||||
_ => {
|
|
||||||
if let Some(b) = &self.broker {
|
|
||||||
b.log(&self.me, &format!("fs_read {path} to {actor}: unexpected reply {reply}"));
|
|
||||||
}
|
|
||||||
Err(ErrorCode::Io)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn change(&self, actor: &str, body: Value) -> FsResult<()> {
|
|
||||||
self.ask(actor, body).await.map(|_| ()).map_err(FsError::from)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn flush(&self, f: &VFile) -> FsResult<()> {
|
|
||||||
if f.dirty.swap(false, Ordering::SeqCst)
|
|
||||||
&& let Err(e) = self.ask(&f.actor, write_body(f)).await
|
|
||||||
{
|
|
||||||
f.dirty.store(true, Ordering::SeqCst);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn errno(v: &Value) -> ErrorCode {
|
|
||||||
match v.as_str().unwrap_or("") {
|
|
||||||
"no-entry" => ErrorCode::NoEntry,
|
|
||||||
"exist" => ErrorCode::Exist,
|
|
||||||
"not-directory" => ErrorCode::NotDirectory,
|
|
||||||
"is-directory" => ErrorCode::IsDirectory,
|
|
||||||
"not-empty" => ErrorCode::NotEmpty,
|
|
||||||
"access" => ErrorCode::Access,
|
|
||||||
"not-permitted" => ErrorCode::NotPermitted,
|
|
||||||
"invalid" => ErrorCode::Invalid,
|
|
||||||
"unsupported" => ErrorCode::Unsupported,
|
|
||||||
"busy" => ErrorCode::Busy,
|
|
||||||
_ => ErrorCode::Io,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn content(v: &Value) -> Result<Vec<u8>, ErrorCode> {
|
|
||||||
if let Some(text) = v["text"].as_str() {
|
|
||||||
return Ok(text.as_bytes().to_vec());
|
|
||||||
}
|
|
||||||
match v["base64"].as_str() {
|
|
||||||
Some(b) => base64_decode(b).ok_or(ErrorCode::IllegalByteSequence),
|
|
||||||
None => Ok(Vec::new()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_body(f: &VFile) -> Value {
|
|
||||||
let data = f.data.lock().unwrap().clone();
|
|
||||||
match String::from_utf8(data) {
|
|
||||||
Ok(text) => json!({"type": "fs_write", "path": f.path, "text": text}),
|
|
||||||
Err(e) => json!({"type": "fs_write", "path": f.path, "base64": base64_encode(e.as_bytes())}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `rel` resolved against the folder `base`, as an absolute path.
|
|
||||||
fn join(base: &str, rel: &str) -> String {
|
|
||||||
let mut parts: Vec<&str> = base.split('/').filter(|p| !p.is_empty()).collect();
|
|
||||||
for p in rel.split('/') {
|
|
||||||
match p {
|
|
||||||
"" | "." => {}
|
|
||||||
".." => {
|
|
||||||
parts.pop();
|
|
||||||
}
|
|
||||||
p => parts.push(p),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
format!("/{}", parts.join("/"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Split a global path under an actor's virtual `fs/` into its owner and
|
|
||||||
/// actor-relative path.
|
|
||||||
fn actor_fs_path(full: &str) -> Option<(String, String)> {
|
|
||||||
let rest = full.strip_prefix("/actor/")?;
|
|
||||||
let (actor, rest) = rest.split_once('/')?;
|
|
||||||
let (fs, path) = rest.split_once('/').unwrap_or((rest, ""));
|
|
||||||
(fs == "fs" && !actor.is_empty()).then(|| (actor.to_string(), format!("/{path}")))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn actor_root(path: &str) -> Option<&str> {
|
|
||||||
let actor = path.strip_prefix("/actor/")?;
|
|
||||||
(!actor.is_empty() && !actor.contains('/')).then_some(actor)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// An actor advertises its virtual filesystem in its leading `//!` docs.
|
|
||||||
/// This is discovery metadata; actual reads still go through `fs_read`.
|
|
||||||
fn declares_actor_fs(source: &str) -> bool {
|
|
||||||
let docs = source
|
|
||||||
.lines()
|
|
||||||
.take_while(|line| line.trim().starts_with("//!"))
|
|
||||||
.map(|line| line.trim().trim_start_matches("//!").trim())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(" ")
|
|
||||||
.to_ascii_lowercase();
|
|
||||||
docs.contains("actor-served files") && docs.contains("fs_read")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn stat(type_: DescriptorType, size: u64) -> types::DescriptorStat {
|
|
||||||
types::DescriptorStat {
|
|
||||||
type_,
|
|
||||||
link_count: 1,
|
|
||||||
size,
|
|
||||||
data_access_timestamp: None,
|
|
||||||
data_modification_timestamp: None,
|
|
||||||
status_change_timestamp: None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn node_stat(node: &Node) -> types::DescriptorStat {
|
|
||||||
match node {
|
|
||||||
Node::File(data) => stat(DescriptorType::RegularFile, data.len() as u64),
|
|
||||||
Node::Dir(_) => stat(DescriptorType::Directory, 0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn hash(actor: &str, path: &str) -> types::MetadataHashValue {
|
|
||||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
|
||||||
(actor, path).hash(&mut h);
|
|
||||||
types::MetadataHashValue { lower: h.finish(), upper: 0 }
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_at(data: &mut Vec<u8>, at: usize, bytes: &[u8]) {
|
|
||||||
if data.len() < at + bytes.len() {
|
|
||||||
data.resize(at + bytes.len(), 0);
|
|
||||||
}
|
|
||||||
data[at..at + bytes.len()].copy_from_slice(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// An output stream into an actor-served file (`pos: None` appends).
|
|
||||||
struct VWriter {
|
|
||||||
file: VFile,
|
|
||||||
pos: Option<usize>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[wasmtime_wasi::async_trait]
|
|
||||||
impl Pollable for VWriter {
|
|
||||||
async fn ready(&mut self) {}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[wasmtime_wasi::async_trait]
|
|
||||||
impl OutputStream for VWriter {
|
|
||||||
fn write(&mut self, bytes: Bytes) -> StreamResult<()> {
|
|
||||||
let mut data = self.file.data.lock().unwrap();
|
|
||||||
let at = self.pos.unwrap_or(data.len());
|
|
||||||
write_at(&mut data, at, &bytes);
|
|
||||||
if let Some(p) = &mut self.pos {
|
|
||||||
*p += bytes.len();
|
|
||||||
}
|
|
||||||
self.file.dirty.store(true, Ordering::SeqCst);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
fn flush(&mut self) -> StreamResult<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
fn check_write(&mut self) -> StreamResult<usize> {
|
|
||||||
Ok(64 * 1024)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl VfsView<'_> {
|
|
||||||
fn asker(&self) -> Asker {
|
|
||||||
Asker { broker: self.state.broker.upgrade(), me: self.state.me.clone() }
|
|
||||||
}
|
|
||||||
|
|
||||||
fn actor_serves_fs(&self, actor: &str) -> bool {
|
|
||||||
self.state
|
|
||||||
.broker
|
|
||||||
.upgrade()
|
|
||||||
.and_then(|broker| std::fs::read_to_string(broker.world.source_dir(actor).join("src/main.rs")).ok())
|
|
||||||
.is_some_and(|source| declares_actor_fs(&source))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn virt(&mut self, fd: &Resource<types::Descriptor>) -> Option<Virt> {
|
|
||||||
self.inner.table.get_any_mut(fd.rep()).ok()?.downcast_ref::<Virt>().cloned()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn file(&mut self, fd: &Resource<types::Descriptor>) -> Option<FsResult<VFile>> {
|
|
||||||
match self.virt(fd)? {
|
|
||||||
Virt::File(f) => Some(Ok(f)),
|
|
||||||
Virt::Dir { .. } => Some(Err(ErrorCode::IsDirectory.into())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The actor and its path, if `path` relative to `fd` is actor-served.
|
|
||||||
fn target(&mut self, fd: &Resource<types::Descriptor>, path: &str) -> Option<(String, String)> {
|
|
||||||
match self.virt(fd) {
|
|
||||||
Some(Virt::Dir { actor, path: base }) => Some((actor, join(&base, path))),
|
|
||||||
Some(Virt::File(f)) => Some((f.actor, join(&f.path, path))),
|
|
||||||
None => {
|
|
||||||
let base = self.state.dirs.get(&fd.rep())?;
|
|
||||||
actor_fs_path(&join(base, path))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn native_directory_entries(&mut self, fd: Resource<types::Descriptor>) -> FsResult<Vec<DirectoryEntry>> {
|
|
||||||
let stream = HostDescriptor::read_directory(&mut self.inner, fd).await?;
|
|
||||||
let stream_rep = stream.rep();
|
|
||||||
let mut entries = Vec::new();
|
|
||||||
loop {
|
|
||||||
match HostDirectoryEntryStream::read_directory_entry(&mut self.inner, Resource::new_borrow(stream_rep)).await? {
|
|
||||||
Some(entry) => entries.push(entry),
|
|
||||||
None => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
HostDirectoryEntryStream::drop(&mut self.inner, stream).map_err(|_| ErrorCode::Io)?;
|
|
||||||
Ok(entries)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn push(&mut self, v: Virt) -> FsResult<Resource<types::Descriptor>> {
|
|
||||||
let r = self.inner.table.push(v)?;
|
|
||||||
Ok(Resource::new_own(r.rep()))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn open(&mut self, actor: String, path: String, oflags: OpenFlags, flags: DescriptorFlags) -> FsResult<Resource<types::Descriptor>> {
|
|
||||||
let write = flags.contains(DescriptorFlags::WRITE);
|
|
||||||
let create = oflags.contains(OpenFlags::CREATE);
|
|
||||||
let exclusive = create && oflags.contains(OpenFlags::EXCLUSIVE);
|
|
||||||
let virt = match self.asker().read(&actor, &path).await {
|
|
||||||
Ok(_) if exclusive => return Err(ErrorCode::Exist.into()),
|
|
||||||
Ok(Node::Dir(_)) if write => return Err(ErrorCode::IsDirectory.into()),
|
|
||||||
Ok(Node::Dir(_)) => Virt::Dir { actor, path },
|
|
||||||
Ok(Node::File(_)) if oflags.contains(OpenFlags::DIRECTORY) => return Err(ErrorCode::NotDirectory.into()),
|
|
||||||
Ok(Node::File(data)) => {
|
|
||||||
let truncate = oflags.contains(OpenFlags::TRUNCATE);
|
|
||||||
let data = if truncate { Vec::new() } else { data };
|
|
||||||
Virt::File(VFile { actor, path, data: Arc::new(Mutex::new(data)), dirty: Arc::new(AtomicBool::new(truncate)), write })
|
|
||||||
}
|
|
||||||
Err(ErrorCode::NoEntry) if create && !oflags.contains(OpenFlags::DIRECTORY) => Virt::File(VFile {
|
|
||||||
actor,
|
|
||||||
path,
|
|
||||||
data: Arc::default(),
|
|
||||||
dirty: Arc::new(AtomicBool::new(true)),
|
|
||||||
write,
|
|
||||||
}),
|
|
||||||
Err(e) => return Err(e.into()),
|
|
||||||
};
|
|
||||||
self.push(virt)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Send the changed file without waiting (the descriptor is gone). The
|
|
||||||
/// request leaves now, so later requests of this instance come after it.
|
|
||||||
fn write_back_later(&self, f: &VFile) {
|
|
||||||
let Some(broker) = self.state.broker.upgrade() else { return };
|
|
||||||
let reply = broker.ask(&f.actor, write_body(f));
|
|
||||||
let (me, what) = (self.state.me.clone(), format!("fs_write {} to {}", f.path, f.actor));
|
|
||||||
tokio::spawn(async move {
|
|
||||||
match reply.await {
|
|
||||||
Some(r) if r["type"] != "fs_error" && r["type"] != "undeliverable" => {}
|
|
||||||
r => broker.log(&me, &format!("{what} failed: {}", r.map_or("no reply".to_string(), |r| r.to_string()))),
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl preopens::Host for VfsView<'_> {
|
|
||||||
fn get_directories(&mut self) -> wasmtime::Result<Vec<(Resource<types::Descriptor>, String)>> {
|
|
||||||
let dirs = preopens::Host::get_directories(&mut self.inner)?;
|
|
||||||
for (fd, name) in &dirs {
|
|
||||||
if name == "/actor" {
|
|
||||||
self.state.dirs.insert(fd.rep(), "/actor".to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(dirs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl types::Host for VfsView<'_> {
|
|
||||||
fn convert_error_code(&mut self, err: FsError) -> wasmtime::Result<ErrorCode> {
|
|
||||||
types::Host::convert_error_code(&mut self.inner, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn filesystem_error_code(&mut self, err: Resource<wasmtime::Error>) -> wasmtime::Result<Option<ErrorCode>> {
|
|
||||||
types::Host::filesystem_error_code(&mut self.inner, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl HostDescriptor for VfsView<'_> {
|
|
||||||
async fn advise(&mut self, fd: Resource<types::Descriptor>, offset: types::Filesize, len: types::Filesize, advice: types::Advice) -> FsResult<()> {
|
|
||||||
if self.virt(&fd).is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
HostDescriptor::advise(&mut self.inner, fd, offset, len, advice).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sync_data(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::File(f)) => self.asker().flush(&f).await,
|
|
||||||
Some(Virt::Dir { .. }) => Ok(()),
|
|
||||||
None => HostDescriptor::sync_data(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_flags(&mut self, fd: Resource<types::Descriptor>) -> FsResult<DescriptorFlags> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::File(f)) if f.write => Ok(DescriptorFlags::READ | DescriptorFlags::WRITE),
|
|
||||||
Some(Virt::File(_)) => Ok(DescriptorFlags::READ),
|
|
||||||
Some(Virt::Dir { .. }) => Ok(DescriptorFlags::READ | DescriptorFlags::MUTATE_DIRECTORY),
|
|
||||||
None => HostDescriptor::get_flags(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_type(&mut self, fd: Resource<types::Descriptor>) -> FsResult<DescriptorType> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::File(_)) => Ok(DescriptorType::RegularFile),
|
|
||||||
Some(Virt::Dir { .. }) => Ok(DescriptorType::Directory),
|
|
||||||
None => HostDescriptor::get_type(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn set_size(&mut self, fd: Resource<types::Descriptor>, size: types::Filesize) -> FsResult<()> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
if !f.write {
|
|
||||||
return Err(ErrorCode::NotPermitted.into());
|
|
||||||
}
|
|
||||||
f.data.lock().unwrap().resize(size as usize, 0);
|
|
||||||
f.dirty.store(true, Ordering::SeqCst);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
None => HostDescriptor::set_size(&mut self.inner, fd, size).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn set_times(&mut self, fd: Resource<types::Descriptor>, atim: types::NewTimestamp, mtim: types::NewTimestamp) -> FsResult<()> {
|
|
||||||
if self.virt(&fd).is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
HostDescriptor::set_times(&mut self.inner, fd, atim, mtim).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read(&mut self, fd: Resource<types::Descriptor>, len: types::Filesize, offset: types::Filesize) -> FsResult<(Vec<u8>, bool)> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
let data = f.data.lock().unwrap();
|
|
||||||
let start = usize::try_from(offset).unwrap_or(usize::MAX).min(data.len());
|
|
||||||
let end = start.saturating_add(usize::try_from(len).unwrap_or(usize::MAX)).min(data.len());
|
|
||||||
Ok((data[start..end].to_vec(), start == end))
|
|
||||||
}
|
|
||||||
None => HostDescriptor::read(&mut self.inner, fd, len, offset).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn write(&mut self, fd: Resource<types::Descriptor>, buf: Vec<u8>, offset: types::Filesize) -> FsResult<types::Filesize> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
if !f.write {
|
|
||||||
return Err(ErrorCode::NotPermitted.into());
|
|
||||||
}
|
|
||||||
write_at(&mut f.data.lock().unwrap(), offset as usize, &buf);
|
|
||||||
f.dirty.store(true, Ordering::SeqCst);
|
|
||||||
Ok(buf.len() as u64)
|
|
||||||
}
|
|
||||||
None => HostDescriptor::write(&mut self.inner, fd, buf, offset).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_directory(&mut self, fd: Resource<types::Descriptor>) -> FsResult<Resource<types::DirectoryEntryStream>> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::Dir { actor, path }) => match self.asker().read(&actor, &path).await? {
|
|
||||||
Node::Dir(entries) => {
|
|
||||||
let r = self.inner.table.push(VDir(entries.into_iter()))?;
|
|
||||||
Ok(Resource::new_own(r.rep()))
|
|
||||||
}
|
|
||||||
Node::File(_) => Err(ErrorCode::NotDirectory.into()),
|
|
||||||
},
|
|
||||||
Some(Virt::File(_)) => Err(ErrorCode::NotDirectory.into()),
|
|
||||||
None => {
|
|
||||||
let actor = self.state.dirs.get(&fd.rep()).and_then(|path| actor_root(path)).map(str::to_string);
|
|
||||||
if actor.as_deref().is_some_and(|id| self.actor_serves_fs(id)) {
|
|
||||||
let mut entries = self.native_directory_entries(fd).await?;
|
|
||||||
if !entries.iter().any(|entry| entry.name == "fs") {
|
|
||||||
entries.push(DirectoryEntry { type_: DescriptorType::Directory, name: "fs".to_string() });
|
|
||||||
}
|
|
||||||
let r = self.inner.table.push(VDir(entries.into_iter()))?;
|
|
||||||
Ok(Resource::new_own(r.rep()))
|
|
||||||
} else {
|
|
||||||
HostDescriptor::read_directory(&mut self.inner, fd).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sync(&mut self, fd: Resource<types::Descriptor>) -> FsResult<()> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::File(f)) => self.asker().flush(&f).await,
|
|
||||||
Some(Virt::Dir { .. }) => Ok(()),
|
|
||||||
None => HostDescriptor::sync(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn create_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => self.asker().change(&actor, json!({"type": "fs_mkdir", "path": path})).await,
|
|
||||||
None => HostDescriptor::create_directory_at(&mut self.inner, fd, path).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn stat(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::DescriptorStat> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::File(f)) => Ok(stat(DescriptorType::RegularFile, f.data.lock().unwrap().len() as u64)),
|
|
||||||
Some(Virt::Dir { .. }) => Ok(stat(DescriptorType::Directory, 0)),
|
|
||||||
None => HostDescriptor::stat(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn stat_at(&mut self, fd: Resource<types::Descriptor>, path_flags: types::PathFlags, path: String) -> FsResult<types::DescriptorStat> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => Ok(node_stat(&self.asker().read(&actor, &path).await?)),
|
|
||||||
None => HostDescriptor::stat_at(&mut self.inner, fd, path_flags, path).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn set_times_at(
|
|
||||||
&mut self,
|
|
||||||
fd: Resource<types::Descriptor>,
|
|
||||||
path_flags: types::PathFlags,
|
|
||||||
path: String,
|
|
||||||
atim: types::NewTimestamp,
|
|
||||||
mtim: types::NewTimestamp,
|
|
||||||
) -> FsResult<()> {
|
|
||||||
if self.target(&fd, &path).is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
HostDescriptor::set_times_at(&mut self.inner, fd, path_flags, path, atim, mtim).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn link_at(
|
|
||||||
&mut self,
|
|
||||||
fd: Resource<types::Descriptor>,
|
|
||||||
old_path_flags: types::PathFlags,
|
|
||||||
old_path: String,
|
|
||||||
new_descriptor: Resource<types::Descriptor>,
|
|
||||||
new_path: String,
|
|
||||||
) -> FsResult<()> {
|
|
||||||
if self.target(&fd, &old_path).is_some() || self.target(&new_descriptor, &new_path).is_some() {
|
|
||||||
return Err(ErrorCode::Unsupported.into());
|
|
||||||
}
|
|
||||||
HostDescriptor::link_at(&mut self.inner, fd, old_path_flags, old_path, new_descriptor, new_path).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn open_at(
|
|
||||||
&mut self,
|
|
||||||
fd: Resource<types::Descriptor>,
|
|
||||||
path_flags: types::PathFlags,
|
|
||||||
path: String,
|
|
||||||
oflags: OpenFlags,
|
|
||||||
flags: DescriptorFlags,
|
|
||||||
) -> FsResult<Resource<types::Descriptor>> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => self.open(actor, path, oflags, flags).await,
|
|
||||||
None => {
|
|
||||||
let child_path = self.state.dirs.get(&fd.rep()).map(|base| join(base, &path));
|
|
||||||
let opened = HostDescriptor::open_at(&mut self.inner, fd, path_flags, path, oflags, flags).await?;
|
|
||||||
if let Some(path) = child_path {
|
|
||||||
if HostDescriptor::get_type(&mut self.inner, Resource::new_borrow(opened.rep())).await? == DescriptorType::Directory {
|
|
||||||
self.state.dirs.insert(opened.rep(), path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(opened)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn drop(&mut self, fd: Resource<types::Descriptor>) -> wasmtime::Result<()> {
|
|
||||||
if self.virt(&fd).is_some() {
|
|
||||||
if let Virt::File(f) = self.inner.table.delete(Resource::<Virt>::new_own(fd.rep()))?
|
|
||||||
&& f.dirty.load(Ordering::SeqCst)
|
|
||||||
{
|
|
||||||
self.write_back_later(&f);
|
|
||||||
}
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
self.state.dirs.remove(&fd.rep());
|
|
||||||
HostDescriptor::drop(&mut self.inner, fd)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn readlink_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<String> {
|
|
||||||
if self.target(&fd, &path).is_some() {
|
|
||||||
return Err(ErrorCode::Invalid.into());
|
|
||||||
}
|
|
||||||
HostDescriptor::readlink_at(&mut self.inner, fd, path).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn remove_directory_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => self.asker().change(&actor, json!({"type": "fs_remove", "path": path})).await,
|
|
||||||
None => HostDescriptor::remove_directory_at(&mut self.inner, fd, path).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn rename_at(&mut self, fd: Resource<types::Descriptor>, old_path: String, new_fd: Resource<types::Descriptor>, new_path: String) -> FsResult<()> {
|
|
||||||
match (self.target(&fd, &old_path), self.target(&new_fd, &new_path)) {
|
|
||||||
(None, None) => HostDescriptor::rename_at(&mut self.inner, fd, old_path, new_fd, new_path).await,
|
|
||||||
(Some((a, from)), Some((b, to))) if a == b => {
|
|
||||||
self.asker().change(&a, json!({"type": "fs_rename", "path": from, "to": to})).await
|
|
||||||
}
|
|
||||||
_ => Err(ErrorCode::Unsupported.into()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn symlink_at(&mut self, fd: Resource<types::Descriptor>, src_path: String, dest_path: String) -> FsResult<()> {
|
|
||||||
if self.target(&fd, &dest_path).is_some() {
|
|
||||||
return Err(ErrorCode::Unsupported.into());
|
|
||||||
}
|
|
||||||
HostDescriptor::symlink_at(&mut self.inner, fd, src_path, dest_path).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn unlink_file_at(&mut self, fd: Resource<types::Descriptor>, path: String) -> FsResult<()> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => self.asker().change(&actor, json!({"type": "fs_remove", "path": path})).await,
|
|
||||||
None => HostDescriptor::unlink_file_at(&mut self.inner, fd, path).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read_via_stream(&mut self, fd: Resource<types::Descriptor>, offset: types::Filesize) -> FsResult<Resource<DynInputStream>> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
let data = f.data.lock().unwrap();
|
|
||||||
let start = usize::try_from(offset).unwrap_or(usize::MAX).min(data.len());
|
|
||||||
let stream: DynInputStream = Box::new(MemoryInputPipe::new(data[start..].to_vec()));
|
|
||||||
std::mem::drop(data);
|
|
||||||
Ok(self.inner.table.push(stream)?)
|
|
||||||
}
|
|
||||||
None => HostDescriptor::read_via_stream(&mut self.inner, fd, offset),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_via_stream(&mut self, fd: Resource<types::Descriptor>, offset: types::Filesize) -> FsResult<Resource<DynOutputStream>> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
if !f.write {
|
|
||||||
return Err(ErrorCode::NotPermitted.into());
|
|
||||||
}
|
|
||||||
let stream: DynOutputStream = Box::new(VWriter { file: f, pos: Some(offset as usize) });
|
|
||||||
Ok(self.inner.table.push(stream)?)
|
|
||||||
}
|
|
||||||
None => HostDescriptor::write_via_stream(&mut self.inner, fd, offset),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn append_via_stream(&mut self, fd: Resource<types::Descriptor>) -> FsResult<Resource<DynOutputStream>> {
|
|
||||||
match self.file(&fd) {
|
|
||||||
Some(f) => {
|
|
||||||
let f = f?;
|
|
||||||
if !f.write {
|
|
||||||
return Err(ErrorCode::NotPermitted.into());
|
|
||||||
}
|
|
||||||
let stream: DynOutputStream = Box::new(VWriter { file: f, pos: None });
|
|
||||||
Ok(self.inner.table.push(stream)?)
|
|
||||||
}
|
|
||||||
None => HostDescriptor::append_via_stream(&mut self.inner, fd),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn is_same_object(&mut self, a: Resource<types::Descriptor>, b: Resource<types::Descriptor>) -> wasmtime::Result<bool> {
|
|
||||||
let key = |v: Virt| match v {
|
|
||||||
Virt::Dir { actor, path } => (actor, path),
|
|
||||||
Virt::File(f) => (f.actor, f.path),
|
|
||||||
};
|
|
||||||
match (self.virt(&a), self.virt(&b)) {
|
|
||||||
(None, None) => HostDescriptor::is_same_object(&mut self.inner, a, b).await,
|
|
||||||
(Some(x), Some(y)) => Ok(key(x) == key(y)),
|
|
||||||
_ => Ok(false),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn metadata_hash(&mut self, fd: Resource<types::Descriptor>) -> FsResult<types::MetadataHashValue> {
|
|
||||||
match self.virt(&fd) {
|
|
||||||
Some(Virt::Dir { actor, path }) => Ok(hash(&actor, &path)),
|
|
||||||
Some(Virt::File(f)) => Ok(hash(&f.actor, &f.path)),
|
|
||||||
None => HostDescriptor::metadata_hash(&mut self.inner, fd).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn metadata_hash_at(&mut self, fd: Resource<types::Descriptor>, path_flags: types::PathFlags, path: String) -> FsResult<types::MetadataHashValue> {
|
|
||||||
match self.target(&fd, &path) {
|
|
||||||
Some((actor, path)) => Ok(hash(&actor, &path)),
|
|
||||||
None => HostDescriptor::metadata_hash_at(&mut self.inner, fd, path_flags, path).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl HostDirectoryEntryStream for VfsView<'_> {
|
|
||||||
async fn read_directory_entry(&mut self, stream: Resource<types::DirectoryEntryStream>) -> FsResult<Option<DirectoryEntry>> {
|
|
||||||
if let Some(dir) = self.inner.table.get_any_mut(stream.rep()).ok().and_then(|a| a.downcast_mut::<VDir>()) {
|
|
||||||
return Ok(dir.0.next());
|
|
||||||
}
|
|
||||||
HostDirectoryEntryStream::read_directory_entry(&mut self.inner, stream).await
|
|
||||||
}
|
|
||||||
|
|
||||||
fn drop(&mut self, stream: Resource<types::DirectoryEntryStream>) -> wasmtime::Result<()> {
|
|
||||||
let ours = self.inner.table.get_any_mut(stream.rep()).ok().is_some_and(|a| a.is::<VDir>());
|
|
||||||
if ours {
|
|
||||||
self.inner.table.delete(Resource::<VDir>::new_own(stream.rep()))?;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
HostDirectoryEntryStream::drop(&mut self.inner, stream)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
||||||
|
|
||||||
fn base64_encode(bytes: &[u8]) -> String {
|
|
||||||
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
|
|
||||||
for chunk in bytes.chunks(3) {
|
|
||||||
let n = chunk.iter().enumerate().fold(0u32, |n, (i, b)| n | (*b as u32) << (16 - 8 * i));
|
|
||||||
for i in 0..4 {
|
|
||||||
out.push(if i <= chunk.len() { B64[(n >> (18 - 6 * i) & 63) as usize] as char } else { '=' });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
fn base64_decode(text: &str) -> Option<Vec<u8>> {
|
|
||||||
let (mut out, mut n, mut bits) = (Vec::new(), 0u32, 0);
|
|
||||||
for c in text.bytes().filter(|c| !c.is_ascii_whitespace() && *c != b'=') {
|
|
||||||
n = (n << 6 | B64.iter().position(|b| *b == c)? as u32) & 0xFF_FFFF;
|
|
||||||
bits += 6;
|
|
||||||
if bits >= 8 {
|
|
||||||
bits -= 8;
|
|
||||||
out.push((n >> bits) as u8);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn actor_fs_paths_and_discovery_docs() {
|
|
||||||
assert_eq!(actor_fs_path("/actor/brain/fs"), Some(("brain".to_string(), "/".to_string())));
|
|
||||||
assert_eq!(actor_fs_path("/actor/brain/fs/docs/readme.md"), Some(("brain".to_string(), "/docs/readme.md".to_string())));
|
|
||||||
assert_eq!(actor_fs_path("/actor/brain/public/fs/file"), None);
|
|
||||||
assert_eq!(actor_root("/actor/brain"), Some("brain"));
|
|
||||||
assert_eq!(actor_root("/actor/brain/fs"), None);
|
|
||||||
assert!(declares_actor_fs("//! brain\n//! Actor-served files include fs_read.\nfn main() {}"));
|
|
||||||
assert!(!declares_actor_fs("//! actor without fs support\nfn main() {}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn base64_round_trip() {
|
|
||||||
for s in ["", "a", "ab", "abc", "abcd", "\u{0}\u{ff}binary"] {
|
|
||||||
let enc = base64_encode(s.as_bytes());
|
|
||||||
assert_eq!(base64_decode(&enc).unwrap(), s.as_bytes());
|
|
||||||
}
|
|
||||||
assert_eq!(base64_encode(b"hello"), "aGVsbG8=");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn join_resolves_dots() {
|
|
||||||
assert_eq!(join("/a/b", "../c/./d"), "/a/c/d");
|
|
||||||
assert_eq!(join("/", "x/fs"), "/x/fs");
|
|
||||||
assert_eq!(join("/", ".."), "/");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
//! File watching: the host watches the whole world folder with the OS file
|
|
||||||
//! watcher (`notify`, inotify on Linux), whoever writes: actors, a 9P mount,
|
|
||||||
//! a shell or `git` on the host. Nothing outside the folder counts.
|
|
||||||
//!
|
|
||||||
//! Events are coalesced per path: once a path is quiet for `QUIET` (or has
|
|
||||||
//! kept changing for `MAX_DELAY`), it becomes one change
|
|
||||||
//! `{path, op: write | create | remove | rename, to?}` with a namespace path
|
|
||||||
//! (`/actor/...`). Then the broker
|
|
||||||
//!
|
|
||||||
//! - sends `fs_changed` to the actors that watch the path (`watch` request),
|
|
||||||
//! - starts or upgrades an actor whose `actor/<id>/actor.wasm` (or folder
|
|
||||||
//! `actor/<id>`) changed, tested first, as at boot (`Broker::check_actor`).
|
|
||||||
//!
|
|
||||||
//! The host's temporary files (`.sausage-tmp-*`) are skipped: a rename of one
|
|
||||||
//! onto a file is a `write` of that file. If the OS drops events, the broker
|
|
||||||
//! runs a full pass and tells every watcher (`Broker::overflowed`).
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::path::Path;
|
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use notify::event::{EventKind, ModifyKind, RenameMode};
|
|
||||||
use notify::{RecursiveMode, Watcher as _};
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
use tokio::time::Instant;
|
|
||||||
|
|
||||||
use crate::broker::Broker;
|
|
||||||
use crate::world::TMP_PREFIX;
|
|
||||||
|
|
||||||
/// A path that is quiet this long becomes one change.
|
|
||||||
pub const QUIET: Duration = Duration::from_millis(100);
|
|
||||||
/// A path that keeps changing becomes a change at least this often.
|
|
||||||
pub const MAX_DELAY: Duration = Duration::from_secs(1);
|
|
||||||
|
|
||||||
/// The running watcher. Dropping it stops watching.
|
|
||||||
pub struct Watcher {
|
|
||||||
_os: notify::RecommendedWatcher,
|
|
||||||
task: tokio::task::JoinHandle<()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for Watcher {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.task.abort();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Watch the world folder of `broker` recursively. A failure to watch part
|
|
||||||
/// of it (for example too few inotify watches) is logged, and the rest is
|
|
||||||
/// watched.
|
|
||||||
pub fn start(broker: Arc<Broker>) -> notify::Result<Watcher> {
|
|
||||||
let (tx, rx) = mpsc::unbounded_channel();
|
|
||||||
let mut os = notify::recommended_watcher(move |event| {
|
|
||||||
let _ = tx.send(event);
|
|
||||||
})?;
|
|
||||||
if let Err(e) = os.watch(&broker.world.root, RecursiveMode::Recursive) {
|
|
||||||
broker.host_error(&format!("cannot watch all of {}: {e}", broker.world.root.display()));
|
|
||||||
}
|
|
||||||
let task = tokio::spawn(run(broker, rx));
|
|
||||||
Ok(Watcher { _os: os, task })
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A change waiting for its path to be quiet.
|
|
||||||
struct Pending {
|
|
||||||
op: &'static str,
|
|
||||||
to: Option<String>,
|
|
||||||
first: Instant,
|
|
||||||
last: Instant,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Pending {
|
|
||||||
fn due(&self) -> Instant {
|
|
||||||
(self.last + QUIET).min(self.first + MAX_DELAY)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run(broker: Arc<Broker>, mut events: mpsc::UnboundedReceiver<notify::Result<notify::Event>>) {
|
|
||||||
let root = broker.world.actors_root();
|
|
||||||
let mut pending: HashMap<String, Pending> = HashMap::new();
|
|
||||||
// Actor checks and full passes. Dropping the set (when the host stops) aborts them.
|
|
||||||
let mut tasks = tokio::task::JoinSet::new();
|
|
||||||
loop {
|
|
||||||
let due = pending.values().map(Pending::due).min();
|
|
||||||
tokio::select! {
|
|
||||||
event = events.recv() => match event {
|
|
||||||
Some(Ok(event)) => {
|
|
||||||
if event.need_rescan() {
|
|
||||||
let broker = broker.clone();
|
|
||||||
tasks.spawn(async move {
|
|
||||||
broker.overflowed().await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let now = Instant::now();
|
|
||||||
for (path, op, to) in changes(&root, &event) {
|
|
||||||
add(&mut pending, path, op, to, now);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(Err(e)) => broker.host_error(&format!("file watcher: {e}")),
|
|
||||||
None => break,
|
|
||||||
},
|
|
||||||
_ = sleep_until(due) => {
|
|
||||||
let now = Instant::now();
|
|
||||||
let ready: Vec<String> = pending.iter().filter(|(_, p)| p.due() <= now).map(|(k, _)| k.clone()).collect();
|
|
||||||
for path in ready {
|
|
||||||
let p = pending.remove(&path).unwrap();
|
|
||||||
let op = if p.op == "moved-in" { "create" } else { p.op };
|
|
||||||
broker.notify_watchers(&path, op, p.to.as_deref());
|
|
||||||
if let Some(id) = new_wasm(&path, op, p.to.as_deref()) {
|
|
||||||
tasks.spawn(broker.clone().check_actor(id));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(_) = tasks.join_next() => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sleep_until(t: Option<Instant>) {
|
|
||||||
match t {
|
|
||||||
Some(t) => tokio::time::sleep_until(t).await,
|
|
||||||
None => std::future::pending().await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The namespace path (`/actor/...`) of a host path, unless it is outside
|
|
||||||
/// `actor/` or a temporary file of the host.
|
|
||||||
fn namespace(root: &Path, path: &Path) -> Option<String> {
|
|
||||||
let rel = path.strip_prefix(root).ok()?;
|
|
||||||
let mut out = String::from("/actor");
|
|
||||||
for part in rel.components() {
|
|
||||||
let name = part.as_os_str().to_str()?;
|
|
||||||
if name.starts_with(TMP_PREFIX) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
out.push('/');
|
|
||||||
out.push_str(name);
|
|
||||||
}
|
|
||||||
Some(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One change: namespace path, op, and the new path of a rename.
|
|
||||||
type Change = (String, &'static str, Option<String>);
|
|
||||||
|
|
||||||
/// The changes of one OS event. inotify reports a rename three times:
|
|
||||||
/// `From` (the old name), `To` (the new name) and `Both`. `To` becomes the
|
|
||||||
/// internal op `moved-in`, which `Both` replaces.
|
|
||||||
fn changes(root: &Path, event: ¬ify::Event) -> Vec<Change> {
|
|
||||||
let path = |i: usize| event.paths.get(i).and_then(|p| namespace(root, p));
|
|
||||||
let one = |op: &'static str| -> Vec<Change> { path(0).map(|p| (p, op, None)).into_iter().collect() };
|
|
||||||
match event.kind {
|
|
||||||
EventKind::Create(_) => one("create"),
|
|
||||||
EventKind::Remove(_) => one("remove"),
|
|
||||||
EventKind::Modify(ModifyKind::Metadata(_)) => vec![],
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => one("remove"),
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::To)) => one("moved-in"),
|
|
||||||
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => match (path(0), path(1)) {
|
|
||||||
(Some(from), Some(to)) => vec![(from, "rename", Some(to))],
|
|
||||||
// Renamed to a temporary name: gone.
|
|
||||||
(Some(from), None) => vec![(from, "remove", None)],
|
|
||||||
// A temporary file renamed onto `to`: an atomic write.
|
|
||||||
(None, Some(to)) => vec![(to, "write", None)],
|
|
||||||
(None, None) => vec![],
|
|
||||||
},
|
|
||||||
EventKind::Modify(_) => one("write"),
|
|
||||||
_ => vec![],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Coalesce a change into `pending`. The last op wins, except that a write
|
|
||||||
/// after a create stays a create. A rename replaces the `moved-in` of its
|
|
||||||
/// new name.
|
|
||||||
fn add(pending: &mut HashMap<String, Pending>, path: String, op: &'static str, to: Option<String>, now: Instant) {
|
|
||||||
if let Some(to) = &to
|
|
||||||
&& pending.get(to).is_some_and(|p| p.op == "moved-in")
|
|
||||||
{
|
|
||||||
pending.remove(to);
|
|
||||||
}
|
|
||||||
match pending.get_mut(&path) {
|
|
||||||
Some(p) => {
|
|
||||||
if !(p.op == "create" && op == "write") {
|
|
||||||
p.op = op;
|
|
||||||
p.to = to;
|
|
||||||
}
|
|
||||||
p.last = now;
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
pending.insert(path, Pending { op, to, first: now, last: now });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The actor to check when `actor/<id>/actor.wasm` or the folder `actor/<id>`
|
|
||||||
/// appeared or changed.
|
|
||||||
fn new_wasm(path: &str, op: &str, to: Option<&str>) -> Option<String> {
|
|
||||||
let target = match op {
|
|
||||||
"remove" => return None,
|
|
||||||
"rename" => to?,
|
|
||||||
_ => path,
|
|
||||||
};
|
|
||||||
let rest = target.strip_prefix("/actor/")?;
|
|
||||||
let id = rest.strip_suffix("/actor.wasm").unwrap_or(rest);
|
|
||||||
(!id.is_empty() && !id.contains('/')).then(|| id.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn wasm_and_folder_changes_name_the_actor() {
|
|
||||||
assert_eq!(new_wasm("/actor/a/actor.wasm", "write", None).as_deref(), Some("a"));
|
|
||||||
assert_eq!(new_wasm("/actor/a", "create", None).as_deref(), Some("a"));
|
|
||||||
assert_eq!(new_wasm("/actor/x/y", "rename", Some("/actor/b/actor.wasm")).as_deref(), Some("b"));
|
|
||||||
assert_eq!(new_wasm("/actor/a/actor.wasm", "remove", None), None);
|
|
||||||
assert_eq!(new_wasm("/actor/a/versions/1/actor.wasm", "write", None), None);
|
|
||||||
assert_eq!(new_wasm("/actor/a/public/x", "write", None), None);
|
|
||||||
assert_eq!(new_wasm("/actor", "write", None), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_rename_is_one_change_and_temp_files_are_skipped() {
|
|
||||||
let root = Path::new("/w/actor");
|
|
||||||
let now = Instant::now();
|
|
||||||
let mut pending = HashMap::new();
|
|
||||||
let ev = |kind, paths: &[&str]| {
|
|
||||||
let mut e = notify::Event::new(kind);
|
|
||||||
for p in paths {
|
|
||||||
e = e.add_path(std::path::PathBuf::from(*p));
|
|
||||||
}
|
|
||||||
e
|
|
||||||
};
|
|
||||||
let name = |mode| EventKind::Modify(ModifyKind::Name(mode));
|
|
||||||
for e in [
|
|
||||||
ev(name(RenameMode::From), &["/w/actor/a/public/x"]),
|
|
||||||
ev(name(RenameMode::To), &["/w/actor/a/public/y"]),
|
|
||||||
ev(name(RenameMode::Both), &["/w/actor/a/public/x", "/w/actor/a/public/y"]),
|
|
||||||
ev(EventKind::Create(notify::event::CreateKind::File), &["/w/actor/a/.sausage-tmp-host-1-actor.json"]),
|
|
||||||
ev(name(RenameMode::Both), &["/w/actor/a/.sausage-tmp-host-1-actor.json", "/w/actor/a/actor.json"]),
|
|
||||||
ev(EventKind::Create(notify::event::CreateKind::File), &["/w/.lock"]),
|
|
||||||
] {
|
|
||||||
for (path, op, to) in changes(root, &e) {
|
|
||||||
add(&mut pending, path, op, to, now);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let mut got: Vec<(String, &str, Option<String>)> =
|
|
||||||
pending.into_iter().map(|(k, p)| (k, p.op, p.to)).collect();
|
|
||||||
got.sort();
|
|
||||||
assert_eq!(
|
|
||||||
got,
|
|
||||||
vec![
|
|
||||||
("/actor/a/actor.json".to_string(), "write", None),
|
|
||||||
("/actor/a/public/x".to_string(), "rename", Some("/actor/a/public/y".to_string())),
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+69
-148
@@ -1,62 +1,47 @@
|
|||||||
//! On-disk layout of a Sausage world. The folder is the source of truth:
|
//! On-disk layout of a Sausage workspace ("world").
|
||||||
//! `sausage run` boots every actor that it finds in `actor/`, and then
|
|
||||||
//! watches the folder (`watch.rs`).
|
|
||||||
//!
|
//!
|
||||||
//! ```text
|
//! ```text
|
||||||
//! <world>/
|
//! <world>/
|
||||||
//! actor/<id>/
|
//! sausage.json registry: actors, versions, subscriptions
|
||||||
//! actor.wasm the current version
|
//! actor/<id>/public/ written by the actor, read-only for others
|
||||||
//! actor.json version, running, subscriptions, tapped, watches (broker)
|
//! actor/<id>/source/ source of the running version (broker)
|
||||||
//! source/ source of the current version
|
//! actor/<id>/status.json state of the actor (broker)
|
||||||
//! public/ written by the actor, read-only for others
|
//! actor/<id>/stderr.log stderr of the actor (broker)
|
||||||
//! data/ state, writable only by the actor
|
//! private/<id>/data/ private state of the actor
|
||||||
//! stderr.log stderr of the actor and notes of the broker
|
//! broker/versions/<id>/<n>/ actor.wasm, source/, meta.json, snapshot/
|
||||||
//! versions/<n>/ every version that was installed: actor.wasm, source/, meta.json
|
//! archive/ removed actors
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
/// Name prefix of the host's temporary files and folders. Copies skip them.
|
use crate::fs::TMP_PREFIX;
|
||||||
pub const TMP_PREFIX: &str = ".sausage-tmp-";
|
|
||||||
|
|
||||||
/// IDs of the actors that the host implements natively. `fs` sends the
|
/// IDs of the actors that the host implements natively.
|
||||||
/// requests for actor-served files (`vfs.rs`).
|
pub const NATIVE: &[&str] = &["broker", "console", "cargo"];
|
||||||
pub const NATIVE: &[&str] = &["broker", "console", "cargo", "fs"];
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct World {
|
pub struct World {
|
||||||
pub root: PathBuf,
|
pub root: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `actor/<id>/actor.json`.
|
#[derive(Default, Serialize, Deserialize, Clone)]
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
pub struct Registry {
|
||||||
pub struct ActorMeta {
|
pub actors: BTreeMap<String, ActorRecord>,
|
||||||
/// The installed version. `None` for native actors and for a new
|
/// Publisher ID → subscriber IDs.
|
||||||
/// `actor.wasm` that the broker has not installed yet.
|
pub subscriptions: BTreeMap<String, BTreeSet<String>>,
|
||||||
#[serde(default)]
|
}
|
||||||
pub version: Option<u64>,
|
|
||||||
#[serde(default = "yes")]
|
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||||
|
pub struct ActorRecord {
|
||||||
|
pub version: u64,
|
||||||
|
pub next_version: u64,
|
||||||
pub running: bool,
|
pub running: bool,
|
||||||
/// The actors whose published messages this actor receives.
|
|
||||||
#[serde(default)]
|
|
||||||
pub subscriptions: BTreeSet<String>,
|
|
||||||
/// Whether this actor gets a copy of every routed message (`tap`).
|
|
||||||
#[serde(default)]
|
|
||||||
pub tapped: bool,
|
|
||||||
/// Namespace paths whose changes this actor gets as `fs_changed` (`watch`).
|
|
||||||
#[serde(default)]
|
|
||||||
pub watches: BTreeSet<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn yes() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `versions/<n>/meta.json`.
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
|
||||||
pub struct VersionMeta {
|
pub struct VersionMeta {
|
||||||
pub version: u64,
|
pub version: u64,
|
||||||
@@ -66,7 +51,7 @@ pub struct VersionMeta {
|
|||||||
pub source: Option<String>,
|
pub source: Option<String>,
|
||||||
/// The actor that built the `.wasm` file.
|
/// The actor that built the `.wasm` file.
|
||||||
pub built_by: Option<String>,
|
pub built_by: Option<String>,
|
||||||
/// The actor that requested the version ("scan" for a file found on disk).
|
/// The actor that requested the spawn or upgrade.
|
||||||
pub requested_by: String,
|
pub requested_by: String,
|
||||||
pub created_at: u64,
|
pub created_at: u64,
|
||||||
}
|
}
|
||||||
@@ -89,14 +74,14 @@ impl World {
|
|||||||
Self { root: root.into() }
|
Self { root: root.into() }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn actors_root(&self) -> PathBuf {
|
pub fn registry_path(&self) -> PathBuf {
|
||||||
self.root.join("actor")
|
self.root.join("sausage.json")
|
||||||
}
|
}
|
||||||
pub fn actor_dir(&self, id: &str) -> PathBuf {
|
pub fn actor_dir(&self, id: &str) -> PathBuf {
|
||||||
self.actors_root().join(id)
|
self.root.join("actor").join(id)
|
||||||
}
|
}
|
||||||
pub fn wasm_path(&self, id: &str) -> PathBuf {
|
pub fn actors_root(&self) -> PathBuf {
|
||||||
self.actor_dir(id).join("actor.wasm")
|
self.root.join("actor")
|
||||||
}
|
}
|
||||||
pub fn public_dir(&self, id: &str) -> PathBuf {
|
pub fn public_dir(&self, id: &str) -> PathBuf {
|
||||||
self.actor_dir(id).join("public")
|
self.actor_dir(id).join("public")
|
||||||
@@ -105,88 +90,37 @@ impl World {
|
|||||||
self.actor_dir(id).join("source")
|
self.actor_dir(id).join("source")
|
||||||
}
|
}
|
||||||
pub fn data_dir(&self, id: &str) -> PathBuf {
|
pub fn data_dir(&self, id: &str) -> PathBuf {
|
||||||
self.actor_dir(id).join("data")
|
self.root.join("private").join(id).join("data")
|
||||||
}
|
}
|
||||||
pub fn versions_dir(&self, id: &str) -> PathBuf {
|
pub fn versions_dir(&self, id: &str) -> PathBuf {
|
||||||
self.actor_dir(id).join("versions")
|
self.root.join("broker").join("versions").join(id)
|
||||||
}
|
}
|
||||||
pub fn version_dir(&self, id: &str, v: u64) -> PathBuf {
|
pub fn version_dir(&self, id: &str, v: u64) -> PathBuf {
|
||||||
self.versions_dir(id).join(v.to_string())
|
self.versions_dir(id).join(v.to_string())
|
||||||
}
|
}
|
||||||
/// Private scratch space of the broker (candidate copies, seeds).
|
|
||||||
pub fn scratch_dir(&self) -> PathBuf {
|
pub fn scratch_dir(&self) -> PathBuf {
|
||||||
self.data_dir("broker").join("scratch")
|
self.root.join("broker").join("scratch")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn exists(&self) -> bool {
|
pub fn exists(&self) -> bool {
|
||||||
self.actors_root().is_dir()
|
self.registry_path().is_file()
|
||||||
}
|
|
||||||
|
|
||||||
/// Create an empty world with the folders of the native actors.
|
|
||||||
pub fn create(root: &Path) -> Result<World> {
|
|
||||||
let world = World::new(std::path::absolute(root)?);
|
|
||||||
if world.exists() {
|
|
||||||
bail!("{} is already a Sausage world", root.display());
|
|
||||||
}
|
|
||||||
for id in NATIVE {
|
|
||||||
world.ensure_actor_dirs(id)?;
|
|
||||||
}
|
|
||||||
Ok(world)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create the directories of one actor.
|
||||||
pub fn ensure_actor_dirs(&self, id: &str) -> Result<()> {
|
pub fn ensure_actor_dirs(&self, id: &str) -> Result<()> {
|
||||||
std::fs::create_dir_all(self.public_dir(id))?;
|
std::fs::create_dir_all(self.public_dir(id))?;
|
||||||
std::fs::create_dir_all(self.data_dir(id))?;
|
std::fs::create_dir_all(self.data_dir(id))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// IDs of every folder in `actor/`, sorted.
|
pub fn load_registry(&self) -> Result<Registry> {
|
||||||
pub fn actor_ids(&self) -> Result<Vec<String>> {
|
let text = std::fs::read_to_string(self.registry_path())
|
||||||
let mut ids: Vec<String> = std::fs::read_dir(self.actors_root())?
|
.with_context(|| format!("no Sausage world at {}", self.root.display()))?;
|
||||||
.filter_map(|e| e.ok())
|
Ok(serde_json::from_str(&text)?)
|
||||||
.filter(|e| e.path().is_dir())
|
|
||||||
.filter_map(|e| e.file_name().into_string().ok())
|
|
||||||
.filter(|id| valid_id(id))
|
|
||||||
.collect();
|
|
||||||
ids.sort();
|
|
||||||
Ok(ids)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn read_meta(&self, id: &str) -> ActorMeta {
|
pub fn save_registry(&self, reg: &Registry) -> Result<()> {
|
||||||
std::fs::read_to_string(self.actor_dir(id).join("actor.json"))
|
write_atomic(&self.registry_path(), serde_json::to_string_pretty(reg)?.as_bytes())
|
||||||
.ok()
|
|
||||||
.and_then(|s| serde_json::from_str(&s).ok())
|
|
||||||
.unwrap_or(ActorMeta { running: true, ..Default::default() })
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn write_meta(&self, id: &str, meta: &ActorMeta) -> Result<()> {
|
|
||||||
write_atomic(&self.actor_dir(id).join("actor.json"), serde_json::to_string_pretty(meta)?.as_bytes())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Make `actor.wasm` and `source/` a copy of `versions/<version>/`,
|
|
||||||
/// unless `actor.wasm` already is that version (it came from there).
|
|
||||||
pub fn install_files(&self, id: &str, version: u64) -> Result<()> {
|
|
||||||
let dir = self.version_dir(id, version);
|
|
||||||
self.ensure_actor_dirs(id)?;
|
|
||||||
let wasm = std::fs::read(dir.join("actor.wasm"))?;
|
|
||||||
if std::fs::read(self.wasm_path(id)).ok().as_ref() == Some(&wasm) {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
write_atomic(&self.wasm_path(id), &wasm)?;
|
|
||||||
replace_tree(&dir.join("source"), &self.source_dir(id))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Version numbers in `versions/`, ascending.
|
|
||||||
pub fn versions(&self, id: &str) -> Vec<u64> {
|
|
||||||
let mut v: Vec<u64> = std::fs::read_dir(self.versions_dir(id))
|
|
||||||
.map(|rd| rd.filter_map(|e| e.ok()?.file_name().to_str()?.parse().ok()).collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
v.sort();
|
|
||||||
v
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn next_version(&self, id: &str) -> u64 {
|
|
||||||
self.versions(id).last().map_or(1, |v| v + 1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Map a path in an actor's namespace to a host path.
|
/// Map a path in an actor's namespace to a host path.
|
||||||
@@ -195,28 +129,25 @@ impl World {
|
|||||||
/// - `/public/...` and `/data/...` belong to `sender`.
|
/// - `/public/...` and `/data/...` belong to `sender`.
|
||||||
/// - Native senders (the console) can also give absolute host paths.
|
/// - Native senders (the console) can also give absolute host paths.
|
||||||
pub fn resolve(&self, sender: &str, path: &str) -> Result<PathBuf> {
|
pub fn resolve(&self, sender: &str, path: &str) -> Result<PathBuf> {
|
||||||
let under = |base: PathBuf, rest: &str| -> Result<PathBuf> {
|
let clean = |base: PathBuf, rest: &str| -> Result<PathBuf> {
|
||||||
let mut out = base;
|
let mut out = base.clone();
|
||||||
for c in Path::new(rest).components() {
|
for c in Path::new(rest).components() {
|
||||||
match c {
|
match c {
|
||||||
Component::Normal(n) => out.push(n),
|
std::path::Component::Normal(n) => out.push(n),
|
||||||
Component::CurDir | Component::RootDir => {}
|
std::path::Component::CurDir | std::path::Component::RootDir => {}
|
||||||
_ => bail!("path must not contain '..': {path}"),
|
_ => bail!("path must not contain '..': {path}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(out)
|
Ok(out)
|
||||||
};
|
};
|
||||||
let prefixed = |prefix: &str| {
|
if let Some(rest) = path.strip_prefix("/actor/") {
|
||||||
path.strip_prefix(prefix).filter(|rest| rest.is_empty() || rest.starts_with('/'))
|
return clean(self.actors_root(), rest);
|
||||||
};
|
|
||||||
if let Some(rest) = prefixed("/actor") {
|
|
||||||
return under(self.actors_root(), rest);
|
|
||||||
}
|
}
|
||||||
if let Some(rest) = prefixed("/public") {
|
if let Some(rest) = path.strip_prefix("/public/").or(path.strip_prefix("/public").filter(|r| r.is_empty())) {
|
||||||
return under(self.public_dir(sender), rest);
|
return clean(self.public_dir(sender), rest);
|
||||||
}
|
}
|
||||||
if let Some(rest) = prefixed("/data") {
|
if let Some(rest) = path.strip_prefix("/data/").or(path.strip_prefix("/data").filter(|r| r.is_empty())) {
|
||||||
return under(self.data_dir(sender), rest);
|
return clean(self.data_dir(sender), rest);
|
||||||
}
|
}
|
||||||
if NATIVE.contains(&sender) && Path::new(path).is_absolute() {
|
if NATIVE.contains(&sender) && Path::new(path).is_absolute() {
|
||||||
return Ok(PathBuf::from(path));
|
return Ok(PathBuf::from(path));
|
||||||
@@ -235,34 +166,33 @@ impl World {
|
|||||||
pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
|
pub fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
|
||||||
let dir = path.parent().context("path has no parent")?;
|
let dir = path.parent().context("path has no parent")?;
|
||||||
std::fs::create_dir_all(dir)?;
|
std::fs::create_dir_all(dir)?;
|
||||||
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("file");
|
let tmp = dir.join(format!(
|
||||||
let tmp = dir.join(format!("{TMP_PREFIX}host-{}-{name}", std::process::id()));
|
"{TMP_PREFIX}host-{}-{}",
|
||||||
|
std::process::id(),
|
||||||
|
path.file_name().and_then(|n| n.to_str()).unwrap_or("file")
|
||||||
|
));
|
||||||
std::fs::write(&tmp, bytes)?;
|
std::fs::write(&tmp, bytes)?;
|
||||||
std::fs::rename(&tmp, path)?;
|
std::fs::rename(&tmp, path)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Folders that a copy of a source folder leaves out.
|
/// Copy a directory tree. Skips `target/` folders and uncommitted files.
|
||||||
const BUILD_DIRS: &[&str] = &["target", ".git"];
|
|
||||||
|
|
||||||
/// Copy a directory tree, except uncommitted files.
|
|
||||||
pub fn copy_tree(from: &Path, to: &Path) -> Result<()> {
|
pub fn copy_tree(from: &Path, to: &Path) -> Result<()> {
|
||||||
copy_filtered(from, to, &[])
|
|
||||||
}
|
|
||||||
|
|
||||||
fn copy_filtered(from: &Path, to: &Path, skip: &[&str]) -> Result<()> {
|
|
||||||
std::fs::create_dir_all(to)?;
|
std::fs::create_dir_all(to)?;
|
||||||
for entry in std::fs::read_dir(from).with_context(|| format!("read {}", from.display()))? {
|
for entry in std::fs::read_dir(from).with_context(|| format!("read {}", from.display()))? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
let name = entry.file_name();
|
let name = entry.file_name();
|
||||||
let name = name.to_string_lossy();
|
let name_str = name.to_string_lossy();
|
||||||
if name.starts_with(TMP_PREFIX) {
|
if name_str.starts_with(TMP_PREFIX) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let dst = to.join(&*name);
|
|
||||||
let ty = entry.file_type()?;
|
let ty = entry.file_type()?;
|
||||||
if ty.is_dir() && !skip.contains(&&*name) {
|
let dst = to.join(&name);
|
||||||
copy_filtered(&entry.path(), &dst, skip)?;
|
if ty.is_dir() {
|
||||||
|
if name_str == "target" || name_str == ".git" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
copy_tree(&entry.path(), &dst)?;
|
||||||
} else if ty.is_file() {
|
} else if ty.is_file() {
|
||||||
std::fs::copy(entry.path(), &dst)?;
|
std::fs::copy(entry.path(), &dst)?;
|
||||||
}
|
}
|
||||||
@@ -270,18 +200,9 @@ fn copy_filtered(from: &Path, to: &Path, skip: &[&str]) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replace the directory `dst` with a copy of `src` (an empty folder if `src`
|
/// Replace the directory `dst` with a copy of `src`, as close to atomic as a
|
||||||
/// does not exist): copy to a sibling, then swap with two renames.
|
/// directory can be: copy to a sibling, then swap with two renames.
|
||||||
pub fn replace_tree(src: &Path, dst: &Path) -> Result<()> {
|
pub fn replace_tree(src: &Path, dst: &Path) -> Result<()> {
|
||||||
replace_filtered(src, dst, &[])
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Like `replace_tree`, for a source folder: leaves out `target/` and `.git/`.
|
|
||||||
pub fn replace_source(src: &Path, dst: &Path) -> Result<()> {
|
|
||||||
replace_filtered(src, dst, BUILD_DIRS)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn replace_filtered(src: &Path, dst: &Path, skip: &[&str]) -> Result<()> {
|
|
||||||
let parent = dst.parent().context("path has no parent")?;
|
let parent = dst.parent().context("path has no parent")?;
|
||||||
let name = dst.file_name().and_then(|n| n.to_str()).unwrap_or("dir");
|
let name = dst.file_name().and_then(|n| n.to_str()).unwrap_or("dir");
|
||||||
let fresh = parent.join(format!("{TMP_PREFIX}new-{name}"));
|
let fresh = parent.join(format!("{TMP_PREFIX}new-{name}"));
|
||||||
@@ -289,7 +210,7 @@ fn replace_filtered(src: &Path, dst: &Path, skip: &[&str]) -> Result<()> {
|
|||||||
let _ = std::fs::remove_dir_all(&fresh);
|
let _ = std::fs::remove_dir_all(&fresh);
|
||||||
let _ = std::fs::remove_dir_all(&old);
|
let _ = std::fs::remove_dir_all(&old);
|
||||||
if src.is_dir() {
|
if src.is_dir() {
|
||||||
copy_filtered(src, &fresh, skip)?;
|
copy_tree(src, &fresh)?;
|
||||||
} else {
|
} else {
|
||||||
std::fs::create_dir_all(&fresh)?;
|
std::fs::create_dir_all(&fresh)?;
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
-105
@@ -1,105 +0,0 @@
|
|||||||
# This file is automatically @generated by Cargo.
|
|
||||||
# It is not intended for manual editing.
|
|
||||||
version = 4
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "itoa"
|
|
||||||
version = "1.0.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "memchr"
|
|
||||||
version = "2.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "proc-macro2"
|
|
||||||
version = "1.0.107"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
||||||
dependencies = [
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "quote"
|
|
||||||
version = "1.0.47"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
||||||
dependencies = [
|
|
||||||
"serde_core",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_core"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
||||||
dependencies = [
|
|
||||||
"serde_derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_derive"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_json"
|
|
||||||
version = "1.0.154"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
||||||
dependencies = [
|
|
||||||
"itoa",
|
|
||||||
"memchr",
|
|
||||||
"serde",
|
|
||||||
"serde_core",
|
|
||||||
"zmij",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "syn"
|
|
||||||
version = "3.0.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toolbox"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-ident"
|
|
||||||
version = "1.0.27"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "zmij"
|
|
||||||
version = "1.0.23"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
||||||
@@ -1,180 +0,0 @@
|
|||||||
use std::collections::HashMap;
|
|
||||||
use std::io::{self, BufRead, Write};
|
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn send(out: &mut impl Write, message: Value) {
|
|
||||||
writeln!(out, "{message}").unwrap();
|
|
||||||
out.flush().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn reply(out: &mut impl Write, message: &Value, body: Value) {
|
|
||||||
send(
|
|
||||||
out,
|
|
||||||
json!({"to": message["from"], "reply_to": message["id"], "body": body}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Connect to `address`, send one line, and return the line that comes back.
|
|
||||||
fn tcp_line(address: &str, line: &str) -> io::Result<String> {
|
|
||||||
let mut stream = std::net::TcpStream::connect(address)?;
|
|
||||||
writeln!(stream, "{line}")?;
|
|
||||||
let mut answer = String::new();
|
|
||||||
io::BufReader::new(stream).read_line(&mut answer)?;
|
|
||||||
Ok(answer.trim_end().to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn io_error(error: io::Error) -> Value {
|
|
||||||
json!({"type": "error", "error": error.to_string()})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
let actor = std::env::var("SAUSAGE_ACTOR").unwrap_or_default();
|
|
||||||
let version = std::env::var("SAUSAGE_VERSION").unwrap_or_else(|_| "candidate".to_string());
|
|
||||||
let candidate = std::env::var_os("SAUSAGE_CANDIDATE").is_some();
|
|
||||||
if actor == "crashloop" && version == "2" && !candidate {
|
|
||||||
std::fs::write("/data/state", "bad-v2").unwrap();
|
|
||||||
panic!("version 2 crashes on startup");
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut open = HashMap::<String, std::fs::File>::new();
|
|
||||||
// Files that this actor serves at /actor/<id>/fs/ (path -> text).
|
|
||||||
let mut served = std::collections::BTreeMap::<String, String>::new();
|
|
||||||
let stdin = io::stdin();
|
|
||||||
let mut out = io::stdout().lock();
|
|
||||||
let mut emit_shutdown = false;
|
|
||||||
if actor == "spawn-race" && !candidate {
|
|
||||||
let stamp = SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap()
|
|
||||||
.as_nanos();
|
|
||||||
std::fs::write(format!("/data/start-{stamp}"), "started").unwrap();
|
|
||||||
}
|
|
||||||
for line in stdin.lock().lines() {
|
|
||||||
let Ok(line) = line else { break };
|
|
||||||
let Ok(message) = serde_json::from_str::<Value>(&line) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let body = &message["body"];
|
|
||||||
let path = body["path"].as_str().unwrap_or_default().to_string();
|
|
||||||
let text = body["text"].as_str().unwrap_or_default();
|
|
||||||
let response = match body["type"].as_str() {
|
|
||||||
Some("shutdown") => {
|
|
||||||
if actor == "queue" {
|
|
||||||
let _ = std::fs::write("/data/shutdown", "stopping");
|
|
||||||
std::thread::sleep(Duration::from_millis(800));
|
|
||||||
}
|
|
||||||
if emit_shutdown {
|
|
||||||
send(
|
|
||||||
&mut out,
|
|
||||||
json!({"to": "console", "reply_to": message["id"], "body": {"type": "shutdown-reply"}}),
|
|
||||||
);
|
|
||||||
send(&mut out, json!({"body": {"type": "shutdown-publish"}}));
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Some("echo") => json!({"type": "echo", "value": body["value"]}),
|
|
||||||
Some("fs_read") => {
|
|
||||||
let dir = if path == "/" { "/".to_string() } else { format!("{path}/") };
|
|
||||||
let mut names: Vec<String> = served
|
|
||||||
.keys()
|
|
||||||
.filter_map(|k| k.strip_prefix(&dir))
|
|
||||||
.map(|rest| match rest.split_once('/') {
|
|
||||||
Some((first, _)) => format!("{first}/"),
|
|
||||||
None => rest.to_string(),
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
names.dedup();
|
|
||||||
if let Some(text) = served.get(&path) {
|
|
||||||
json!({"type": "fs_file", "text": text})
|
|
||||||
} else if path == "/" || !names.is_empty() {
|
|
||||||
let entries: Vec<Value> = names
|
|
||||||
.iter()
|
|
||||||
.map(|n| match n.strip_suffix('/') {
|
|
||||||
Some(d) => json!({"name": d, "type": "dir"}),
|
|
||||||
None => json!({"name": n, "type": "file"}),
|
|
||||||
})
|
|
||||||
.collect();
|
|
||||||
json!({"type": "fs_dir", "entries": entries})
|
|
||||||
} else {
|
|
||||||
json!({"type": "fs_error", "errno": "no-entry"})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some("fs_write") => {
|
|
||||||
served.insert(path.clone(), text.to_string());
|
|
||||||
json!({"type": "fs_done"})
|
|
||||||
}
|
|
||||||
Some("fs_remove") => match served.remove(&path) {
|
|
||||||
Some(_) => json!({"type": "fs_done"}),
|
|
||||||
None => json!({"type": "fs_error", "errno": "no-entry"}),
|
|
||||||
},
|
|
||||||
Some("publish") => {
|
|
||||||
send(
|
|
||||||
&mut out,
|
|
||||||
json!({"body": {"type": "published", "value": body["value"]}}),
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Some("version") => json!({"type": "version", "version": version}),
|
|
||||||
Some("ordered") => json!({"type": "ordered", "n": body["n"], "version": version}),
|
|
||||||
Some("write") => match std::fs::write(&path, text) {
|
|
||||||
Ok(()) => json!({"type": "ok"}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
Some("mkdir") => match std::fs::create_dir_all(&path) {
|
|
||||||
Ok(()) => json!({"type": "ok"}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
Some("rename") => match std::fs::rename(&path, body["to"].as_str().unwrap_or_default())
|
|
||||||
{
|
|
||||||
Ok(()) => json!({"type": "ok"}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
Some("open") => {
|
|
||||||
let result = (|| -> io::Result<()> {
|
|
||||||
let mut file = std::fs::File::create(&path)?;
|
|
||||||
file.write_all(text.as_bytes())?;
|
|
||||||
open.insert(path.clone(), file);
|
|
||||||
Ok(())
|
|
||||||
})();
|
|
||||||
match result {
|
|
||||||
Ok(()) => json!({"type": "opened"}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some("close") => json!({"type": "closed", "closed": open.remove(&path).is_some()}),
|
|
||||||
Some("arm-shutdown") => {
|
|
||||||
emit_shutdown = true;
|
|
||||||
json!({"type": "armed"})
|
|
||||||
}
|
|
||||||
Some("ignore-stdin") => json!({"type": "ignoring"}),
|
|
||||||
Some("tcp") => match tcp_line(body["address"].as_str().unwrap_or_default(), text) {
|
|
||||||
Ok(line) => json!({"type": "tcp", "line": line}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
Some("read") => match std::fs::read_to_string(&path) {
|
|
||||||
Ok(text) => json!({"type": "content", "text": text}),
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
Some("list") => match std::fs::read_dir(&path) {
|
|
||||||
Ok(entries) => {
|
|
||||||
let mut names: Vec<String> = entries
|
|
||||||
.filter_map(|entry| entry.ok())
|
|
||||||
.map(|entry| entry.file_name().to_string_lossy().to_string())
|
|
||||||
.collect();
|
|
||||||
names.sort();
|
|
||||||
json!({"type": "entries", "names": names})
|
|
||||||
}
|
|
||||||
Err(error) => io_error(error),
|
|
||||||
},
|
|
||||||
_ => json!({"type": "error", "error": "unknown message"}),
|
|
||||||
};
|
|
||||||
reply(&mut out, &message, response);
|
|
||||||
if body["type"] == "ignore-stdin" {
|
|
||||||
loop {
|
|
||||||
std::thread::sleep(Duration::from_secs(60));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,555 +0,0 @@
|
|||||||
//! Integration tests for the agent's inference and restart behavior.
|
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
mod common;
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::process::Command;
|
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use common::{create_world, receive_message, receive_reply, request, source_with_test, start};
|
|
||||||
use sausage::{Envelope, Host, World};
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
|
||||||
use tokio::sync::{mpsc, oneshot};
|
|
||||||
use tokio::task::{JoinHandle, JoinSet};
|
|
||||||
|
|
||||||
const APIS: [Api; 2] = [Api::OpenAi, Api::Anthropic];
|
|
||||||
const TEST_TOKEN: &str = "fake-token";
|
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
|
||||||
enum Api {
|
|
||||||
OpenAi,
|
|
||||||
Anthropic,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Api {
|
|
||||||
fn name(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::OpenAi => "openai-responses",
|
|
||||||
Self::Anthropic => "anthropic-messages",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn path(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::OpenAi => "/v1/responses",
|
|
||||||
Self::Anthropic => "/v1/messages",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn text_response(self, text: &str) -> Value {
|
|
||||||
match self {
|
|
||||||
Self::OpenAi => {
|
|
||||||
json!({"output": [{"type": "message", "content": [{"type": "output_text", "text": text}]}]})
|
|
||||||
}
|
|
||||||
Self::Anthropic => json!({"content": [{"type": "text", "text": text}]}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tool_response(self, id: &str, name: &str, arguments: &Value) -> Value {
|
|
||||||
match self {
|
|
||||||
Self::OpenAi => {
|
|
||||||
json!({"output": [{"type": "function_call", "call_id": id, "name": name, "arguments": arguments.to_string()}]})
|
|
||||||
}
|
|
||||||
Self::Anthropic => {
|
|
||||||
json!({"content": [{"type": "tool_use", "id": id, "name": name, "input": arguments}]})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn check_request(self, request: &HttpRequest) {
|
|
||||||
assert_eq!(request.method, "POST");
|
|
||||||
assert_eq!(request.path, self.path());
|
|
||||||
assert_eq!(request.headers["content-type"], "application/json");
|
|
||||||
assert_eq!(request.body["model"], "fake-model");
|
|
||||||
match self {
|
|
||||||
Self::OpenAi => assert_eq!(request.headers["authorization"], "Bearer fake-token"),
|
|
||||||
Self::Anthropic => {
|
|
||||||
assert_eq!(request.headers["x-api-key"], TEST_TOKEN);
|
|
||||||
assert_eq!(request.headers["anthropic-version"], "2023-06-01");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct HttpRequest {
|
|
||||||
method: String,
|
|
||||||
path: String,
|
|
||||||
headers: HashMap<String, String>,
|
|
||||||
body: Value,
|
|
||||||
}
|
|
||||||
|
|
||||||
enum ReplyPlan {
|
|
||||||
Send(Value),
|
|
||||||
Hold(Value, oneshot::Receiver<()>),
|
|
||||||
}
|
|
||||||
|
|
||||||
struct FakeServer {
|
|
||||||
base_url: String,
|
|
||||||
plans: mpsc::Sender<ReplyPlan>,
|
|
||||||
requests: mpsc::Receiver<HttpRequest>,
|
|
||||||
task: JoinHandle<()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FakeServer {
|
|
||||||
async fn start() -> Self {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let address = listener.local_addr().unwrap();
|
|
||||||
let (plans, mut plan_rx) = mpsc::channel(16);
|
|
||||||
let (request_tx, requests) = mpsc::channel(16);
|
|
||||||
let task = tokio::spawn(async move {
|
|
||||||
let mut handlers = JoinSet::new();
|
|
||||||
loop {
|
|
||||||
tokio::select! {
|
|
||||||
accepted = listener.accept() => {
|
|
||||||
let (mut stream, _) = accepted.expect("fake listener accept failed");
|
|
||||||
let request = read_http_request(&mut stream).await;
|
|
||||||
let Some(plan) = plan_rx.recv().await else { break };
|
|
||||||
if request_tx.send(request).await.is_err() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
handlers.spawn(async move {
|
|
||||||
let (body, release) = match plan {
|
|
||||||
ReplyPlan::Send(body) => (body, None),
|
|
||||||
ReplyPlan::Hold(body, release) => (body, Some(release)),
|
|
||||||
};
|
|
||||||
if let Some(release) = release {
|
|
||||||
let _ = release.await;
|
|
||||||
}
|
|
||||||
write_http_response(stream, body).await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Some(_) = handlers.join_next(), if !handlers.is_empty() => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
Self {
|
|
||||||
base_url: format!("http://127.0.0.1:{}", address.port()),
|
|
||||||
plans,
|
|
||||||
requests,
|
|
||||||
task,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn answer(&self, body: Value) {
|
|
||||||
self.plans.send(ReplyPlan::Send(body)).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn hold(&self, body: Value) -> oneshot::Sender<()> {
|
|
||||||
let (release, wait) = oneshot::channel();
|
|
||||||
self.plans.send(ReplyPlan::Hold(body, wait)).await.unwrap();
|
|
||||||
release
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn next_request(&mut self) -> HttpRequest {
|
|
||||||
tokio::time::timeout(Duration::from_secs(20), self.requests.recv())
|
|
||||||
.await
|
|
||||||
.expect("timed out waiting for fake inference request")
|
|
||||||
.expect("fake server request channel closed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Drop for FakeServer {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.task.abort();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_http_request(stream: &mut TcpStream) -> HttpRequest {
|
|
||||||
let mut received = Vec::new();
|
|
||||||
let mut chunk = [0; 4096];
|
|
||||||
let header_end = loop {
|
|
||||||
let count = stream.read(&mut chunk).await.expect("read HTTP request");
|
|
||||||
assert_ne!(count, 0, "client closed before HTTP headers arrived");
|
|
||||||
received.extend_from_slice(&chunk[..count]);
|
|
||||||
if let Some(end) = received.windows(4).position(|part| part == b"\r\n\r\n") {
|
|
||||||
break end;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let header_text = String::from_utf8_lossy(&received[..header_end]);
|
|
||||||
let mut lines = header_text.split("\r\n");
|
|
||||||
let mut request_line = lines.next().unwrap().split_whitespace();
|
|
||||||
let method = request_line.next().unwrap().to_string();
|
|
||||||
let path = request_line.next().unwrap().to_string();
|
|
||||||
let mut headers = HashMap::new();
|
|
||||||
for line in lines {
|
|
||||||
if let Some((name, value)) = line.split_once(':') {
|
|
||||||
headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let body_start = header_end + 4;
|
|
||||||
let request_body = if let Some(length) = headers.get("content-length") {
|
|
||||||
let body_length = length.parse::<usize>().unwrap();
|
|
||||||
while received.len() < body_start + body_length {
|
|
||||||
let count = stream.read(&mut chunk).await.expect("read HTTP body");
|
|
||||||
assert_ne!(count, 0, "client closed before HTTP body arrived");
|
|
||||||
received.extend_from_slice(&chunk[..count]);
|
|
||||||
}
|
|
||||||
received[body_start..body_start + body_length].to_vec()
|
|
||||||
} else {
|
|
||||||
assert!(
|
|
||||||
headers
|
|
||||||
.get("transfer-encoding")
|
|
||||||
.is_some_and(|value| value.eq_ignore_ascii_case("chunked")),
|
|
||||||
"request has neither Content-Length nor chunked transfer encoding: {headers:?}"
|
|
||||||
);
|
|
||||||
read_chunked_body(stream, &mut received, body_start).await
|
|
||||||
};
|
|
||||||
let body = serde_json::from_slice(&request_body).unwrap();
|
|
||||||
HttpRequest {
|
|
||||||
method,
|
|
||||||
path,
|
|
||||||
headers,
|
|
||||||
body,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn read_chunked_body(
|
|
||||||
stream: &mut TcpStream,
|
|
||||||
received: &mut Vec<u8>,
|
|
||||||
mut offset: usize,
|
|
||||||
) -> Vec<u8> {
|
|
||||||
let mut body = Vec::new();
|
|
||||||
loop {
|
|
||||||
let line_end = loop {
|
|
||||||
if let Some(end) = received[offset..]
|
|
||||||
.windows(2)
|
|
||||||
.position(|part| part == b"\r\n")
|
|
||||||
{
|
|
||||||
break offset + end;
|
|
||||||
}
|
|
||||||
let mut chunk = [0; 4096];
|
|
||||||
let count = stream.read(&mut chunk).await.expect("read chunk size");
|
|
||||||
assert_ne!(count, 0, "client closed before chunk size arrived");
|
|
||||||
received.extend_from_slice(&chunk[..count]);
|
|
||||||
};
|
|
||||||
let size = String::from_utf8_lossy(&received[offset..line_end]);
|
|
||||||
let size = size.split(';').next().unwrap().trim();
|
|
||||||
let size = usize::from_str_radix(size, 16).unwrap();
|
|
||||||
offset = line_end + 2;
|
|
||||||
if size == 0 {
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
while received.len() < offset + size + 2 {
|
|
||||||
let mut chunk = [0; 4096];
|
|
||||||
let count = stream.read(&mut chunk).await.expect("read chunk data");
|
|
||||||
assert_ne!(count, 0, "client closed before chunk data arrived");
|
|
||||||
received.extend_from_slice(&chunk[..count]);
|
|
||||||
}
|
|
||||||
body.extend_from_slice(&received[offset..offset + size]);
|
|
||||||
assert_eq!(&received[offset + size..offset + size + 2], b"\r\n");
|
|
||||||
offset += size + 2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn write_http_response(mut stream: TcpStream, body: Value) {
|
|
||||||
let body = serde_json::to_vec(&body).unwrap();
|
|
||||||
let head = format!(
|
|
||||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",
|
|
||||||
body.len()
|
|
||||||
);
|
|
||||||
let mut response = head.into_bytes();
|
|
||||||
response.extend_from_slice(&body);
|
|
||||||
let _ = stream.write_all(&response).await;
|
|
||||||
let _ = stream.shutdown().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn agent_wasm() -> &'static PathBuf {
|
|
||||||
static WASM: OnceLock<PathBuf> = OnceLock::new();
|
|
||||||
WASM.get_or_init(|| {
|
|
||||||
let root = common::repo_root();
|
|
||||||
let output = Command::new("cargo")
|
|
||||||
.args([
|
|
||||||
"build",
|
|
||||||
"--release",
|
|
||||||
"-p",
|
|
||||||
"agent",
|
|
||||||
"--target",
|
|
||||||
"wasm32-wasip2",
|
|
||||||
"--target-dir",
|
|
||||||
"target/test-actors",
|
|
||||||
])
|
|
||||||
.current_dir(&root)
|
|
||||||
.output()
|
|
||||||
.expect("run cargo to build the agent actor");
|
|
||||||
assert!(
|
|
||||||
output.status.success(),
|
|
||||||
"building agent failed\nstdout:\n{}\nstderr:\n{}",
|
|
||||||
String::from_utf8_lossy(&output.stdout),
|
|
||||||
String::from_utf8_lossy(&output.stderr)
|
|
||||||
);
|
|
||||||
let wasm = root.join("target/test-actors/wasm32-wasip2/release/agent.wasm");
|
|
||||||
assert!(
|
|
||||||
wasm.is_file(),
|
|
||||||
"agent build did not create {}",
|
|
||||||
wasm.display()
|
|
||||||
);
|
|
||||||
wasm
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn start_agent(
|
|
||||||
api: Api,
|
|
||||||
fake: &FakeServer,
|
|
||||||
label: &str,
|
|
||||||
) -> (World, Host, mpsc::Receiver<Envelope>) {
|
|
||||||
let world = create_world(label);
|
|
||||||
let data = world.data_dir("agent");
|
|
||||||
std::fs::create_dir_all(&data).unwrap();
|
|
||||||
let config = json!({"api": api.name(), "base_url": fake.base_url, "model": "fake-model", "token": TEST_TOKEN});
|
|
||||||
std::fs::write(
|
|
||||||
data.join("config.json"),
|
|
||||||
serde_json::to_vec(&config).unwrap(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let source = source_with_test(
|
|
||||||
&world,
|
|
||||||
"agent-candidate",
|
|
||||||
json!({"steps": [{"send": {"type": "text", "text": "candidate"}, "expect": {"type": "text"}}]}),
|
|
||||||
);
|
|
||||||
let (host, mut mailbox) = start(&world).await;
|
|
||||||
let spawned = common::spawn_actor(&host, &mut mailbox, "agent", agent_wasm(), &source).await;
|
|
||||||
assert_eq!(
|
|
||||||
spawned.body["type"], "spawned",
|
|
||||||
"agent spawn failed: {spawned:?}"
|
|
||||||
);
|
|
||||||
let subscribed = request(
|
|
||||||
&host,
|
|
||||||
&mut mailbox,
|
|
||||||
json!({"type": "subscribe", "actor": "agent"}),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert_eq!(subscribed.body["type"], "subscribed");
|
|
||||||
(world, host, mailbox)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn send_text(host: &Host, text: &str) -> String {
|
|
||||||
host.broker.send(
|
|
||||||
"console",
|
|
||||||
Some("agent"),
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
json!({"type": "text", "text": text}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn assert_published_text(mailbox: &mut mpsc::Receiver<Envelope>, text: &str) {
|
|
||||||
let message = receive_message(mailbox, Duration::from_secs(20)).await;
|
|
||||||
assert_eq!(message.from, "agent");
|
|
||||||
assert_eq!(message.to, "console");
|
|
||||||
assert_eq!(message.reply_to, None);
|
|
||||||
assert_eq!(message.body, json!({"type": "text", "text": text}));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn history(world: &World) -> Vec<Value> {
|
|
||||||
std::fs::read_to_string(world.data_dir("agent").join("history.jsonl"))
|
|
||||||
.unwrap()
|
|
||||||
.lines()
|
|
||||||
.map(|line| serde_json::from_str(line).unwrap())
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assert_history_tools_have_results(history: &[Value]) {
|
|
||||||
let calls: Vec<_> = history
|
|
||||||
.iter()
|
|
||||||
.filter(|entry| entry["kind"] == "tool_call")
|
|
||||||
.collect();
|
|
||||||
let results: Vec<_> = history
|
|
||||||
.iter()
|
|
||||||
.filter(|entry| entry["kind"] == "tool_result")
|
|
||||||
.collect();
|
|
||||||
assert_eq!(calls.len(), results.len());
|
|
||||||
for call in calls {
|
|
||||||
assert!(
|
|
||||||
results.iter().any(|result| result["id"] == call["id"]),
|
|
||||||
"tool call has no result: {call}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assert_tool_pair(
|
|
||||||
request: &HttpRequest,
|
|
||||||
api: Api,
|
|
||||||
id: &str,
|
|
||||||
name: &str,
|
|
||||||
arguments: &Value,
|
|
||||||
) -> String {
|
|
||||||
match api {
|
|
||||||
Api::OpenAi => {
|
|
||||||
let input = request.body["input"].as_array().unwrap();
|
|
||||||
let call_index = input
|
|
||||||
.iter()
|
|
||||||
.position(|item| item["type"] == "function_call")
|
|
||||||
.unwrap();
|
|
||||||
let result_index = input
|
|
||||||
.iter()
|
|
||||||
.position(|item| item["type"] == "function_call_output")
|
|
||||||
.unwrap();
|
|
||||||
assert!(call_index < result_index);
|
|
||||||
let call = &input[call_index];
|
|
||||||
assert_eq!(call["call_id"], id);
|
|
||||||
assert_eq!(call["name"], name);
|
|
||||||
assert_eq!(
|
|
||||||
serde_json::from_str::<Value>(call["arguments"].as_str().unwrap()).unwrap(),
|
|
||||||
*arguments
|
|
||||||
);
|
|
||||||
let result = &input[result_index];
|
|
||||||
assert_eq!(result["call_id"], id);
|
|
||||||
result["output"].as_str().unwrap().to_string()
|
|
||||||
}
|
|
||||||
Api::Anthropic => {
|
|
||||||
let messages = request.body["messages"].as_array().unwrap();
|
|
||||||
let mut call_position = None;
|
|
||||||
let mut result_position = None;
|
|
||||||
let mut call = None;
|
|
||||||
let mut result = None;
|
|
||||||
for (index, message) in messages.iter().enumerate() {
|
|
||||||
let Some(blocks) = message["content"].as_array() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
for block in blocks {
|
|
||||||
match block["type"].as_str() {
|
|
||||||
Some("tool_use") => {
|
|
||||||
call_position = Some(index);
|
|
||||||
call = Some(block);
|
|
||||||
}
|
|
||||||
Some("tool_result") => {
|
|
||||||
result_position = Some(index);
|
|
||||||
result = Some(block);
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert!(call_position.unwrap() < result_position.unwrap());
|
|
||||||
let call = call.unwrap();
|
|
||||||
assert_eq!(call["id"], id);
|
|
||||||
assert_eq!(call["name"], name);
|
|
||||||
assert_eq!(call["input"], *arguments);
|
|
||||||
let result = result.unwrap();
|
|
||||||
assert_eq!(result["tool_use_id"], id);
|
|
||||||
result["content"].as_str().unwrap().to_string()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn text_answers_use_both_inference_apis() {
|
|
||||||
for api in APIS {
|
|
||||||
let mut fake = FakeServer::start().await;
|
|
||||||
fake.answer(api.text_response("hello from the fake model"))
|
|
||||||
.await;
|
|
||||||
let (_, host, mut mailbox) =
|
|
||||||
start_agent(api, &fake, &format!("agent-text-{}", api.name())).await;
|
|
||||||
send_text(&host, "say hello");
|
|
||||||
api.check_request(&fake.next_request().await);
|
|
||||||
assert_published_text(&mut mailbox, "hello from the fake model").await;
|
|
||||||
host.stop().await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn tool_calls_pair_with_results_in_both_inference_apis() {
|
|
||||||
let args = json!({"path": "/actor"});
|
|
||||||
for api in APIS {
|
|
||||||
let mut fake = FakeServer::start().await;
|
|
||||||
fake.answer(api.tool_response("list-1", "list_dir", &args))
|
|
||||||
.await;
|
|
||||||
let (world, host, mut mailbox) =
|
|
||||||
start_agent(api, &fake, &format!("agent-tools-{}", api.name())).await;
|
|
||||||
send_text(&host, "list actors");
|
|
||||||
api.check_request(&fake.next_request().await);
|
|
||||||
fake.answer(api.text_response("the actor list is ready"))
|
|
||||||
.await;
|
|
||||||
let second = fake.next_request().await;
|
|
||||||
api.check_request(&second);
|
|
||||||
let output = assert_tool_pair(&second, api, "list-1", "list_dir", &args);
|
|
||||||
let result: Value = serde_json::from_str(&output).unwrap();
|
|
||||||
assert!(
|
|
||||||
result["entries"]
|
|
||||||
.as_array()
|
|
||||||
.unwrap()
|
|
||||||
.iter()
|
|
||||||
.any(|entry| entry["name"] == "agent")
|
|
||||||
);
|
|
||||||
assert_published_text(&mut mailbox, "the actor list is ready").await;
|
|
||||||
|
|
||||||
host.stop().await;
|
|
||||||
let history = history(&world);
|
|
||||||
assert_eq!(
|
|
||||||
history
|
|
||||||
.iter()
|
|
||||||
.map(|entry| entry["kind"].as_str().unwrap())
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
["user", "tool_call", "tool_result", "assistant"]
|
|
||||||
);
|
|
||||||
assert_eq!(history[1]["id"], "list-1");
|
|
||||||
assert_eq!(history[2]["id"], "list-1");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn restart_resumes_a_turn_with_completed_tool_history() {
|
|
||||||
let args = json!({"path": "/actor"});
|
|
||||||
for api in [Api::OpenAi] {
|
|
||||||
let mut fake = FakeServer::start().await;
|
|
||||||
fake.answer(api.tool_response("list-1", "list_dir", &args))
|
|
||||||
.await;
|
|
||||||
let (world, host, mut mailbox) =
|
|
||||||
start_agent(api, &fake, &format!("agent-resume-{}", api.name())).await;
|
|
||||||
let id = send_text(&host, "list actors before the restart");
|
|
||||||
api.check_request(&fake.next_request().await);
|
|
||||||
let release = fake.hold(api.text_response("discarded response")).await;
|
|
||||||
let second = fake.next_request().await;
|
|
||||||
api.check_request(&second);
|
|
||||||
assert_tool_pair(&second, api, "list-1", "list_dir", &args);
|
|
||||||
|
|
||||||
let stopped = request(
|
|
||||||
&host,
|
|
||||||
&mut mailbox,
|
|
||||||
json!({"type": "stop", "actor": "agent"}),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert_eq!(stopped.body["type"], "stopped");
|
|
||||||
assert_eq!(
|
|
||||||
serde_json::from_slice::<Value>(
|
|
||||||
&std::fs::read(world.data_dir("agent").join("turn.json")).unwrap()
|
|
||||||
)
|
|
||||||
.unwrap(),
|
|
||||||
json!(true)
|
|
||||||
);
|
|
||||||
let _ = release.send(());
|
|
||||||
fake.answer(api.text_response("resumed answer")).await;
|
|
||||||
let start_id = host.broker.send(
|
|
||||||
"console",
|
|
||||||
Some("broker"),
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
json!({"type": "start", "actor": "agent"}),
|
|
||||||
);
|
|
||||||
let resumed = fake.next_request().await;
|
|
||||||
api.check_request(&resumed);
|
|
||||||
assert_tool_pair(&resumed, api, "list-1", "list_dir", &args);
|
|
||||||
let started = receive_reply(&mut mailbox, &start_id, Duration::from_secs(20)).await;
|
|
||||||
assert_eq!(started.body["type"], "started");
|
|
||||||
assert_published_text(&mut mailbox, "resumed answer").await;
|
|
||||||
|
|
||||||
let history = history(&world);
|
|
||||||
assert_history_tools_have_results(&history);
|
|
||||||
assert_eq!(
|
|
||||||
history
|
|
||||||
.iter()
|
|
||||||
.filter(|entry| entry["kind"] == "user"
|
|
||||||
&& entry["text"].as_str().unwrap_or_default().contains(&id))
|
|
||||||
.count(),
|
|
||||||
1
|
|
||||||
);
|
|
||||||
host.stop().await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
//! Tests of the `sausage` command line.
|
|
||||||
|
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::process::{Command, Stdio};
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
fn empty_dir(label: &str) -> PathBuf {
|
|
||||||
let nonce = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
|
|
||||||
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
|
||||||
.parent()
|
|
||||||
.unwrap()
|
|
||||||
.join("target/test-cli")
|
|
||||||
.join(format!("{label}-{}-{nonce}", std::process::id()));
|
|
||||||
std::fs::create_dir_all(&dir).unwrap();
|
|
||||||
dir
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Bare `sausage` (as in `nix run <flake>`) without a world starts `init`
|
|
||||||
/// for ./world instead of printing a usage error.
|
|
||||||
#[test]
|
|
||||||
fn bare_command_without_a_world_starts_init() {
|
|
||||||
let dir = empty_dir("bare");
|
|
||||||
let output = Command::new(env!("CARGO_BIN_EXE_sausage"))
|
|
||||||
.current_dir(&dir)
|
|
||||||
.env_remove("SAUSAGE_API")
|
|
||||||
.env_remove("SAUSAGE_BASE_URL")
|
|
||||||
.env_remove("SAUSAGE_MODEL")
|
|
||||||
.env_remove("SAUSAGE_API_TOKEN")
|
|
||||||
.stdin(Stdio::null())
|
|
||||||
.output()
|
|
||||||
.unwrap();
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
||||||
// With no answers on stdin, init stops at the first required value.
|
|
||||||
assert!(!output.status.success(), "stdout:\n{stdout}\nstderr:\n{stderr}");
|
|
||||||
assert!(stdout.contains("No world in ./world yet"), "stdout:\n{stdout}\nstderr:\n{stderr}");
|
|
||||||
assert!(stderr.contains("SAUSAGE_BASE_URL"), "stdout:\n{stdout}\nstderr:\n{stderr}");
|
|
||||||
assert!(!stderr.contains("Usage:"), "stderr:\n{stderr}");
|
|
||||||
// It asks before it creates anything.
|
|
||||||
assert!(!dir.join("world").exists());
|
|
||||||
}
|
|
||||||
@@ -1,248 +0,0 @@
|
|||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::process::Command;
|
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
use sausage::{Envelope, Host, World};
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use tokio::sync::mpsc;
|
|
||||||
|
|
||||||
const REPLY_TIMEOUT: Duration = Duration::from_secs(10);
|
|
||||||
static WORLD_SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
|
||||||
|
|
||||||
pub fn repo_root() -> PathBuf {
|
|
||||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
|
||||||
.parent()
|
|
||||||
.unwrap()
|
|
||||||
.to_path_buf()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn create_world(label: &str) -> World {
|
|
||||||
let nonce = SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap()
|
|
||||||
.as_nanos();
|
|
||||||
let sequence = WORLD_SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
|
||||||
let path = repo_root()
|
|
||||||
.join("target/test-worlds")
|
|
||||||
.join(format!("{label}-{}-{nonce}-{sequence}", std::process::id()));
|
|
||||||
World::create(&path).unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn actor_fixture_source(name: &str) -> PathBuf {
|
|
||||||
repo_root().join("cli/tests/actors").join(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn cargo_source(name: &str) -> PathBuf {
|
|
||||||
repo_root().join("cli/tests/crates").join(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn toolbox_wasm() -> &'static PathBuf {
|
|
||||||
static WASM: OnceLock<PathBuf> = OnceLock::new();
|
|
||||||
WASM.get_or_init(|| build_fixture("toolbox"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_fixture(name: &str) -> PathBuf {
|
|
||||||
let root = repo_root();
|
|
||||||
let manifest = actor_fixture_source(name).join("Cargo.toml");
|
|
||||||
let target_dir = root.join("target/test-actors");
|
|
||||||
let output = Command::new("cargo")
|
|
||||||
.args([
|
|
||||||
"build",
|
|
||||||
"--release",
|
|
||||||
"--target",
|
|
||||||
"wasm32-wasip2",
|
|
||||||
"--target-dir",
|
|
||||||
])
|
|
||||||
.arg(&target_dir)
|
|
||||||
.arg("--manifest-path")
|
|
||||||
.arg(&manifest)
|
|
||||||
.current_dir(&root)
|
|
||||||
.output()
|
|
||||||
.expect("run cargo to build the test actor");
|
|
||||||
assert!(
|
|
||||||
output.status.success(),
|
|
||||||
"building fixture {name} failed\nstdout:\n{}\nstderr:\n{}",
|
|
||||||
String::from_utf8_lossy(&output.stdout),
|
|
||||||
String::from_utf8_lossy(&output.stderr)
|
|
||||||
);
|
|
||||||
let wasm = target_dir
|
|
||||||
.join("wasm32-wasip2/release")
|
|
||||||
.join(format!("{name}.wasm"));
|
|
||||||
assert!(
|
|
||||||
wasm.is_file(),
|
|
||||||
"fixture build did not create {}",
|
|
||||||
wasm.display()
|
|
||||||
);
|
|
||||||
wasm
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn passing_source(world: &World, name: &str) -> PathBuf {
|
|
||||||
source_with_test(
|
|
||||||
world,
|
|
||||||
name,
|
|
||||||
json!({
|
|
||||||
"steps": [{
|
|
||||||
"send": {"type": "echo", "value": "candidate"},
|
|
||||||
"expect": {"type": "echo", "value": "candidate"}
|
|
||||||
}]
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn source_with_test(world: &World, name: &str, test: Value) -> PathBuf {
|
|
||||||
let source = world.source_dir("console").join("test-fixtures").join(name);
|
|
||||||
std::fs::create_dir_all(source.join("tests")).unwrap();
|
|
||||||
std::fs::write(
|
|
||||||
source.join("tests/candidate.json"),
|
|
||||||
serde_json::to_vec(&test).unwrap(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
source
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn start(world: &World) -> (Host, mpsc::Receiver<Envelope>) {
|
|
||||||
let host = Host::start(&world.root).await.unwrap();
|
|
||||||
let mailbox = host.mailbox("console");
|
|
||||||
(host, mailbox)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn request(host: &Host, mailbox: &mut mpsc::Receiver<Envelope>, body: Value) -> Envelope {
|
|
||||||
let id = host
|
|
||||||
.broker
|
|
||||||
.send("console", Some("broker"), None, None, body);
|
|
||||||
receive_reply(mailbox, &id, REPLY_TIMEOUT).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn receive_message(
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
timeout: Duration,
|
|
||||||
) -> Envelope {
|
|
||||||
tokio::time::timeout(timeout, mailbox.recv())
|
|
||||||
.await
|
|
||||||
.expect("timed out waiting for a message")
|
|
||||||
.expect("console mailbox closed")
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn receive_reply(
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
id: &str,
|
|
||||||
timeout: Duration,
|
|
||||||
) -> Envelope {
|
|
||||||
tokio::time::timeout(timeout, async {
|
|
||||||
loop {
|
|
||||||
let envelope = mailbox.recv().await.expect("console mailbox closed");
|
|
||||||
if envelope.reply_to.as_deref() == Some(id) {
|
|
||||||
return envelope;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.unwrap_or_else(|_| panic!("timed out waiting for reply to {id}"))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn receive_replies(
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
ids: &[String],
|
|
||||||
timeout: Duration,
|
|
||||||
) -> Vec<Envelope> {
|
|
||||||
let mut replies = Vec::with_capacity(ids.len());
|
|
||||||
tokio::time::timeout(timeout, async {
|
|
||||||
while replies.len() < ids.len() {
|
|
||||||
let message = mailbox.recv().await.expect("console mailbox closed");
|
|
||||||
assert!(
|
|
||||||
ids.iter()
|
|
||||||
.any(|id| message.reply_to.as_deref() == Some(id.as_str())),
|
|
||||||
"unexpected message while waiting for replies: {message:?}"
|
|
||||||
);
|
|
||||||
replies.push(message);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.expect("timed out waiting for replies");
|
|
||||||
replies
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn ask(
|
|
||||||
host: &Host,
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
actor: &str,
|
|
||||||
body: Value,
|
|
||||||
) -> Envelope {
|
|
||||||
let id = host.broker.send("console", Some(actor), None, None, body);
|
|
||||||
receive_reply(mailbox, &id, REPLY_TIMEOUT).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn ask_type(
|
|
||||||
host: &Host,
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
actor: &str,
|
|
||||||
body: Value,
|
|
||||||
expected: &str,
|
|
||||||
) -> Envelope {
|
|
||||||
let reply = ask(host, mailbox, actor, body).await;
|
|
||||||
assert_eq!(reply.body["type"], expected, "unexpected reply: {reply:?}");
|
|
||||||
reply
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn spawn_request(host: &Host, actor: &str, wasm: &Path, source: &Path) -> String {
|
|
||||||
host.broker.send(
|
|
||||||
"console",
|
|
||||||
Some("broker"),
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
json!({
|
|
||||||
"type": "spawn",
|
|
||||||
"actor": actor,
|
|
||||||
"wasm": wasm.display().to_string(),
|
|
||||||
"source": source.display().to_string()
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn spawn_actor(
|
|
||||||
host: &Host,
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
actor: &str,
|
|
||||||
wasm: &Path,
|
|
||||||
source: &Path,
|
|
||||||
) -> Envelope {
|
|
||||||
let id = spawn_request(host, actor, wasm, source);
|
|
||||||
receive_reply(mailbox, &id, REPLY_TIMEOUT).await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn spawn_type(
|
|
||||||
host: &Host,
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
actor: &str,
|
|
||||||
wasm: &Path,
|
|
||||||
source: &Path,
|
|
||||||
) -> Envelope {
|
|
||||||
let reply = spawn_actor(host, mailbox, actor, wasm, source).await;
|
|
||||||
assert_eq!(
|
|
||||||
reply.body["type"], "spawned",
|
|
||||||
"unexpected spawn reply: {reply:?}"
|
|
||||||
);
|
|
||||||
reply
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn upgrade_actor(
|
|
||||||
host: &Host,
|
|
||||||
mailbox: &mut mpsc::Receiver<Envelope>,
|
|
||||||
actor: &str,
|
|
||||||
wasm: &Path,
|
|
||||||
source: &Path,
|
|
||||||
) -> Envelope {
|
|
||||||
request(
|
|
||||||
host,
|
|
||||||
mailbox,
|
|
||||||
json!({
|
|
||||||
"type": "upgrade",
|
|
||||||
"actor": actor,
|
|
||||||
"wasm": wasm.display().to_string(),
|
|
||||||
"source": source.display().to_string()
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
fn main() {
|
|
||||||
missing_function();
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
fn main() {}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
#[test]
|
|
||||||
fn deliberate_unit_test_failure() {
|
|
||||||
assert_eq!(2 + 2, 5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "cargo-valid"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2024"
|
|
||||||
|
|
||||||
[workspace]
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
fn main() {}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
#[test]
|
|
||||||
fn unit_tests_run_on_the_host_target() {
|
|
||||||
assert_eq!(2 + 2, 4);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-1150
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "toolbox"
|
name = "bench"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
[workspace]
|
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
//! {"type":"bench","target":"echo","n":1000}: ping-pong with the target, one
|
||||||
|
//! message at a time, and report the round-trip times.
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut out = std::io::stdout().lock();
|
||||||
|
let mut lines = std::io::stdin().lock().lines();
|
||||||
|
while let Some(Ok(line)) = lines.next() {
|
||||||
|
let Ok(msg) = serde_json::from_str::<Value>(&line) else { continue };
|
||||||
|
match msg["body"]["type"].as_str() {
|
||||||
|
Some("shutdown") => break,
|
||||||
|
Some("bench") => {
|
||||||
|
let target = msg["body"]["target"].as_str().unwrap_or("echo").to_string();
|
||||||
|
let n = msg["body"]["n"].as_u64().unwrap_or(1000);
|
||||||
|
let mut times = Vec::new();
|
||||||
|
let started = Instant::now();
|
||||||
|
for i in 0..n {
|
||||||
|
let t = Instant::now();
|
||||||
|
writeln!(out, "{}", json!({"to": target, "id": i, "body": {"type": "ping", "i": i}})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
// Wait for the reply.
|
||||||
|
loop {
|
||||||
|
let Some(Ok(l)) = lines.next() else { return };
|
||||||
|
let Ok(r) = serde_json::from_str::<Value>(&l) else { continue };
|
||||||
|
if r["reply_to"].as_str().is_some_and(|s| s.ends_with(&format!("/{i}"))) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
times.push(t.elapsed().as_micros() as u64);
|
||||||
|
}
|
||||||
|
let total = started.elapsed().as_secs_f64();
|
||||||
|
times.sort();
|
||||||
|
let p = |q: f64| times[((times.len() as f64 - 1.0) * q) as usize];
|
||||||
|
let reply = json!({"type": "bench_result", "n": n, "total_s": total,
|
||||||
|
"msgs_per_s": (n as f64 / total).round(),
|
||||||
|
"p50_us": p(0.5), "p90_us": p(0.9), "p99_us": p(0.99), "max_us": p(1.0)});
|
||||||
|
writeln!(out, "{}", json!({"to": msg["from"], "reply_to": msg["id"], "body": reply})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "cargo-fail"
|
name = "crasher"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[workspace]
|
[dependencies]
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
//! A bad version: it starts, waits 3 seconds (long enough for the smoke
|
||||||
|
//! test), then panics. Used to test the automatic rollback.
|
||||||
|
fn main() {
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||||
|
panic!("this version is broken");
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "cargo-test-fail"
|
name = "echo"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[workspace]
|
[dependencies]
|
||||||
|
serde_json = "1"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
use serde_json::{json, Value};
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut out = std::io::stdout().lock();
|
||||||
|
for line in std::io::stdin().lock().lines() {
|
||||||
|
let Ok(msg) = serde_json::from_str::<Value>(&line.unwrap()) else { continue };
|
||||||
|
if msg["body"]["type"] == "shutdown" {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let env = json!({"to": msg["from"], "reply_to": msg["id"], "body": {"type": "echo", "got": msg["body"]}});
|
||||||
|
writeln!(out, "{env}").unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
[package]
|
||||||
|
name = "fetch"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
serde_json = "1"
|
||||||
|
wstd = "0.6"
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
//! Fetches URLs over HTTP(S) and writes the body to /public/fetch/<n>.
|
||||||
|
//! Reads stdin and runs requests at the same time: while a fetch runs,
|
||||||
|
//! `ping` still answers at once.
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::cell::RefCell;
|
||||||
|
use std::rc::Rc;
|
||||||
|
use wstd::http::{Body, Client, Request};
|
||||||
|
use wstd::io::{AsyncRead, AsyncWrite};
|
||||||
|
|
||||||
|
type Out = Rc<RefCell<Vec<String>>>;
|
||||||
|
|
||||||
|
async fn flush(out: &Out) {
|
||||||
|
let lines: Vec<String> = out.borrow_mut().drain(..).collect();
|
||||||
|
let mut stdout = wstd::io::stdout();
|
||||||
|
for l in lines {
|
||||||
|
let _ = stdout.write_all(format!("{l}\n").as_bytes()).await;
|
||||||
|
}
|
||||||
|
let _ = stdout.flush().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch(n: u64, msg: Value, out: Out) {
|
||||||
|
let url = msg["body"]["url"].as_str().unwrap_or_default().to_string();
|
||||||
|
let started = wstd::time::Instant::now();
|
||||||
|
let result = async {
|
||||||
|
let req = Request::get(&url).body(Body::empty()).map_err(|e| e.to_string())?;
|
||||||
|
let resp = Client::new().send(req).await.map_err(|e| format!("{e:?}"))?;
|
||||||
|
let status = resp.status().as_u16();
|
||||||
|
let bytes = resp.into_body().contents().await.map_err(|e| format!("{e:?}"))?.to_vec();
|
||||||
|
std::fs::create_dir_all("/public/fetch").map_err(|e| e.to_string())?;
|
||||||
|
let path = format!("/public/fetch/{n}");
|
||||||
|
std::fs::write(&path, &bytes).map_err(|e| e.to_string())?;
|
||||||
|
Ok::<_, String>((status, bytes.len(), path))
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
let ms = started.elapsed().as_millis();
|
||||||
|
let body = match result {
|
||||||
|
Ok((status, size, path)) => {
|
||||||
|
let id = std::env::var("SAUSAGE_ACTOR").unwrap_or("fetch".into());
|
||||||
|
json!({"type": "fetched", "status": status, "bytes": size, "path": path.replacen("/public", &format!("/actor/{id}/public"), 1), "ms": ms})
|
||||||
|
}
|
||||||
|
Err(e) => json!({"type": "error", "reason": e, "ms": ms}),
|
||||||
|
};
|
||||||
|
out.borrow_mut().push(json!({"to": msg["from"], "reply_to": msg["id"], "body": body}).to_string());
|
||||||
|
flush(&out).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
wstd::runtime::block_on(async {
|
||||||
|
let out: Out = Rc::new(RefCell::new(Vec::new()));
|
||||||
|
let mut stdin = wstd::io::stdin();
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
let mut chunk = vec![0u8; 8192];
|
||||||
|
let mut n = 0u64;
|
||||||
|
loop {
|
||||||
|
let Ok(len) = stdin.read(&mut chunk).await else { break };
|
||||||
|
if len == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
buf.extend_from_slice(&chunk[..len]);
|
||||||
|
while let Some(pos) = buf.iter().position(|b| *b == b'\n') {
|
||||||
|
let line: Vec<u8> = buf.drain(..=pos).collect();
|
||||||
|
let Ok(msg) = serde_json::from_slice::<Value>(&line) else { continue };
|
||||||
|
match msg["body"]["type"].as_str() {
|
||||||
|
Some("shutdown") => return,
|
||||||
|
Some("fetch") => {
|
||||||
|
n += 1;
|
||||||
|
wstd::runtime::spawn(fetch(n, msg, out.clone())).detach();
|
||||||
|
}
|
||||||
|
Some("ping") => {
|
||||||
|
out.borrow_mut().push(json!({"to": msg["from"], "reply_to": msg["id"], "body": {"type": "pong"}}).to_string());
|
||||||
|
flush(&out).await;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
[package]
|
||||||
|
name = "fstest"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
serde_json = "1"
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
//! Filesystem probe for the spike. Keeps files open across messages.
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut open: HashMap<String, std::fs::File> = HashMap::new();
|
||||||
|
let mut out = std::io::stdout().lock();
|
||||||
|
for line in std::io::stdin().lock().lines() {
|
||||||
|
let Ok(line) = line else { break };
|
||||||
|
let Ok(msg) = serde_json::from_str::<Value>(&line) else { continue };
|
||||||
|
let b = &msg["body"];
|
||||||
|
let path = b["path"].as_str().unwrap_or_default().to_string();
|
||||||
|
let r = |res: std::io::Result<Value>| match res {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => json!({"type": "error", "error": e.to_string()}),
|
||||||
|
};
|
||||||
|
let reply = match b["type"].as_str() {
|
||||||
|
Some("shutdown") => break,
|
||||||
|
Some("write") => r(std::fs::write(&path, b["text"].as_str().unwrap_or_default()).map(|_| json!({"type": "ok"}))),
|
||||||
|
Some("open") => r((|| {
|
||||||
|
let mut f = std::fs::File::create(&path)?;
|
||||||
|
f.write_all(b["text"].as_str().unwrap_or_default().as_bytes())?;
|
||||||
|
open.insert(path.clone(), f);
|
||||||
|
Ok(json!({"type": "ok", "open": true}))
|
||||||
|
})()),
|
||||||
|
Some("append") => r((|| {
|
||||||
|
let mut f = std::fs::OpenOptions::new().append(true).open(&path)?;
|
||||||
|
f.write_all(b["text"].as_str().unwrap_or_default().as_bytes())?;
|
||||||
|
open.insert(path.clone(), f);
|
||||||
|
Ok(json!({"type": "ok", "open": true}))
|
||||||
|
})()),
|
||||||
|
Some("close") => json!({"type": "ok", "closed": open.remove(&path).is_some()}),
|
||||||
|
Some("read") => r(std::fs::read_to_string(&path).map(|t| json!({"type": "content", "text": t}))),
|
||||||
|
Some("list") => r(std::fs::read_dir(&path).map(|rd| {
|
||||||
|
let mut names: Vec<String> = rd.filter_map(|e| e.ok()).map(|e| e.file_name().to_string_lossy().to_string()).collect();
|
||||||
|
names.sort();
|
||||||
|
json!({"type": "entries", "names": names})
|
||||||
|
})),
|
||||||
|
Some("remove") => r(std::fs::remove_file(&path).map(|_| json!({"type": "ok"}))),
|
||||||
|
Some("crash") => panic!("asked to crash"),
|
||||||
|
_ => json!({"type": "error", "error": "unknown message"}),
|
||||||
|
};
|
||||||
|
writeln!(out, "{}", json!({"to": msg["from"], "reply_to": msg["id"], "body": reply})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
Generated
-115
@@ -1,115 +0,0 @@
|
|||||||
# This file is automatically @generated by Cargo.
|
|
||||||
# It is not intended for manual editing.
|
|
||||||
version = 4
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "itoa"
|
|
||||||
version = "1.0.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "memchr"
|
|
||||||
version = "2.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "notes"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"sausage-actor",
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "proc-macro2"
|
|
||||||
version = "1.0.107"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
||||||
dependencies = [
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "quote"
|
|
||||||
version = "1.0.47"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sausage-actor"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
||||||
dependencies = [
|
|
||||||
"serde_core",
|
|
||||||
"serde_derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_core"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
||||||
dependencies = [
|
|
||||||
"serde_derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_derive"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_json"
|
|
||||||
version = "1.0.154"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
||||||
dependencies = [
|
|
||||||
"itoa",
|
|
||||||
"memchr",
|
|
||||||
"serde",
|
|
||||||
"serde_core",
|
|
||||||
"zmij",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "syn"
|
|
||||||
version = "3.0.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-ident"
|
|
||||||
version = "1.0.27"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "zmij"
|
|
||||||
version = "1.0.23"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
||||||
@@ -4,7 +4,4 @@ version = "0.1.0"
|
|||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
sausage-actor = { version = "0.1", path = "../../sdk" }
|
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
[workspace]
|
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
//! Version 2 adds a note count and fixes the crash command.
|
//! Keeps a list of notes in /data/notes.json and publishes them as
|
||||||
use sausage_actor::{json, messages, reply};
|
//! /public/notes.txt. Version 2: adds `count` and no longer crashes.
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
const DATA: &str = "/data/notes.json";
|
const DATA: &str = "/data/notes.json";
|
||||||
|
|
||||||
@@ -11,21 +13,27 @@ fn save(notes: &[String]) {
|
|||||||
fn main() {
|
fn main() {
|
||||||
let mut notes: Vec<String> = std::fs::read_to_string(DATA)
|
let mut notes: Vec<String> = std::fs::read_to_string(DATA)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|text| serde_json::from_str(&text).ok())
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
for message in messages() {
|
let mut out = std::io::stdout().lock();
|
||||||
let body = &message.body;
|
for line in std::io::stdin().lock().lines() {
|
||||||
let response = match message.kind() {
|
let Ok(line) = line else { break };
|
||||||
"add" => {
|
let Ok(msg) = serde_json::from_str::<Value>(&line) else { continue };
|
||||||
|
let body = &msg["body"];
|
||||||
|
let reply = match body["type"].as_str() {
|
||||||
|
Some("shutdown") => break,
|
||||||
|
Some("add") => {
|
||||||
notes.push(body["text"].as_str().unwrap_or_default().to_string());
|
notes.push(body["text"].as_str().unwrap_or_default().to_string());
|
||||||
save(¬es);
|
save(¬es);
|
||||||
json!({"type": "added", "count": notes.len()})
|
json!({"type": "added", "count": notes.len()})
|
||||||
}
|
}
|
||||||
"list" => json!({"type": "notes", "items": notes}),
|
Some("list") => json!({"type": "notes", "items": notes}),
|
||||||
"count" => json!({"type": "count", "count": notes.len(), "version": 2}),
|
Some("count") => json!({"type": "count", "count": notes.len(), "version": 2}),
|
||||||
"crash" => json!({"type": "not_crashing", "version": 2}),
|
// Version 2 fixes the crash.
|
||||||
|
Some("crash") => json!({"type": "not_crashing", "version": 2}),
|
||||||
_ => json!({"type": "error", "reason": "unknown message"}),
|
_ => json!({"type": "error", "reason": "unknown message"}),
|
||||||
};
|
};
|
||||||
let _ = reply(&message, response);
|
writeln!(out, "{}", json!({"to": msg["from"], "reply_to": msg["id"], "body": reply})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
{
|
|
||||||
"description": "version 2 counts notes and no longer crashes",
|
|
||||||
"steps": [
|
|
||||||
{"send": {"type": "add", "text": "test note"}, "expect": {"type": "added"}},
|
|
||||||
{"send": {"type": "list"}, "expect": {"type": "notes"}},
|
|
||||||
{"send": {"type": "count"}, "expect": {"type": "count", "version": 2}},
|
|
||||||
{"send": {"type": "crash"}, "expect": {"type": "not_crashing"}}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{"send": {"type": "add", "text": "test note"}}
|
||||||
|
{"expect": {"type": "added"}}
|
||||||
|
{"send": {"type": "list"}}
|
||||||
|
{"expect": {"type": "notes"}}
|
||||||
|
{"send": {"type": "count"}}
|
||||||
|
{"expect": {"type": "count", "version": 2}}
|
||||||
|
{"send": {"type": "crash"}}
|
||||||
|
{"expect": {"type": "not_crashing"}}
|
||||||
Generated
-115
@@ -1,115 +0,0 @@
|
|||||||
# This file is automatically @generated by Cargo.
|
|
||||||
# It is not intended for manual editing.
|
|
||||||
version = 4
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "itoa"
|
|
||||||
version = "1.0.18"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "memchr"
|
|
||||||
version = "2.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "notes"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"sausage-actor",
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "proc-macro2"
|
|
||||||
version = "1.0.107"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
||||||
dependencies = [
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "quote"
|
|
||||||
version = "1.0.47"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sausage-actor"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
||||||
dependencies = [
|
|
||||||
"serde_core",
|
|
||||||
"serde_derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_core"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
||||||
dependencies = [
|
|
||||||
"serde_derive",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_derive"
|
|
||||||
version = "1.0.229"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_json"
|
|
||||||
version = "1.0.154"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
|
||||||
dependencies = [
|
|
||||||
"itoa",
|
|
||||||
"memchr",
|
|
||||||
"serde",
|
|
||||||
"serde_core",
|
|
||||||
"zmij",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "syn"
|
|
||||||
version = "3.0.9"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "unicode-ident"
|
|
||||||
version = "1.0.27"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "zmij"
|
|
||||||
version = "1.0.23"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
||||||
@@ -4,7 +4,4 @@ version = "0.1.0"
|
|||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
sausage-actor = { version = "0.1", path = "../../sdk" }
|
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
[workspace]
|
|
||||||
|
|||||||
+17
-10
@@ -1,5 +1,7 @@
|
|||||||
//! Keep notes in `/data/notes.json` and publish them at `/public/notes.txt`.
|
//! Keeps a list of notes in /data/notes.json and publishes them as
|
||||||
use sausage_actor::{json, messages, reply};
|
//! /public/notes.txt.
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
use std::io::{BufRead, Write};
|
||||||
|
|
||||||
const DATA: &str = "/data/notes.json";
|
const DATA: &str = "/data/notes.json";
|
||||||
|
|
||||||
@@ -11,20 +13,25 @@ fn save(notes: &[String]) {
|
|||||||
fn main() {
|
fn main() {
|
||||||
let mut notes: Vec<String> = std::fs::read_to_string(DATA)
|
let mut notes: Vec<String> = std::fs::read_to_string(DATA)
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|text| serde_json::from_str(&text).ok())
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
for message in messages() {
|
let mut out = std::io::stdout().lock();
|
||||||
let body = &message.body;
|
for line in std::io::stdin().lock().lines() {
|
||||||
let response = match message.kind() {
|
let Ok(line) = line else { break };
|
||||||
"add" => {
|
let Ok(msg) = serde_json::from_str::<Value>(&line) else { continue };
|
||||||
|
let body = &msg["body"];
|
||||||
|
let reply = match body["type"].as_str() {
|
||||||
|
Some("shutdown") => break,
|
||||||
|
Some("add") => {
|
||||||
notes.push(body["text"].as_str().unwrap_or_default().to_string());
|
notes.push(body["text"].as_str().unwrap_or_default().to_string());
|
||||||
save(¬es);
|
save(¬es);
|
||||||
json!({"type": "added", "count": notes.len()})
|
json!({"type": "added", "count": notes.len()})
|
||||||
}
|
}
|
||||||
"list" => json!({"type": "notes", "items": notes}),
|
Some("list") => json!({"type": "notes", "items": notes}),
|
||||||
"crash" => panic!("asked to crash"),
|
Some("crash") => panic!("asked to crash"),
|
||||||
_ => json!({"type": "error", "reason": "unknown message"}),
|
_ => json!({"type": "error", "reason": "unknown message"}),
|
||||||
};
|
};
|
||||||
let _ = reply(&message, response);
|
writeln!(out, "{}", json!({"to": msg["from"], "reply_to": msg["id"], "body": reply})).unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
{
|
|
||||||
"description": "add a note, then list the notes",
|
|
||||||
"steps": [
|
|
||||||
{"send": {"type": "add", "text": "test note"}, "expect": {"type": "added"}},
|
|
||||||
{"send": {"type": "list"}, "expect": {"type": "notes"}}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{"send": {"type": "add", "text": "test note"}}
|
||||||
|
{"expect": {"type": "added"}}
|
||||||
|
{"send": {"type": "list"}}
|
||||||
|
{"expect": {"type": "notes"}}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
[package]
|
||||||
|
name = "ticker"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2024"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
serde_json = "1"
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
//! Sends {"type":"add","text":"tick N"} to `notes` every 50 ms, N = 1..=200.
|
||||||
|
//! Does not read stdin, so the broker kills it at shutdown; that is fine.
|
||||||
|
use std::io::Write;
|
||||||
|
fn main() {
|
||||||
|
let mut out = std::io::stdout().lock();
|
||||||
|
for n in 1..=200 {
|
||||||
|
let msg = serde_json::json!({"to": "notes", "body": {"type": "add", "text": format!("tick {n}")}});
|
||||||
|
writeln!(out, "{msg}").unwrap();
|
||||||
|
out.flush().unwrap();
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||||
|
}
|
||||||
|
// Stay alive without doing anything.
|
||||||
|
loop {
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(3600));
|
||||||
|
}
|
||||||
|
}
|
||||||
Generated
-50
@@ -1,38 +1,5 @@
|
|||||||
{
|
{
|
||||||
"nodes": {
|
"nodes": {
|
||||||
"crane": {
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1791754810,
|
|
||||||
"narHash": "sha256-cN0Mgk+sQtCJcGWs9H4M0CnLRAuC4I8aCwiBTcBJs6E=",
|
|
||||||
"owner": "ipetkov",
|
|
||||||
"repo": "crane",
|
|
||||||
"rev": "bd63bf1d0a0f6dcacd7549c714a671ee63b0db6a",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "ipetkov",
|
|
||||||
"repo": "crane",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"flake-utils": {
|
|
||||||
"inputs": {
|
|
||||||
"systems": "systems"
|
|
||||||
},
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1791914995,
|
|
||||||
"narHash": "sha256-Zf/xfQEAasnwnmEfHqW/4hjDE/s44yn2hD4RuVIZRjA=",
|
|
||||||
"owner": "numtide",
|
|
||||||
"repo": "flake-utils",
|
|
||||||
"rev": "b7a79d3fea1e214f6d668d521cecec0c01e26544",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "numtide",
|
|
||||||
"repo": "flake-utils",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1791637130,
|
"lastModified": 1791637130,
|
||||||
@@ -51,8 +18,6 @@
|
|||||||
},
|
},
|
||||||
"root": {
|
"root": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"crane": "crane",
|
|
||||||
"flake-utils": "flake-utils",
|
|
||||||
"nixpkgs": "nixpkgs",
|
"nixpkgs": "nixpkgs",
|
||||||
"rust-overlay": "rust-overlay"
|
"rust-overlay": "rust-overlay"
|
||||||
}
|
}
|
||||||
@@ -76,21 +41,6 @@
|
|||||||
"repo": "rust-overlay",
|
"repo": "rust-overlay",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"systems": {
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1681028828,
|
|
||||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
|
||||||
"owner": "nix-systems",
|
|
||||||
"repo": "default",
|
|
||||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "nix-systems",
|
|
||||||
"repo": "default",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"root": "root",
|
"root": "root",
|
||||||
|
|||||||
@@ -1,171 +1,26 @@
|
|||||||
{
|
{
|
||||||
description = "Sausage: grow a system of WASI actors by talking to it";
|
description = "Sausage: WASI actors with a message broker";
|
||||||
|
|
||||||
inputs = {
|
inputs = {
|
||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||||
flake-utils.url = "github:numtide/flake-utils";
|
|
||||||
rust-overlay.url = "github:oxalica/rust-overlay";
|
rust-overlay.url = "github:oxalica/rust-overlay";
|
||||||
rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
|
rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
|
||||||
crane.url = "github:ipetkov/crane";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
outputs = { nixpkgs, flake-utils, rust-overlay, crane, ... }:
|
outputs = { nixpkgs, rust-overlay, ... }:
|
||||||
flake-utils.lib.eachSystem flake-utils.lib.allSystems (system:
|
|
||||||
let
|
let
|
||||||
pkgs = import nixpkgs {
|
system = "x86_64-linux";
|
||||||
inherit system;
|
pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default ]; };
|
||||||
overlays = [ rust-overlay.overlays.default ];
|
|
||||||
};
|
|
||||||
# The toolchain that builds the host and, at runtime, the actors.
|
|
||||||
rust = pkgs.rust-bin.stable.latest.default.override {
|
rust = pkgs.rust-bin.stable.latest.default.override {
|
||||||
targets = [ "wasm32-wasip2" ];
|
targets = [ "wasm32-wasip2" ];
|
||||||
extensions = [ "rust-src" ];
|
extensions = [ "rust-src" ];
|
||||||
};
|
};
|
||||||
# The runtime cargo actor needs Cargo, rustc (with wasm32-wasip2), and
|
in {
|
||||||
# the native linker, even when the host itself is started with a bare
|
devShells.${system}.default = pkgs.mkShell {
|
||||||
# PATH. Keep this same runtime path in the packaged host and exercise
|
packages = [ rust pkgs.wasmtime pkgs.wasm-tools pkgs.pkg-config pkgs.openssl ];
|
||||||
# the installed binary below with PATH deliberately empty.
|
|
||||||
runtimePath = pkgs.lib.makeBinPath [ rust pkgs.stdenv.cc ];
|
|
||||||
craneLib = (crane.mkLib pkgs).overrideToolchain rust;
|
|
||||||
common = {
|
|
||||||
pname = "sausage";
|
|
||||||
version = "0.1.0";
|
|
||||||
src = ./.;
|
|
||||||
};
|
|
||||||
# The test packages for `checks.tests`.
|
|
||||||
testPackages = "-p sausage -p sausage-actor -p agent";
|
|
||||||
cargoArtifacts = craneLib.buildDepsOnly (common // {
|
|
||||||
doCheck = false;
|
|
||||||
# The dependencies of the host binary and of the tests.
|
|
||||||
buildPhaseCargoCommand = ''
|
|
||||||
cargoWithProfile build --locked -p sausage
|
|
||||||
cargoWithProfile test --locked --no-run ${testPackages}
|
|
||||||
'';
|
|
||||||
});
|
|
||||||
sausage = craneLib.buildPackage (common // {
|
|
||||||
inherit cargoArtifacts;
|
|
||||||
doNotRemoveReferencesToRustToolchain = true;
|
|
||||||
cargoExtraArgs = "--locked -p sausage";
|
|
||||||
doCheck = false;
|
|
||||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
|
||||||
# The `cargo` actor runs cargo at runtime. Build scripts need a C linker.
|
|
||||||
postInstall = ''
|
|
||||||
wrapProgram $out/bin/sausage --prefix PATH : ${runtimePath}
|
|
||||||
'';
|
|
||||||
meta.mainProgram = "sausage";
|
|
||||||
});
|
|
||||||
# `nix run .#loop -- <world>` runs the newest `sausage run <world>`
|
|
||||||
# (without a folder: bare `sausage`, i.e. ./world, created if needed)
|
|
||||||
# from the public repo, again and again: after a crash or a broker
|
|
||||||
# `exit` request it refreshes the flake and starts the host again,
|
|
||||||
# so the world updates itself. `/quit` (exit code 0) ends the loop.
|
|
||||||
# SAUSAGE_FLAKE overrides where it fetches from (any flake URL,
|
|
||||||
# e.g. a fork or a local checkout).
|
|
||||||
repoUrl = "git+https://gitea.surma.technology/surma/sausage";
|
|
||||||
loop = pkgs.writeShellScriptBin "sausage-loop" ''
|
|
||||||
url="''${SAUSAGE_FLAKE:-${repoUrl}}"
|
|
||||||
# No folder: bare `sausage` (creates ./world first if needed).
|
|
||||||
if [ $# -gt 0 ]; then set -- run "$@"; fi
|
|
||||||
while true; do
|
|
||||||
nix run --refresh "$url#default" -- "$@"
|
|
||||||
code=$?
|
|
||||||
if [ "$code" -eq 0 ]; then exit 0; fi
|
|
||||||
echo "sausage-loop: sausage exited with $code, starting the newest version in 2 s …" >&2
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
'';
|
|
||||||
in
|
|
||||||
{
|
|
||||||
packages.default = sausage;
|
|
||||||
|
|
||||||
# `nix flake check` runs every test of the host, the SDK, and the
|
|
||||||
# agent. The tests build WASM actors with cargo, offline, from the
|
|
||||||
# vendored crates of Cargo.lock.
|
|
||||||
# The binary the host loop runs (`nix run`), built from the same
|
|
||||||
# cached dependencies as the tests.
|
|
||||||
checks.package = sausage;
|
|
||||||
checks.tests = craneLib.cargoTest (common // {
|
|
||||||
inherit cargoArtifacts;
|
|
||||||
cargoExtraArgs = "--locked ${testPackages}";
|
|
||||||
# Nothing builds on the test run's target folder: don't store it.
|
|
||||||
doInstallCargoArtifacts = false;
|
|
||||||
preCheck = ''
|
|
||||||
# Its own lock file pins crates that the vendored set may lack.
|
|
||||||
rm -f cli/tests/actors/toolbox/Cargo.lock
|
|
||||||
'';
|
|
||||||
});
|
|
||||||
# Exercise the installed host exactly as the loop launches it: the
|
|
||||||
# caller has no Cargo on PATH, so only the packaged runtime toolchain
|
|
||||||
# can run the native tests and the wasm32-wasip2 build.
|
|
||||||
checks.runtime-toolchain = pkgs.runCommand "sausage-runtime-toolchain" { } ''
|
|
||||||
set -eu
|
|
||||||
root="$TMPDIR/runtime-toolchain"
|
|
||||||
world="$root/world"
|
|
||||||
emptyPath="$root/empty-bin"
|
|
||||||
mkdir -p "$emptyPath" "$root/home" "$root/cargo-home"
|
|
||||||
for actor in broker console cargo fs; do
|
|
||||||
mkdir -p "$world/actor/$actor/public" "$world/actor/$actor/data"
|
|
||||||
done
|
|
||||||
source="$world/actor/console/public/probe"
|
|
||||||
mkdir -p "$source/src" "$source/tests"
|
|
||||||
cat > "$source/Cargo.toml" <<'TOML'
|
|
||||||
[package]
|
|
||||||
name = "runtime-probe"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2024"
|
|
||||||
TOML
|
|
||||||
printf 'fn main() {}\n' > "$source/src/main.rs"
|
|
||||||
cat > "$source/tests/probe.rs" <<'RUST'
|
|
||||||
#[test]
|
|
||||||
fn native_test_harness_runs() {
|
|
||||||
assert_eq!(2 + 2, 4);
|
|
||||||
}
|
|
||||||
RUST
|
|
||||||
echo "runtime-check: run packaged host with an empty PATH"
|
|
||||||
coproc host {
|
|
||||||
${pkgs.coreutils}/bin/env -i PATH="$emptyPath" HOME="$root/home" TMPDIR="$root" \
|
|
||||||
CARGO_HOME="$root/cargo-home" ${sausage}/bin/sausage run "$world"
|
|
||||||
}
|
|
||||||
hostInput=''${host[1]}
|
|
||||||
hostOutput=''${host[0]}
|
|
||||||
hostPid=$host_PID
|
|
||||||
trap 'kill "$hostPid" 2>/dev/null || true' EXIT
|
|
||||||
printf '%s\n' '@cargo {"type":"build","source":"/actor/console/public/probe"}' >&"$hostInput"
|
|
||||||
built=0
|
|
||||||
while IFS= read -r -t 300 line <&"$hostOutput"; do
|
|
||||||
echo "$line"
|
|
||||||
case "$line" in
|
|
||||||
*'"type":"build_failed"'*)
|
|
||||||
echo "packaged cargo actor build failed with an empty caller PATH" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
*'"type":"built"'*)
|
|
||||||
case "$line" in
|
|
||||||
*'"passed":1'*) built=1 ;;
|
|
||||||
*) echo "native test did not pass" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
if [ "$built" -ne 1 ]; then
|
|
||||||
echo "packaged host did not build the probe" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
printf '%s\n' '/quit' >&"$hostInput"
|
|
||||||
wait "$hostPid"
|
|
||||||
mkdir -p "$out"
|
|
||||||
touch "$out/passed"
|
|
||||||
'';
|
|
||||||
|
|
||||||
apps.default = { type = "app"; program = "${sausage}/bin/sausage"; };
|
|
||||||
apps.loop = { type = "app"; program = "${loop}/bin/sausage-loop"; };
|
|
||||||
|
|
||||||
devShells.default = pkgs.mkShell {
|
|
||||||
packages = [ rust pkgs.wasmtime pkgs.wasm-tools ];
|
|
||||||
shellHook = ''
|
shellHook = ''
|
||||||
export CARGO_HOME="$PWD/.cargo-home"
|
export CARGO_HOME="$PWD/.cargo-home"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
});
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +0,0 @@
|
|||||||
[package]
|
|
||||||
name = "sausage-actor"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2024"
|
|
||||||
description = "SDK for Sausage actors."
|
|
||||||
|
|
||||||
[features]
|
|
||||||
default = []
|
|
||||||
async = ["dep:async-channel", "dep:wstd"]
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
async-channel = { version = "2", optional = true }
|
|
||||||
serde = { version = "1", features = ["derive"] }
|
|
||||||
serde_json = "1"
|
|
||||||
wstd = { version = "0.6", optional = true }
|
|
||||||
-229
@@ -1,229 +0,0 @@
|
|||||||
//! Small runtime helpers for Sausage actors.
|
|
||||||
|
|
||||||
pub use serde_json::{Value, json};
|
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use std::io::BufRead;
|
|
||||||
|
|
||||||
/// A message that an actor receives from the broker.
|
|
||||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
|
||||||
pub struct Message {
|
|
||||||
/// The message ID.
|
|
||||||
pub id: String,
|
|
||||||
/// The ID of the sending actor.
|
|
||||||
pub from: String,
|
|
||||||
/// The ID of the message that this message answers.
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub reply_to: Option<String>,
|
|
||||||
/// The message body.
|
|
||||||
pub body: Value,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Message {
|
|
||||||
/// Return the body's `type` string, or an empty string when it is missing.
|
|
||||||
pub fn kind(&self) -> &str {
|
|
||||||
self.body["type"].as_str().unwrap_or("")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse(line: &[u8]) -> Option<Message> {
|
|
||||||
serde_json::from_slice::<Message>(line).ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_shutdown(message: &Message) -> bool {
|
|
||||||
message.from == "broker" && message.kind() == "shutdown"
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return this actor's ID from `SAUSAGE_ACTOR`, or an empty string if unset.
|
|
||||||
pub fn me() -> String {
|
|
||||||
std::env::var("SAUSAGE_ACTOR").unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read messages from stdin until EOF or a shutdown message from the broker.
|
|
||||||
///
|
|
||||||
/// Invalid JSON lines are skipped.
|
|
||||||
pub fn messages() -> impl Iterator<Item = Message> {
|
|
||||||
let stdin = std::io::stdin();
|
|
||||||
let mut lines = stdin.lock().lines();
|
|
||||||
std::iter::from_fn(move || {
|
|
||||||
loop {
|
|
||||||
let Ok(line) = lines.next()? else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let Some(message) = parse(line.as_bytes()) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
return (!is_shutdown(&message)).then_some(message);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Send a message to `to` and return its full ID (`<actor>/<own-id>`).
|
|
||||||
pub fn send(to: &str, body: Value) -> String {
|
|
||||||
let id = next_id();
|
|
||||||
write_message(json!({"to": to, "id": id, "body": body}));
|
|
||||||
format!("{}/{id}", me())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reply to `message` and return the new message's full ID.
|
|
||||||
pub fn reply(message: &Message, body: Value) -> String {
|
|
||||||
let id = next_id();
|
|
||||||
write_message(json!({
|
|
||||||
"to": message.from,
|
|
||||||
"id": id,
|
|
||||||
"reply_to": message.id,
|
|
||||||
"body": body,
|
|
||||||
}));
|
|
||||||
format!("{}/{id}", me())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Write a file so that readers see either the old or the new content,
|
|
||||||
/// never half of it: write a temporary file next to `path`, then rename it.
|
|
||||||
pub fn write_atomic(path: impl AsRef<std::path::Path>, contents: impl AsRef<[u8]>) -> std::io::Result<()> {
|
|
||||||
let path = path.as_ref();
|
|
||||||
let tmp = path.with_file_name(format!(".{}.tmp-{}", path.file_name().unwrap_or_default().to_string_lossy(), next_id()));
|
|
||||||
std::fs::write(&tmp, contents)?;
|
|
||||||
std::fs::rename(&tmp, path)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Publish a message to this actor's subscribers and return its full ID.
|
|
||||||
pub fn publish(body: Value) -> String {
|
|
||||||
let id = next_id();
|
|
||||||
write_message(json!({"id": id, "body": body}));
|
|
||||||
format!("{}/{id}", me())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_message(message: Value) {
|
|
||||||
use std::io::Write;
|
|
||||||
|
|
||||||
let mut stdout = std::io::stdout().lock();
|
|
||||||
writeln!(stdout, "{message}").unwrap();
|
|
||||||
stdout.flush().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn next_id() -> String {
|
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
|
||||||
|
|
||||||
static START: OnceLock<u128> = OnceLock::new();
|
|
||||||
static SEQUENCE: AtomicUsize = AtomicUsize::new(0);
|
|
||||||
let start = START.get_or_init(|| {
|
|
||||||
SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_nanos()
|
|
||||||
});
|
|
||||||
format!("{start}-{}", SEQUENCE.fetch_add(1, Ordering::Relaxed))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// An asynchronous inbox for actors that use HTTP or timers.
|
|
||||||
#[cfg(feature = "async")]
|
|
||||||
pub struct Inbox {
|
|
||||||
rx: async_channel::Receiver<Message>,
|
|
||||||
_reader: wstd::runtime::Task<()>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "async")]
|
|
||||||
impl Inbox {
|
|
||||||
/// Start the stdin reader inside the wstd runtime and return its inbox.
|
|
||||||
/// The inbox owns the reader task and must be dropped to cancel it.
|
|
||||||
pub fn start() -> Self {
|
|
||||||
let (tx, rx) = async_channel::unbounded();
|
|
||||||
let reader = wstd::runtime::spawn(read_messages(tx));
|
|
||||||
Self { rx, _reader: reader }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Wait for the next message, or return `None` at EOF or shutdown.
|
|
||||||
pub async fn recv(&self) -> Option<Message> {
|
|
||||||
self.rx.recv().await.ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the next available message without waiting.
|
|
||||||
pub fn try_recv(&self) -> Option<Message> {
|
|
||||||
self.rx.try_recv().ok()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return whether the reader closed the inbox, even when messages remain buffered.
|
|
||||||
pub fn is_closed(&self) -> bool {
|
|
||||||
self.rx.is_closed()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "async")]
|
|
||||||
async fn read_messages(tx: async_channel::Sender<Message>) {
|
|
||||||
use wstd::io::AsyncRead;
|
|
||||||
|
|
||||||
let mut stdin = wstd::io::stdin();
|
|
||||||
let mut buf = Vec::new();
|
|
||||||
let mut chunk = [0; 8192];
|
|
||||||
loop {
|
|
||||||
let Ok(n) = stdin.read(&mut chunk).await else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
if n == 0 {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
buf.extend_from_slice(&chunk[..n]);
|
|
||||||
while let Some(pos) = buf.iter().position(|b| *b == b'\n') {
|
|
||||||
let message = parse(&buf[..pos]);
|
|
||||||
buf.drain(..=pos);
|
|
||||||
let Some(message) = message else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if is_shutdown(&message) {
|
|
||||||
tx.close();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if tx.send(message).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tx.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kind_reads_body_type() {
|
|
||||||
let message: Message = serde_json::from_value(json!({
|
|
||||||
"id": "broker/1",
|
|
||||||
"from": "broker",
|
|
||||||
"body": {"type": "ping"}
|
|
||||||
}))
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(message.kind(), "ping");
|
|
||||||
assert_eq!(message.reply_to, None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn kind_is_empty_when_type_is_missing() {
|
|
||||||
let message: Message = serde_json::from_value(json!({
|
|
||||||
"id": "broker/1",
|
|
||||||
"from": "broker",
|
|
||||||
"body": {}
|
|
||||||
}))
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(message.kind(), "");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn write_atomic_replaces_the_file() {
|
|
||||||
let dir = std::env::temp_dir().join(format!("sausage-actor-{}", next_id()));
|
|
||||||
std::fs::create_dir_all(&dir).unwrap();
|
|
||||||
let path = dir.join("state.json");
|
|
||||||
write_atomic(&path, "old").unwrap();
|
|
||||||
write_atomic(&path, "new").unwrap();
|
|
||||||
assert_eq!(std::fs::read_to_string(&path).unwrap(), "new");
|
|
||||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1);
|
|
||||||
std::fs::remove_dir_all(dir).unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn generated_ids_differ() {
|
|
||||||
assert_ne!(next_id(), next_id());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user