diff --git a/README.md b/README.md index 852ac42..4d9d483 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ nix run git+https://gitea.surma.technology/surma/sausage -- run # starts the `init` asks nothing. It writes `world/config/agent.yaml`, with a comment for each field and an example for each API. Edit it before the first chat: choose -the API, the base URL, the model, and the key. Until then, a chat answers with +the API, the base URL, the model, the key, and the thinking level. Until then, a chat answers with an error that names the missing field. Both commands take another folder as an argument, for example `... -- run my-world`. diff --git a/agent/src/llm.rs b/agent/src/llm.rs index 019fb21..5d24396 100644 --- a/agent/src/llm.rs +++ b/agent/src/llm.rs @@ -3,6 +3,7 @@ //! The conversation is stored in a neutral format, so the agent can switch //! between the three shapes. +use sausage_plugin::schema; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; #[cfg(target_os = "wasi")] @@ -22,17 +23,50 @@ pub struct Config { pub token: String, /// Permit an empty token and omit authentication headers. pub allow_no_token: bool, - /// Optional reasoning effort for OpenAI models only (wins over `thinking` there). - #[serde(default)] - pub reasoning_effort: Option, /// An optional cap on generated tokens. #[serde(default)] pub max_tokens: Option, - /// Thinking effort for all APIs: "low", "medium", "high", "xhigh", "max" (`LEVELS`), or "off"/unset for none. - /// Anthropic: adaptive thinking + output_config.effort; OpenAI: reasoning.effort. A level the model rejects is - /// replaced by the nearest accepted one (logged), never by "off". + /// The thinking level of each turn. Unset: the model's default. #[serde(default)] - pub thinking: Option, + pub thinking: Option, +} + +/// How much the model thinks before it answers. +/// +/// OpenAI Responses gets `reasoning.effort`, Chat Completions gets `reasoning_effort`, and Anthropic +/// gets adaptive thinking with `output_config.effort`. If the model rejects a level, the agent uses +/// the nearest level that the model accepts, never `off`. +#[schema] +#[derive(Clone, Copy, PartialEq, Debug)] +#[serde(rename_all = "lowercase")] +pub enum Thinking { + /// No thinking: OpenAI gets the effort `none`, and Anthropic gets disabled thinking. If an + /// OpenAI model cannot switch thinking off, the agent uses `low`. + Off, + /// The least thinking. + Low, + /// Moderate thinking. + Medium, + /// Much thinking. + High, + /// More thinking than `high`, where the model supports it. + Xhigh, + /// The most thinking that the model supports. + Max, +} + +impl Thinking { + /// The effort that an OpenAI request carries: `none`, or one of `LEVELS`. + fn effort(self) -> &'static str { + match self { + Thinking::Off => "none", + Thinking::Low => "low", + Thinking::Medium => "medium", + Thinking::High => "high", + Thinking::Xhigh => "xhigh", + Thinking::Max => "max", + } + } } impl Config { @@ -98,7 +132,11 @@ fn fallback_effort(want: &str, error: &str, tried: &[String]) -> Option } else { LEVELS.to_vec() }; - let at = LEVELS.iter().position(|l| *l == want).unwrap_or(1) as i32; + // `none` is below all levels, so its nearest level is `low`. + let at = LEVELS + .iter() + .position(|l| *l == want) + .map_or(-1, |i| i as i32); pool.into_iter() .filter(|l| *l != want && !tried.iter().any(|t| t == l)) .min_by_key(|l| { @@ -130,6 +168,22 @@ mod tests { } let cfg = Config::parse(b"api: openai-chat-completions\nbase_url: http://localhost/v1\nmodel: local\ntoken: ''\nallow_no_token: true").unwrap(); assert!(cfg.bearer().is_empty()); + assert_eq!(cfg.thinking, None); + } + + #[test] + fn thinking_levels() { + use super::{Config, LEVELS, Thinking}; + let base = "api: openai-responses\nbase_url: url\nmodel: model\ntoken: key\nthinking: "; + let parse = |level: &str| Config::parse(format!("{base}{level}").as_bytes()); + assert_eq!(parse("off").unwrap().thinking, Some(Thinking::Off)); + assert_eq!(Thinking::Off.effort(), "none"); + for level in LEVELS { + assert_eq!(parse(level).unwrap().thinking.unwrap().effort(), level); + } + let error = parse("hihg").err().unwrap(); + assert!(error.contains("/config/agent.yaml"), "{error}"); + assert!(error.contains("thinking"), "{error}"); } #[test] @@ -178,10 +232,16 @@ mod tests { fallback_effort("xhigh", anthropic, &["xhigh".into()]).as_deref(), Some("high") ); - let openai = "HTTP 400: Invalid value: 'ultra'. Supported values are: 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', and 'max'."; + let openai = "HTTP 400: Invalid value: 'max'. Supported values are: 'none', 'minimal', 'low', 'medium', and 'high'."; assert_eq!( - fallback_effort("ultra", openai, &["ultra".into()]).as_deref(), - Some("medium") + fallback_effort("max", openai, &["max".into()]).as_deref(), + Some("high") + ); + // A thinking-only model rejects `none`: use the lowest level. + let glm = "HTTP 400: GLM-5.3 is a thinking-only model; disabling thinking (reasoning_effort='none') is not supported."; + assert_eq!( + fallback_effort("none", glm, &["none".into()]).as_deref(), + Some("low") ); // No list in the error: step to the neighbour, never to off. assert_eq!( @@ -254,31 +314,14 @@ pub async fn complete( tools: &[Tool], history: &[Entry], ) -> Result, String> { - let want = match cfg.api.as_str() { - "openai-responses" | "openai-chat-completions" => cfg - .reasoning_effort - .clone() - .or_else(|| cfg.thinking.clone()), - _ => cfg.thinking.clone(), - } - .filter(|t| t != "off"); + let want = cfg.thinking.map(|level| level.effort().to_string()); let mut effort = want.clone(); let mut tried = Vec::new(); loop { let result = match cfg.api.as_str() { "openai-responses" => openai(cfg, system, tools, history, effort.as_deref()).await, "openai-chat-completions" => chat(cfg, system, tools, history, effort.as_deref()).await, - "anthropic-messages" => { - anthropic( - cfg, - system, - tools, - history, - want.is_some(), - effort.as_deref(), - ) - .await - } + "anthropic-messages" => anthropic(cfg, system, tools, history, effort.as_deref()).await, other => return Err(format!("unknown API shape {other:?}")), }; match (result, effort.clone()) { @@ -430,9 +473,9 @@ async fn anthropic( system: &str, tools: &[Tool], history: &[Entry], - thinking: bool, effort: Option<&str>, ) -> Result, String> { + let disabled = cfg.thinking == Some(Thinking::Off); // Merge consecutive blocks of the same role into one message. let mut messages: Vec<(String, Vec)> = Vec::new(); let mut push = |role: &str, block: Value| match messages.last_mut() { @@ -462,7 +505,7 @@ async fn anthropic( ), // Thinking blocks go back unchanged (with their signature): required next to tool use. Entry::Opaque { api, model, item } - if api == "anthropic-messages" && model == &cfg.model && thinking => + if api == "anthropic-messages" && model == &cfg.model && !disabled => { push("assistant", item.clone()) } @@ -486,10 +529,15 @@ async fn anthropic( "tools": tools, "max_tokens": cfg.max_tokens.unwrap_or(32000), }); - if thinking { - body["thinking"] = json!({"type": "adaptive"}); - if let Some(effort) = effort { - body["output_config"] = json!({"effort": effort}); + // Unset: the model's default. Newer models think by default. + match cfg.thinking { + None => {} + Some(Thinking::Off) => body["thinking"] = json!({"type": "disabled"}), + Some(_) => { + body["thinking"] = json!({"type": "adaptive"}); + if let Some(effort) = effort { + body["output_config"] = json!({"effort": effort}); + } } } let url = format!("{}/v1/messages", cfg.base_url.trim_end_matches('/')); diff --git a/agent/src/main.rs b/agent/src/main.rs index 933406b..83032fc 100644 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -4,7 +4,7 @@ mod llm; mod tools; use history::{is_too_large, next_turn_start}; -use llm::{Config, Entry}; +use llm::{Config, Entry, Thinking}; use sausage_plugin::{Ctx, Error, Result, lock, rpc, schema}; use serde_json::json; use std::io::Write; @@ -22,6 +22,10 @@ struct Agent; struct ChatParams { /// The complete text of the person's request. text: String, + /// The thinking level of this turn only. Without it, the turn uses `thinking` from + /// `/config/agent.yaml`. + #[serde(default, skip_serializing_if = "Option::is_none")] + thinking: Option, } #[rpc(grace_seconds = 600)] @@ -32,7 +36,8 @@ impl Agent { async fn chat(&self, ctx: &Ctx, p: ChatParams) -> Result { let _guard = lock(HISTORY).await?; std::fs::create_dir_all("/state/agent")?; - let cfg = Config::load().map_err(|error| Error::new(-32603, error))?; + let mut cfg = Config::load().map_err(|error| Error::new(-32603, error))?; + cfg.thinking = p.thinking.or(cfg.thinking); let mut entries = load_history(HISTORY)?; repair_tools(&mut entries)?; let current = entries.len(); diff --git a/cli/src/agent.yaml b/cli/src/agent.yaml index 07d4c99..ceefeb2 100644 --- a/cli/src/agent.yaml +++ b/cli/src/agent.yaml @@ -4,32 +4,37 @@ api: openai-responses # OpenAI base URLs include /v1. Do not include /responses or /chat/completions. base_url: https://api.openai.com/v1 # model is the provider's model identifier. -model: gpt-4.1 +model: gpt-6.1-sol # token is the API key. Replace CHANGE_ME with your key. # An empty token requires allow_no_token: true. Then no authentication header is sent. token: CHANGE_ME allow_no_token: false +# thinking is how much the model thinks before it answers: +# off, low, medium, high, xhigh, or max. Remove the line to use the model's default, +# for example for a model without thinking. +# If the model rejects a level, the agent uses the nearest level that the model accepts. +# If the model cannot switch thinking off, off becomes low. +# A caller can set another level for one turn: agent.chat {"text": "...", "thinking": "high"}. +# The agent reads this file at the start of each turn, so a change applies to the next turn. +thinking: medium # Optional output token cap. Anthropic defaults to 32000 if omitted. # max_tokens: 32000 -# Optional thinking effort: off, low, medium, high, xhigh, or max. -# The agent retries a rejected effort with the nearest accepted level. -# thinking: high -# Optional OpenAI override for thinking effort. -# reasoning_effort: high # OpenAI Responses example (base_url includes the version): # api: openai-responses # base_url: https://api.openai.com/v1 -# model: gpt-4.1 +# model: gpt-6.1-sol # token: YOUR_OPENAI_KEY # allow_no_token: false +# thinking: medium # OpenRouter example (OpenAI Chat Completions, the format of most providers): # api: openai-chat-completions # base_url: https://openrouter.ai/api/v1 -# model: anthropic/claude-sonnet-4.5 +# model: anthropic/claude-sonnet-5.5 # token: YOUR_OPENROUTER_KEY # allow_no_token: false +# thinking: medium # Local Ollama example (OpenAI Chat Completions, no key): # api: openai-chat-completions @@ -41,6 +46,7 @@ allow_no_token: false # Anthropic Messages example (base_url excludes the version): # api: anthropic-messages # base_url: https://api.anthropic.com -# model: claude-sonnet-4-5 +# model: claude-sonnet-5-5 # token: YOUR_ANTHROPIC_KEY # allow_no_token: false +# thinking: medium diff --git a/cli/tests/host/agent.rs b/cli/tests/host/agent.rs index 35f80f2..a33d08a 100644 --- a/cli/tests/host/agent.rs +++ b/cli/tests/host/agent.rs @@ -241,6 +241,10 @@ async fn unconfigured_chat_names_the_configuration() { "api: openai-responses\nbase_url: http://localhost/v1\nmodel: test\ntoken: CHANGE_ME", "token", ), + ( + "api: openai-responses\nbase_url: http://localhost/v1\nmodel: test\ntoken: key\nthinking: hihg", + "thinking", + ), ] { std::fs::write(&path, config).unwrap(); let error = host @@ -507,11 +511,7 @@ async fn private_items_are_scoped_to_the_model() { for api in [APIS[0], APIS[1]] { let mut server = Llm::new(api).await; let world = TestWorld::new(&server); - let config_path = world.0.join("config/agent.yaml"); - let mut config: Value = - serde_yaml_ng::from_slice(&std::fs::read(&config_path).unwrap()).unwrap(); - config["thinking"] = json!("high"); - std::fs::write(config_path, serde_yaml_ng::to_string(&config).unwrap()).unwrap(); + // Without a thinking level, newer models think by default, so their items return. std::fs::create_dir_all(world.0.join("state/agent")).unwrap(); let mut entries = vec![json!({"kind":"user","text":"old turn"})]; for (model, marker) in [ @@ -546,6 +546,60 @@ async fn private_items_are_scoped_to_the_model() { } } +/// Each API gets the configured thinking level. A level in the call applies to that turn only. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn thinking_levels_reach_each_api() { + for api in APIS { + let mut server = Llm::new(api).await; + let world = TestWorld::new(&server); + let config_path = world.0.join("config/agent.yaml"); + let mut config: Value = + serde_yaml_ng::from_slice(&std::fs::read(&config_path).unwrap()).unwrap(); + config["thinking"] = json!("high"); + std::fs::write(config_path, serde_yaml_ng::to_string(&config).unwrap()).unwrap(); + let host = world.start().await; + // Off is `none` for OpenAI, and disabled thinking without an effort for Anthropic. + let off = if api == APIS[1] { None } else { Some("none") }; + for (thinking, effort) in [ + (None, Some("high")), + (Some("low"), Some("low")), + (Some("off"), off), + (None, Some("high")), + ] { + let mut params = json!({"text":"hello"}); + if let Some(thinking) = thinking { + params["thinking"] = json!(thinking); + } + let caller = host.clone(); + let turn = + tokio::spawn(async move { caller.call("agent.chat", params).await.unwrap() }); + let (body, reply) = server.next().await; + let sent = if api == APIS[0] { + &body["reasoning"]["effort"] + } else if api == APIS[2] { + &body["reasoning_effort"] + } else { + let mode = if thinking == Some("off") { + "disabled" + } else { + "adaptive" + }; + assert_eq!(body["thinking"]["type"], mode, "{body}"); + &body["output_config"]["effort"] + }; + assert_eq!(sent.as_str(), effort, "{api} {thinking:?}"); + reply.send((200, server.answer("done", &[]))).unwrap(); + assert_eq!(turn.await.unwrap(), "done"); + } + let error = host + .call("agent.chat", json!({"text":"hello","thinking":"hihg"})) + .await + .unwrap_err(); + assert_eq!(error["code"], -32602, "{error}"); + host.stop().await; + } +} + /// Separate assistant messages retain a separator in the final answer. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn final_texts_have_a_separator() {