dev-mcp: add view_image tool

crates/sprout-agent/src/agent.rs

@@ -11,6 +11,7 @@ use crate::mcp::McpRegistry;
 
 use crate::types::{
     AgentError, ContentBlock, HistoryItem, ProviderStop, StopReason, ToolCall, ToolResult,
+    ToolResultContent,
 };
 use crate::wire::{self, WireSender};
 
@@ -214,13 +215,17 @@ impl RunCtx<'_> {
         for (i, call) in calls.iter().enumerate() {
             let mut result = results[i].take().unwrap_or_else(|| ToolResult {
                 provider_id: call.provider_id.clone(),
-                text: "internal error: missing result".into(),
+                content: vec![ToolResultContent::Text(
+                    "internal error: missing result".into(),
+                )],
                 is_error: true,
             });
             // On tool error: append a reflection prompt so the LLM
             // diagnoses the failure before blindly retrying.
             if result.is_error {
-                result.text.push_str(ERROR_REFLECTION_SUFFIX);
+                result
+                    .content
+                    .push(ToolResultContent::Text(ERROR_REFLECTION_SUFFIX.to_string()));
             }
             self.history.push(HistoryItem::ToolResult(result));
         }
@@ -445,7 +450,7 @@ async fn emit_completed(wire: &WireSender, sid: &str, call: &ToolCall, result: &
                 "sessionUpdate": "tool_call_update",
                 "toolCallId": call.provider_id,
                 "status": "completed",
-                "content": [{ "type": "content", "content": { "type": "text", "text": result.text } }],
+                "content": [{ "type": "content", "content": { "type": "text", "text": result.text() } }],
                 "rawOutput": { "isError": result.is_error },
             }),
         ),
@@ -537,7 +542,9 @@ pub(crate) fn push_hook_outputs_as_tool_results(
         });
         history.push(HistoryItem::ToolResult(ToolResult {
             provider_id,
-            text: format_hook_output_body(hook, server, text),
+            content: vec![ToolResultContent::Text(format_hook_output_body(
+                hook, server, text,
+            ))],
             is_error: false,
         }));
     }
@@ -555,7 +562,7 @@ fn unique_nonce() -> u64 {
 fn synthetic_tool_result(call: &ToolCall, msg: String) -> ToolResult {
     ToolResult {
         provider_id: call.provider_id.clone(),
-        text: msg,
+        content: vec![ToolResultContent::Text(msg)],
         is_error: true,
     }
 }

crates/sprout-agent/src/config.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 pub const PROTOCOL_VERSION: u32 = 1;
 
 pub const MAX_PROMPT_BYTES: usize = 1024 * 1024;
-pub const MAX_TOOL_RESULT_BYTES: usize = 256 * 1024;
+pub const MAX_TOOL_RESULT_BYTES: usize = 8 * 1024 * 1024;
 pub const MAX_TOOL_CALLS_PER_TURN: usize = 64;
 
 /// Leaves headroom for the summary call.
@@ -105,7 +105,7 @@ impl Config {
             mcp_restart_max_ms: parse_env("SPROUT_AGENT_MCP_RESTART_MAX_MS", 30_000u64)?,
             max_sessions: parse_env("SPROUT_AGENT_MAX_SESSIONS", usize::MAX)?,
             max_line_bytes: parse_env("SPROUT_AGENT_MAX_LINE_BYTES", 4 * 1024 * 1024)?,
-            max_history_bytes: parse_env("SPROUT_AGENT_MAX_HISTORY_BYTES", 1024 * 1024)?,
+            max_history_bytes: parse_env("SPROUT_AGENT_MAX_HISTORY_BYTES", 16 * 1024 * 1024)?,
             max_handoffs: parse_env("SPROUT_AGENT_MAX_HANDOFFS", 5)?,
             max_parallel_tools: parse_env("SPROUT_AGENT_MAX_PARALLEL_TOOLS", 8usize)?,
             hook_timeout: Duration::from_millis(parse_env(

crates/sprout-agent/src/handoff.rs

@@ -180,7 +180,7 @@ fn push_history_snippet(out: &mut String, item: &HistoryItem) {
         }
         HistoryItem::ToolResult(r) => {
             out.push_str(if r.is_error { "[tool_err] " } else { "[tool] " });
-            out.push_str(&clamp_for_snippet(&r.text));
+            out.push_str(&clamp_for_snippet(&r.text()));
             out.push('\n');
         }
     }

crates/sprout-agent/src/llm.rs

@@ -2,7 +2,9 @@ use reqwest::Client;
 use serde_json::{json, Value};
 
 use crate::config::{Config, Provider};
-use crate::types::{AgentError, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef};
+use crate::types::{
+    AgentError, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef, ToolResultContent,
+};
 
 const MAX_LLM_RESPONSE_BYTES: usize = 16 * 1024 * 1024;
 const MAX_LLM_ERROR_BODY_BYTES: usize = 4 * 1024;
@@ -127,7 +129,7 @@ fn anthropic_body(cfg: &Config, history: &[HistoryItem], tools: &[ToolDef]) -> V
             }
             HistoryItem::ToolResult(r) => pending.push(json!({
                 "type": "tool_result", "tool_use_id": r.provider_id,
-                "content": [{ "type": "text", "text": r.text }], "is_error": r.is_error })),
+                "content": anthropic_tool_result_content(&r.content), "is_error": r.is_error })),
         }
     }
     flush(&mut messages, &mut pending);
@@ -146,6 +148,19 @@ fn anthropic_body(cfg: &Config, history: &[HistoryItem], tools: &[ToolDef]) -> V
     body
 }
 
+fn anthropic_tool_result_content(content: &[ToolResultContent]) -> Vec<Value> {
+    content
+        .iter()
+        .map(|c| match c {
+            ToolResultContent::Text(text) => json!({ "type": "text", "text": text }),
+            ToolResultContent::Image { data, mime_type } => json!({
+                "type": "image",
+                "source": { "type": "base64", "media_type": mime_type, "data": data },
+            }),
+        })
+        .collect()
+}
+
 fn openai_body(cfg: &Config, history: &[HistoryItem], tools: &[ToolDef]) -> Value {
     let mut messages: Vec<Value> = vec![json!({ "role": "system", "content": cfg.system_prompt })];
     for item in history {
@@ -170,8 +185,15 @@ fn openai_body(cfg: &Config, history: &[HistoryItem], tools: &[ToolDef]) -> Valu
                 }
                 messages.push(Value::Object(msg));
             }
-            HistoryItem::ToolResult(r) => messages.push(json!({
-                "role": "tool", "tool_call_id": r.provider_id, "content": r.text })),
+            HistoryItem::ToolResult(r) => {
+                messages.push(json!({
+                    "role": "tool", "tool_call_id": r.provider_id,
+                    "content": openai_tool_text_content(&r.content) }));
+                let image_content = openai_image_user_content(&r.content);
+                if !image_content.is_empty() {
+                    messages.push(json!({ "role": "user", "content": image_content }));
+                }
+            }
         }
     }
     let tools_json: Vec<Value> = tools
@@ -192,6 +214,33 @@ fn openai_body(cfg: &Config, history: &[HistoryItem], tools: &[ToolDef]) -> Valu
     body
 }
 
+fn openai_tool_text_content(content: &[ToolResultContent]) -> String {
+    let mut parts = Vec::new();
+    for c in content {
+        match c {
+            ToolResultContent::Text(text) => parts.push(text.clone()),
+            ToolResultContent::Image { data, mime_type } => parts.push(format!(
+                "This tool result included an image ({mime_type}, {} base64 bytes) that is provided in the next user message.",
+                data.len()
+            )),
+        }
+    }
+    parts.join("\n")
+}
+
+fn openai_image_user_content(content: &[ToolResultContent]) -> Vec<Value> {
+    content
+        .iter()
+        .filter_map(|c| match c {
+            ToolResultContent::Image { data, mime_type } => Some(json!({
+                "type": "image_url",
+                "image_url": { "url": format!("data:{mime_type};base64,{data}") },
+            })),
+            ToolResultContent::Text(_) => None,
+        })
+        .collect()
+}
+
 fn map_stop(s: Option<&str>) -> ProviderStop {
     match s {
         Some("end_turn" | "stop") => ProviderStop::EndTurn,
@@ -389,3 +438,90 @@ where
     }
     Err(AgentError::Llm("exhausted retries".into()))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::{Config, HookServers, Provider};
+    use crate::types::{HistoryItem, ToolCall, ToolResult, ToolResultContent};
+    use std::time::Duration;
+
+    fn cfg(provider: Provider) -> Config {
+        Config {
+            provider,
+            system_prompt: "system".into(),
+            max_rounds: 10,
+            max_output_tokens: 1024,
+            llm_timeout: Duration::from_secs(10),
+            tool_timeout: Duration::from_secs(10),
+            mcp_init_timeout: Duration::from_secs(10),
+            mcp_max_restart_attempts: 1,
+            mcp_restart_base_ms: 1,
+            mcp_restart_max_ms: 1,
+            max_sessions: 1,
+            max_line_bytes: 1024 * 1024,
+            max_history_bytes: 16 * 1024 * 1024,
+            max_handoffs: 1,
+            max_parallel_tools: 1,
+            hook_timeout: Duration::from_secs(1),
+            stop_max_rejections: 0,
+            hook_servers: HookServers::None,
+            api_key: "key".into(),
+            model: "model".into(),
+            base_url: "http://example.invalid".into(),
+            anthropic_api_version: "2023-06-01".into(),
+        }
+    }
+
+    fn image_history() -> Vec<HistoryItem> {
+        vec![
+            HistoryItem::User("describe the image".into()),
+            HistoryItem::Assistant {
+                text: String::new(),
+                tool_calls: vec![ToolCall {
+                    provider_id: "toolu_1".into(),
+                    name: "dev__view_image".into(),
+                    arguments: serde_json::json!({"source":"x.png"}),
+                }],
+            },
+            HistoryItem::ToolResult(ToolResult {
+                provider_id: "toolu_1".into(),
+                content: vec![
+                    ToolResultContent::Text("10×10, 70 B (image/png from x.png)".into()),
+                    ToolResultContent::Image {
+                        data: "aW1n".into(),
+                        mime_type: "image/png".into(),
+                    },
+                ],
+                is_error: false,
+            }),
+        ]
+    }
+
+    #[test]
+    fn anthropic_tool_result_preserves_image_block() {
+        let body = anthropic_body(&cfg(Provider::Anthropic), &image_history(), &[]);
+        let content = &body["messages"][2]["content"][0]["content"];
+        assert_eq!(content[0]["type"], "text");
+        assert_eq!(content[1]["type"], "image");
+        assert_eq!(content[1]["source"]["type"], "base64");
+        assert_eq!(content[1]["source"]["media_type"], "image/png");
+        assert_eq!(content[1]["source"]["data"], "aW1n");
+    }
+
+    #[test]
+    fn openai_tool_result_adds_followup_image_user_message() {
+        let body = openai_body(&cfg(Provider::OpenAi), &image_history(), &[]);
+        assert_eq!(body["messages"][3]["role"], "tool");
+        assert!(body["messages"][3]["content"]
+            .as_str()
+            .unwrap()
+            .contains("provided in the next user message"));
+        assert_eq!(body["messages"][4]["role"], "user");
+        assert_eq!(body["messages"][4]["content"][0]["type"], "image_url");
+        assert_eq!(
+            body["messages"][4]["content"][0]["image_url"]["url"],
+            "data:image/png;base64,aW1n"
+        );
+    }
+}