Implement cost optimizations: caching, token-efficient tools, and output truncation

Author: alexylon

src/api/anthropic.rs

@@ -23,6 +23,10 @@ impl AnthropicClient {
                 .map_err(|e| SofosError::Config(format!("Invalid API key format: {}", e)))?,
         );
         headers.insert("anthropic-version", HeaderValue::from_static(API_VERSION));
+        headers.insert(
+            "anthropic-beta",
+            HeaderValue::from_static("token-efficient-tools-2025-02-19"),
+        );
         headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
 
         let client = reqwest::Client::builder()

src/repl/request_builder.rs

@@ -62,16 +62,16 @@ impl<'a> RequestBuilder<'a> {
             reasoning: reasoning_config,
         };
 
-        // For Anthropic, drop tool cache metadata to avoid cache block limits
+        // For Anthropic, enable cache on last tool to mark cache breakpoint
         if matches!(self.client, Anthropic(_)) {
             if let Some(tools) = request.tools.as_mut() {
-                for tool in tools.iter_mut() {
-                    match tool {
-                        crate::api::Tool::Regular { cache_control, .. }
-                        | crate::api::Tool::AnthropicWebSearch { cache_control, .. } => {
-                            *cache_control = None;
+                if let Some(last_tool) = tools.last_mut() {
+                    match last_tool {
+                        Tool::Regular { cache_control, .. }
+                        | Tool::AnthropicWebSearch { cache_control, .. } => {
+                            *cache_control = Some(crate::api::CacheControl::ephemeral(None));
                         }
-                        crate::api::Tool::OpenAIWebSearch { .. } => {}
+                        Tool::OpenAIWebSearch { .. } => {}
                     }
                 }
             }

src/tools/bashexec.rs

@@ -6,6 +6,24 @@ use std::process::Command;
 use std::sync::{Arc, Mutex};
 
 const MAX_OUTPUT_SIZE: usize = 10 * 1024 * 1024; // 10MB limit
+const MAX_TOOL_OUTPUT_TOKENS: usize = 8_000; // ~28KB, prevents excessive context usage
+
+/// Truncate bash output if it exceeds token limit for context efficiency
+fn truncate_for_context(content: &str, max_tokens: usize) -> String {
+    let estimated_tokens = content.len() / 4;
+    if estimated_tokens > max_tokens {
+        let truncate_at = max_tokens * 4;
+        let truncated_content = &content[..truncate_at.min(content.len())];
+        format!(
+            "{}...\n\n[TRUNCATED: Output has ~{} tokens, showing first ~{} tokens. Re-run with output redirection if you need the full output.]",
+            truncated_content,
+            estimated_tokens,
+            max_tokens
+        )
+    } else {
+        content.to_string()
+    }
+}
 
 /// Convert Unix signal number to human-readable name
 #[cfg(unix)]
@@ -157,10 +175,11 @@ impl BashExecutor {
                     }
                 }
             };
-            return Ok(format!(
+            let error_output = format!(
                 "Command failed with {}\nSTDOUT:\n{}\nSTDERR:\n{}",
                 exit_info, stdout, stderr
-            ));
+            );
+            return Ok(truncate_for_context(&error_output, MAX_TOOL_OUTPUT_TOKENS));
         }
 
         let mut result = String::new();
@@ -180,7 +199,7 @@ impl BashExecutor {
             result = "Command executed successfully (no output)".to_string();
         }
 
-        Ok(result)
+        Ok(truncate_for_context(&result, MAX_TOOL_OUTPUT_TOKENS))
     }
 
     fn enforce_read_permissions(

src/tools/filesystem.rs

@@ -4,6 +4,24 @@ use std::fs;
 use std::path::{Path, PathBuf};
 
 const MAX_FILE_SIZE: u64 = 50 * 1024 * 1024; // 50MB limit
+const MAX_TOOL_OUTPUT_TOKENS: usize = 8_000; // ~28KB, prevents excessive context usage
+
+/// Truncate file content if it exceeds token limit for context efficiency
+fn truncate_for_context(content: &str, max_tokens: usize) -> String {
+    let estimated_tokens = content.len() / 4;
+    if estimated_tokens > max_tokens {
+        let truncate_at = max_tokens * 4;
+        let truncated_content = &content[..truncate_at.min(content.len())];
+        format!(
+            "{}...\n\n[TRUNCATED: File has ~{} tokens, showing first ~{} tokens. Use search_code or request specific line ranges if you need more.]",
+            truncated_content,
+            estimated_tokens,
+            max_tokens
+        )
+    } else {
+        content.to_string()
+    }
+}
 
 /// FileSystemTool provides secure file operations sandboxed to a workspace directory
 #[derive(Clone)]
@@ -87,8 +105,10 @@ impl FileSystemTool {
             )));
         }
 
-        fs::read_to_string(&validated_path)
-            .with_context(|| format!("Failed to read file: {}", path))
+        let content = fs::read_to_string(&validated_path)
+            .with_context(|| format!("Failed to read file: {}", path))?;
+
+        Ok(truncate_for_context(&content, MAX_TOOL_OUTPUT_TOKENS))
     }
 
     /// Read a file that may be outside the workspace
@@ -123,7 +143,10 @@ impl FileSystemTool {
             )));
         }
 
-        fs::read_to_string(&canonical).with_context(|| format!("Failed to read file: {}", original))
+        let content = fs::read_to_string(&canonical)
+            .with_context(|| format!("Failed to read file: {}", original))?;
+
+        Ok(truncate_for_context(&content, MAX_TOOL_OUTPUT_TOKENS))
     }
 
     pub fn write_file(&self, path: &str, content: &str) -> Result<()> {

src/ui/mod.rs

@@ -527,6 +527,7 @@ impl UI {
             "claude-opus-4-5" => (5.0, 25.0),
             "claude-haiku-4-5" => (1.0, 5.0),
             "gpt-5.1-codex-max" | "gpt-5.1-codex" | "gpt-5-codex" => (1.25, 10.0),
+            "gpt-5.2" => (1.75, 14.0),
             // Default fallback (use Sonnet 4.5 pricing)
             _ => (3.0, 15.0),
         };