Add support for prompt caching in anthropic

Author: ericdallo

CHANGELOG.md

@@ -6,6 +6,7 @@
 - Add MCP/tool support for ollama models.
 - Improve ollama integration only requiring `ollama serve` to be running.
 - Improve chat history and context passed to all LLM providers.
+- Add support for prompt caching for Anthropic models.
 
 ## 0.1.0
 

docs/capabilities.md

@@ -8,13 +8,15 @@ ECA support built-in tools to avoid user extra installation and configuration, t
 
 Provides access to filesystem under workspace root, listing and reading files and directories.
 
+- ''
+
 ## Supported LLM models and capaibilities
 
-| model     | MCP / tools | thinking/reasioning |
-|-----------|-------------|---------------------|
-| OpenAI    | √           | X                   |
-| Anthropic | √           | X                   |
-| Ollama    | √           | X                   |
+| model     | MCP / tools | thinking/reasioning | prompt caching |
+|-----------|-------------|---------------------|----------------|
+| OpenAI    | √           | X                   | X              |
+| Anthropic | √           | X                   | √              |
+| Ollama    | √           | X                   | X              |
 
 ### OpenAI
 

src/eca/llm_providers/anthropic.clj

@@ -26,7 +26,8 @@
                   :input_schema (:parameters tool))) tools)
     web-search (conj {:type "web_search_20250305"
                       :name "web_search"
-                      :max_uses 10})))
+                      :max_uses 10
+                      :cache_control {:type "ephemeral"}})))
 
 (defn ^:private base-request! [{:keys [rid body api-key content-block* on-error on-response]}]
   (let [api-key (or api-key
@@ -73,7 +74,8 @@
                               :content (llm-util/stringfy-tool-result content)}]}
                   msg))
               past-messages)
-        {:role "user" :content user-prompt}))
+        ;; TODO add cache_control to last non thinking message
+        {:role "user" :content user-prompt :cache_control {:type "ephemeral"}}))
 
 (defn completion!
   [{:keys [model user-prompt temperature context max-tokens
@@ -89,7 +91,7 @@
               ;; TODO support :thinking
               :stream true
               :tools (->tools tools web-search)
-              :system context}
+              :system [{:type "text" :text context :cache_control {:type "ephemeral"}}]}
         on-response-fn
         (fn handle-response [event data content-block*]
           (case event

test/eca/llm_providers/anthropic_test.clj

@@ -7,13 +7,13 @@
 (deftest ->messages-with-history-test
   (testing "no previous history"
     (is (match?
-         [{:role "user" :content "Hey"}]
+         [{:role "user" :content "Hey" :cache_control {:type "ephemeral"}}]
          (#'llm-providers.anthropic/->messages-with-history [] "Hey"))))
   (testing "With basic text history"
     (is (match?
          [{:role "user" :content "Count with me: 1"}
           {:role "assistant" :content "2"}
-          {:role "user" :content "3"}]
+          {:role "user" :content "3" :cache_control {:type "ephemeral"}}]
          (#'llm-providers.anthropic/->messages-with-history
           [{:role "user" :content "Count with me: 1"}
            {:role "assistant" :content "2"}]
@@ -30,7 +30,7 @@
                                    :tool-use-id "call-1"
                                    :content "Allowed directories: /foo/bar\n"}]}
           {:role "assistant" :content "I see /foo/bar"}
-          {:role "user" :content "Thanks"}]
+          {:role "user" :content "Thanks" :cache_control {:type "ephemeral"}}]
          (#'llm-providers.anthropic/->messages-with-history
           [{:role "user" :content "List the files you are allowed"}
            {:role "assistant" :content "Ok!"}