Skip to content

Commit 3d63924

Browse files
ericdalloericdallo
committed
Add integration tests for remote mcp
1 parent 6337eb8 commit 3d63924

5 files changed

Lines changed: 419 additions & 5 deletions

File tree

‎AGENTS.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@ ECA Agent Guide (AGENTS.md)
66
- Native image (GraalVM, `GRAALVM_HOME` set): `bb native-cli`
77
- Test (Kaocha via `:test` alias):
88
- Run all unit tests: `bb test` (same as `clojure -M:test`)
9-
- Run a single test namespace: `clojure -M:test --focus eca.main-test`
10-
- Run a single test var: `clojure -M:test --focus eca.main-test/parse-opts-test`
11-
- Integration tests (requires built `./eca` or `eca.exe`): `bb integration-test`
9+
- Run a single unit test namespace: `clojure -M:test --focus eca.main-test`
10+
- Run a single unit test var: `clojure -M:test --focus eca.main-test/parse-opts-test`
11+
- Run all integration tests (requires built `./eca` or `eca.exe`): `bb integration-test`
12+
- Run a single integration test: `bb integration-test --dev --ns integration.chat.mcp-remote-test`
1213
- Lint/format:
1314
- Lint: `clj-kondo --lint src test dev integration-test`
1415
- Formatting not enforced; follow idiomatic Clojure (`cljfmt` optional).

‎integration-test/entrypoint.clj‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
[clojure.test :as t]
77
[integration.eca :as eca]
88
[integration.chat.hooks-test]
9-
[llm-mock.server :as llm-mock.server]))
9+
[llm-mock.server :as llm-mock.server]
10+
[mcp-mock.server :as mcp-mock.server]))
1011

1112
(def namespaces
1213
'[integration.initialize-test
@@ -18,6 +19,7 @@
1819
integration.chat.custom-provider-test
1920
integration.chat.hooks-test
2021
integration.chat.commands-test
22+
integration.chat.mcp-remote-test
2123
integration.rewrite.openai-test])
2224

2325
(defn timeout [timeout-ms callback]
@@ -125,6 +127,7 @@ LogLevel Info" (:host proxy-conf) (:port proxy-conf) (:user proxy-conf) (:pass p
125127
(println "Preparing mcp-server-sample")
126128
(shell {:out nil :dir "integration-test/mcp-server-sample"} "clojure -Stree")
127129
(llm-mock.server/start!)
130+
(mcp-mock.server/start!)
128131

129132
(let [timeout-minutes (if (re-find #"(?i)win|mac" (System/getProperty "os.name"))
130133
10 ;; win and mac ci runs take longer
@@ -134,6 +137,7 @@ LogLevel Info" (:host proxy-conf) (:port proxy-conf) (:user proxy-conf) (:pass p
134137
(apply t/run-tests nses)))]
135138

136139
(llm-mock.server/stop!)
140+
(mcp-mock.server/stop!)
137141

138142
(when (= test-results :timed-out)
139143
(println)

‎integration-test/integration/chat/mcp_remote_test.clj‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
(ns integration.chat.mcp-remote-test
2+
(:require
3+
[clojure.string :as string]
4+
[clojure.test :refer [deftest is testing]]
5+
[integration.eca :as eca]
6+
[integration.fixture :as fixture]
7+
[integration.helper :refer [match-content]]
8+
[llm-mock.mocks :as llm.mocks]
9+
[matcher-combinators.matchers :as m]
10+
[matcher-combinators.test :refer [match?]]
11+
[mcp-mock.server :as mcp-mock]))
12+
13+
(eca/clean-after-test)
14+
15+
(def ^:private mcp-server-config
16+
{:mcpServers {"test-mcp" {:url (str "http://localhost:" mcp-mock/port "/mcp")}}})
17+
18+
(defn ^:private init-with-mcp-remote! []
19+
(eca/start-process!)
20+
(mcp-mock/reset-requests!)
21+
(eca/request! (fixture/initialize-request
22+
{:initializationOptions
23+
(merge fixture/default-init-options mcp-server-config)}))
24+
(eca/notify! (fixture/initialized-notification)))
25+
26+
(deftest mcp-remote-server-connects
27+
(init-with-mcp-remote!)
28+
29+
(testing "Native tools loaded"
30+
(is (match? {:type "native"}
31+
(eca/client-awaits-server-notification :tool/serverUpdated))))
32+
33+
(testing "MCP remote server starting"
34+
(is (match? {:type "mcp"
35+
:name "testMcp"}
36+
(eca/client-awaits-server-notification :tool/serverUpdated))))
37+
38+
(testing "MCP remote server running with tools"
39+
(is (match? {:type "mcp"
40+
:name "testMcp"
41+
:tools (m/embeds [{:name "echo"} {:name "add"}])}
42+
(eca/client-awaits-server-notification :tool/serverUpdated))))
43+
44+
(testing "MCP mock received initialize request"
45+
(let [init-reqs (mcp-mock/get-requests-by-method "initialize")]
46+
(is (= 1 (count init-reqs)))
47+
(is (match? {:method "initialize"
48+
:params {:capabilities map?
49+
:clientInfo map?}}
50+
(first init-reqs)))))
51+
52+
(testing "MCP mock received tools/list request"
53+
(let [tools-reqs (mcp-mock/get-requests-by-method "tools/list")]
54+
(is (= 1 (count tools-reqs))))))
55+
56+
(deftest mcp-remote-tool-call-in-chat
57+
(init-with-mcp-remote!)
58+
59+
;; Wait for MCP server to be ready
60+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; native
61+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp starting
62+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp running
63+
64+
(testing "LLM invokes an MCP tool and ECA processes it"
65+
(mcp-mock/reset-requests!)
66+
(llm.mocks/set-case! :mcp-tool-call-0)
67+
68+
(let [resp (eca/request! (fixture/chat-prompt-request
69+
{:model "anthropic/claude-sonnet-4-6"
70+
:message "Call the echo tool"}))
71+
chat-id (:chatId resp)]
72+
73+
(is (match? {:chatId string?
74+
:model "anthropic/claude-sonnet-4-6"
75+
:status "prompting"}
76+
resp))
77+
78+
;; User message
79+
(match-content chat-id "user" {:type "text" :text "Call the echo tool\n"})
80+
81+
;; Title notification
82+
(match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
83+
84+
;; Progress: waiting model
85+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
86+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
87+
88+
;; Assistant text before tool use
89+
(match-content chat-id "assistant" {:type "text" :text "I will call the echo tool"})
90+
91+
;; Tool call prepare/run
92+
(match-content chat-id "assistant" {:type "toolCallPrepare"
93+
:origin "mcp"
94+
:name "echo"
95+
:id "mcp-tool-1"})
96+
(match-content chat-id "assistant" {:type "toolCallPrepare"
97+
:origin "mcp"
98+
:name "echo"
99+
:id "mcp-tool-1"})
100+
101+
;; Usage from first LLM turn
102+
(match-content chat-id "system" {:type "usage"})
103+
104+
(match-content chat-id "assistant" {:type "toolCallRun"
105+
:origin "mcp"
106+
:name "echo"
107+
:id "mcp-tool-1"
108+
:arguments {:message "hello from mcp"}
109+
:manualApproval false})
110+
(match-content chat-id "assistant" {:type "toolCallRunning"
111+
:origin "mcp"
112+
:name "echo"
113+
:id "mcp-tool-1"
114+
:arguments {:message "hello from mcp"}})
115+
(match-content chat-id "system" {:type "progress" :state "running" :text "Calling tool"})
116+
117+
;; Tool called result — echo returns the same message
118+
(match-content chat-id "assistant" {:type "toolCalled"
119+
:origin "mcp"
120+
:name "echo"
121+
:id "mcp-tool-1"
122+
:arguments {:message "hello from mcp"}
123+
:error nil
124+
:outputs [{:type "text" :text "hello from mcp"}]})
125+
126+
;; Second LLM turn: final response after tool result
127+
(match-content chat-id "assistant" {:type "text" :text "The echo tool returned: hello from mcp"})
128+
(match-content chat-id "system" {:type "usage"})
129+
(match-content chat-id "system" {:type "progress" :state "finished"})
130+
131+
(testing "MCP mock received tools/call request with correct arguments"
132+
(let [call-reqs (mcp-mock/get-requests-by-method "tools/call")]
133+
(is (= 1 (count call-reqs)))
134+
(is (match? {:method "tools/call"
135+
:params {:name "echo"
136+
:arguments {:message "hello from mcp"}}}
137+
(first call-reqs)))))
138+
139+
(testing "LLM received tool result in second request"
140+
(let [req-body (llm.mocks/get-req-body :mcp-tool-call-0)]
141+
(is (match?
142+
{:messages (m/embeds
143+
[{:role "user"
144+
:content [{:type "tool_result"
145+
:tool_use_id "mcp-tool-1"
146+
:content "hello from mcp\n"}]}])
147+
:tools (m/embeds [{:name "testMcp__echo"}
148+
{:name "testMcp__add"}])}
149+
req-body)))))))
150+
151+
(deftest mcp-remote-instructions-in-prompt
152+
(init-with-mcp-remote!)
153+
154+
;; Wait for MCP server to be ready
155+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; native
156+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp starting
157+
(eca/client-awaits-server-notification :tool/serverUpdated) ;; mcp running
158+
159+
(testing "MCP server instructions appear in the system prompt sent to LLM"
160+
(llm.mocks/set-case! :simple-text-0)
161+
(let [resp (eca/request! (fixture/chat-prompt-request
162+
{:model "anthropic/claude-sonnet-4-6"
163+
:message "hello"}))
164+
chat-id (:chatId resp)]
165+
166+
;; Consume notifications so the chat completes
167+
(match-content chat-id "user" {:type "text"})
168+
(match-content chat-id "system" {:type "metadata"})
169+
(match-content chat-id "system" {:type "progress" :state "running"})
170+
(match-content chat-id "system" {:type "progress" :state "running"})
171+
(match-content chat-id "assistant" {:type "text"})
172+
(match-content chat-id "assistant" {:type "text"})
173+
(match-content chat-id "system" {:type "usage"})
174+
(match-content chat-id "system" {:type "progress" :state "finished"})
175+
176+
(let [req-body (llm.mocks/get-req-body :simple-text-0)
177+
system-text (->> (:system req-body)
178+
(map :text)
179+
(string/join "\n"))]
180+
(is (string/includes? system-text "mcp-server-instruction name=\"testMcp\"")
181+
"System prompt should contain MCP server instruction block")
182+
(is (string/includes? system-text "This is a test MCP server for integration testing.")
183+
"System prompt should contain the MCP server's instructions text")))))

‎integration-test/llm_mock/anthropic.clj‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,49 @@
205205
:output_tokens 30}})
206206
(hk/close ch)))))
207207

208+
(defn ^:private mcp-tool-call-0 [ch body]
209+
(let [second-stage? (some (fn [{:keys [content]}]
210+
(some #(= "tool_result" (:type %)) content))
211+
(:messages body))]
212+
(if-not second-stage?
213+
(do
214+
;; Text before tool use
215+
(sse-send! ch "content_block_delta"
216+
{:type "content_block_delta"
217+
:index 0
218+
:delta {:type "text_delta" :text "I will call the echo tool"}})
219+
;; Tool use block start
220+
(sse-send! ch "content_block_start"
221+
{:type "content_block_start"
222+
:index 1
223+
:content_block {:type "tool_use"
224+
:id "mcp-tool-1"
225+
:name "testMcp__echo"}})
226+
;; Stream JSON args
227+
(sse-send! ch "content_block_delta"
228+
{:type "content_block_delta"
229+
:index 1
230+
:delta {:type "input_json_delta"
231+
:partial_json "{\"message\":\"hello from mcp\"}"}})
232+
(sse-send! ch "message_delta"
233+
{:type "message_delta"
234+
:delta {:stop_reason "tool_use"}
235+
:usage {:input_tokens 10
236+
:output_tokens 20}})
237+
(hk/close ch))
238+
;; Second stage after tool results
239+
(do
240+
(sse-send! ch "content_block_delta"
241+
{:type "content_block_delta"
242+
:index 0
243+
:delta {:type "text_delta" :text "The echo tool returned: hello from mcp"}})
244+
(sse-send! ch "message_delta"
245+
{:type "message_delta"
246+
:delta {:stop_reason "end_turn"}
247+
:usage {:input_tokens 15
248+
:output_tokens 10}})
249+
(hk/close ch)))))
250+
208251
(defn ^:private chat-title-text-0 [ch]
209252
(hk/send! ch
210253
(json/generate-string
@@ -233,4 +276,5 @@
233276
:simple-text-2 (simple-text-2 ch)
234277
:reasoning-0 (reasoning-0 ch)
235278
:reasoning-1 (reasoning-1 ch)
236-
:tool-calling-0 (tool-calling-0 ch body)))))})))
279+
:tool-calling-0 (tool-calling-0 ch body)
280+
:mcp-tool-call-0 (mcp-tool-call-0 ch body)))))})))

0 commit comments

Comments
 (0)