Skip to content

Commit bba276e

Browse files
itkoneneca-agent
itkonen
and
eca-agent
committed
Fix tool-call streaming CPU spike
Reuse the prompt-turn tool list while streaming tool-call arguments instead of recomputing all available tools for every streamed delta. Add a dev benchmark for reproducing the streamed tool-call prepare path. 🤖 Generated with [eca](https://eca.dev) Co-Authored-By: eca-agent <git@eca.dev>
1 parent d647690 commit bba276e

3 files changed

Lines changed: 245 additions & 2 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## Unreleased
44

5+
- Improve CPU usage while streaming tool-call arguments by reusing the prompt's tool list.
6+
57
## 0.133.3
68

79
- Add unit and integration tests covering parent↔subagent end-to-end communication so regressions like the v0.133.1 spawn-agent breakage are caught automatically.

‎dev/eca/perf/stream_tool_calls.clj‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
(ns eca.perf.stream-tool-calls
2+
"Small ad-hoc benchmarks for streamed tool-call argument processing.
3+
4+
Run with:
5+
clojure -M:dev -m eca.perf.stream-tool-calls
6+
7+
These benchmarks avoid live LLM and editor dependencies so the same workload can
8+
be replayed before/after optimizations. Numbers are intentionally simple wall
9+
clock timings; compare relative changes on the same machine/JVM."
10+
(:require
11+
[cheshire.core :as json]
12+
[eca.features.chat.tool-calls :as tc]
13+
[eca.features.tools :as f.tools]
14+
[eca.llm-util :as llm-util]
15+
[eca.messenger :as messenger]
16+
[eca.shared :as shared])
17+
(:import
18+
[java.io BufferedReader StringReader]))
19+
20+
(set! *warn-on-reflection* true)
21+
22+
(def default-chunks 5000)
23+
(def default-chunk-size 128)
24+
25+
(defn- now []
26+
(System/nanoTime))
27+
28+
(defn- elapsed-ms [start-ns]
29+
(/ (double (- (now) start-ns)) 1000000.0))
30+
31+
(defn- fmt-ms [n]
32+
(format "%.2f" (double n)))
33+
34+
(defn- chunk-text [chunk-size]
35+
(apply str (take chunk-size (cycle "abcdefghijklmnopqrstuvwxyz0123456789"))))
36+
37+
(defn- make-db []
38+
{:workspace-folders [{:uri "file:///tmp" :name "tmp"}]
39+
:chats {"perf-chat" {:id "perf-chat"
40+
:status :running
41+
:messages []}}
42+
:tool-servers {}})
43+
44+
(defn- make-config []
45+
{:toolCall {:approval {:byDefault "allow"}
46+
:readFile {:maxLines 2000}}
47+
:disabledTools []})
48+
49+
(defrecord CountingMessenger [state* mode]
50+
messenger/IMessenger
51+
(chat-content-received [_this data]
52+
(let [json-bytes (when (= mode :json)
53+
(count (json/generate-string data)))]
54+
(swap! state*
55+
(fn [state]
56+
(cond-> (-> state
57+
(update :notifications (fnil inc 0))
58+
(update :argument-bytes + (count (or (get-in data [:content :arguments-text]) ""))))
59+
json-bytes (update :json-bytes + json-bytes))))))
60+
(chat-cleared [_this _params])
61+
(chat-status-changed [_this _params])
62+
(chat-deleted [_this _params])
63+
(chat-opened [_this _params])
64+
(rewrite-content-received [_this _data])
65+
(tool-server-updated [_this _params])
66+
(tool-server-removed [_this _params])
67+
(config-updated [_this _params])
68+
(provider-updated [_this _params])
69+
(jobs-updated [_this _params])
70+
(showMessage [_this _msg])
71+
(progress [_this _params])
72+
(editor-diagnostics [_this _uri])
73+
(ask-question [_this _params]))
74+
75+
(defn- make-chat-ctx [db* messenger]
76+
{:chat-id "perf-chat"
77+
:request-id "perf-request"
78+
:db* db*
79+
:messenger messenger})
80+
81+
(defn bench-transition-prepare!
82+
"Benchmark only the tool-call state transition + messenger notification path."
83+
[{:keys [chunks chunk-size messenger-mode]
84+
:or {chunks default-chunks
85+
chunk-size default-chunk-size
86+
messenger-mode :noop}}]
87+
(let [db* (atom (make-db))
88+
state* (atom {:notifications 0
89+
:argument-bytes 0
90+
:json-bytes 0})
91+
m (->CountingMessenger state* messenger-mode)
92+
chat-ctx (make-chat-ctx db* m)
93+
arg-chunk (chunk-text chunk-size)
94+
event-data {:name "write_file"
95+
:server "eca"
96+
:full-name "eca__write_file"
97+
:origin :native
98+
:arguments-text arg-chunk}
99+
start (now)]
100+
(dotimes [_ chunks]
101+
(tc/transition-tool-call! db* chat-ctx "call-1" :tool-prepare event-data))
102+
(assoc @state*
103+
:bench :transition-prepare
104+
:messenger-mode messenger-mode
105+
:chunks chunks
106+
:chunk-size chunk-size
107+
:elapsed-ms (elapsed-ms start))))
108+
109+
(defn bench-on-prepare-like!
110+
"Benchmark the heavier chat.clj-like callback path that recomputes all-tools,
111+
resolves the tool, computes summary, and then transitions for every delta."
112+
[{:keys [chunks chunk-size messenger-mode cache-tools?]
113+
:or {chunks default-chunks
114+
chunk-size default-chunk-size
115+
messenger-mode :noop
116+
cache-tools? false}}]
117+
(let [db* (atom (make-db))
118+
config (make-config)
119+
agent nil
120+
state* (atom {:notifications 0
121+
:argument-bytes 0
122+
:json-bytes 0})
123+
m (->CountingMessenger state* messenger-mode)
124+
chat-ctx (make-chat-ctx db* m)
125+
cached-tools (when cache-tools?
126+
(vec (f.tools/all-tools "perf-chat" agent @db* config)))
127+
arg-chunk (chunk-text chunk-size)
128+
start (now)]
129+
(dotimes [_ chunks]
130+
(let [all-tools (or cached-tools
131+
(vec (f.tools/all-tools "perf-chat" agent @db* config)))
132+
full-name "eca__write_file"
133+
tool (f.tools/resolve-tool full-name all-tools)]
134+
(tc/transition-tool-call!
135+
db* chat-ctx "call-1" :tool-prepare
136+
{:name (or (:name tool) full-name)
137+
:server (:name (:server tool))
138+
:full-name (or (:full-name tool) full-name)
139+
:origin (or (:origin tool) :unknown)
140+
:arguments-text arg-chunk
141+
:summary (f.tools/tool-call-summary all-tools full-name nil config @db*)})))
142+
(assoc @state*
143+
:bench :on-prepare-like
144+
:messenger-mode messenger-mode
145+
:cache-tools? cache-tools?
146+
:chunks chunks
147+
:chunk-size chunk-size
148+
:elapsed-ms (elapsed-ms start))))
149+
150+
(defn bench-all-tools!
151+
[{:keys [iterations]
152+
:or {iterations default-chunks}}]
153+
(let [db (make-db)
154+
config (make-config)
155+
start (now)
156+
last-count* (volatile! nil)]
157+
(dotimes [_ iterations]
158+
(vreset! last-count* (count (f.tools/all-tools "perf-chat" nil db config))))
159+
{:bench :all-tools
160+
:iterations iterations
161+
:tool-count @last-count*
162+
:elapsed-ms (elapsed-ms start)}))
163+
164+
(defn bench-camel-case!
165+
[{:keys [iterations argument-size]
166+
:or {iterations default-chunks
167+
argument-size default-chunk-size}}]
168+
(let [payload {:chat-id "perf-chat"
169+
:role :assistant
170+
:content {:type :toolCallPrepare
171+
:id "call-1"
172+
:name "write_file"
173+
:server "eca"
174+
:origin :native
175+
:arguments-text (chunk-text argument-size)}}
176+
start (now)
177+
last* (volatile! nil)]
178+
(dotimes [_ iterations]
179+
(vreset! last* (shared/map->camel-cased-map payload)))
180+
{:bench :camel-case
181+
:iterations iterations
182+
:argument-size argument-size
183+
:last-keys (keys @last*)
184+
:elapsed-ms (elapsed-ms start)}))
185+
186+
(defn bench-event-data-seq!
187+
[{:keys [chunks chunk-size]
188+
:or {chunks default-chunks
189+
chunk-size default-chunk-size}}]
190+
(let [delta (chunk-text chunk-size)
191+
line (str "data: " (json/generate-string {:type "response.function_call_arguments.delta"
192+
:delta delta}) "\n\n")
193+
body (apply str (repeat chunks line))
194+
rdr (BufferedReader. (StringReader. body))
195+
start (now)
196+
parsed-count (count (llm-util/event-data-seq rdr))]
197+
{:bench :event-data-seq
198+
:chunks chunks
199+
:chunk-size chunk-size
200+
:parsed-count parsed-count
201+
:elapsed-ms (elapsed-ms start)}))
202+
203+
(defn- print-result! [{:keys [elapsed-ms chunks iterations] :as result}]
204+
(let [n (or chunks iterations)
205+
per-op (when (and n (pos? n)) (/ elapsed-ms n))]
206+
(println
207+
(str (name (:bench result))
208+
" "
209+
(pr-str (dissoc result :elapsed-ms))
210+
" elapsed-ms=" (fmt-ms elapsed-ms)
211+
(when per-op
212+
(str " ms/op=" (fmt-ms per-op)))))))
213+
214+
(defn run-suite! [{:keys [chunks chunk-size]
215+
:or {chunks default-chunks
216+
chunk-size default-chunk-size}}]
217+
(println "Streaming tool-call benchmark")
218+
(println (str "chunks=" chunks " chunk-size=" chunk-size))
219+
(println "")
220+
;; Warm up enough to load namespaces/classes and reduce first-call noise.
221+
(bench-transition-prepare! {:chunks 100 :chunk-size chunk-size :messenger-mode :noop})
222+
(bench-on-prepare-like! {:chunks 100 :chunk-size chunk-size :messenger-mode :noop})
223+
(doseq [result [(bench-transition-prepare! {:chunks chunks :chunk-size chunk-size :messenger-mode :noop})
224+
(bench-transition-prepare! {:chunks chunks :chunk-size chunk-size :messenger-mode :json})
225+
(bench-on-prepare-like! {:chunks chunks :chunk-size chunk-size :messenger-mode :noop :cache-tools? false})
226+
(bench-on-prepare-like! {:chunks chunks :chunk-size chunk-size :messenger-mode :noop :cache-tools? true})
227+
(bench-on-prepare-like! {:chunks chunks :chunk-size chunk-size :messenger-mode :json :cache-tools? false})
228+
(bench-on-prepare-like! {:chunks chunks :chunk-size chunk-size :messenger-mode :json :cache-tools? true})
229+
(bench-all-tools! {:iterations chunks})
230+
(bench-camel-case! {:iterations chunks :argument-size chunk-size})
231+
(bench-event-data-seq! {:chunks chunks :chunk-size chunk-size})]]
232+
(print-result! result)))
233+
234+
(defn- parse-long-or [s default]
235+
(if (some? s)
236+
(Long/parseLong s)
237+
default))
238+
239+
(defn -main [& args]
240+
(let [[chunks chunk-size] args]
241+
(run-suite! {:chunks (parse-long-or chunks default-chunks)
242+
:chunk-size (parse-long-or chunk-size default-chunk-size)})))

‎src/eca/features/chat.clj‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -828,8 +828,7 @@
828828
(lifecycle/finish-chat-prompt! :idle chat-ctx)))))
829829
:on-prepare-tool-call (fn [{:keys [id full-name arguments-text]}]
830830
(lifecycle/assert-chat-not-stopped! chat-ctx)
831-
(let [all-tools (f.tools/all-tools chat-id agent @db* config)
832-
tool (f.tools/resolve-tool full-name all-tools)
831+
(let [tool (f.tools/resolve-tool full-name all-tools)
833832
resolved-full-name (or (:full-name tool) full-name)]
834833
(when-not tool
835834
(logger/warn logger-tag "Tool not found for prepare"

0 commit comments

Comments
 (0)