Skip to content

Commit c97ab6a

Browse files
anticomputerCopilot
andcommitted
feat(anthropic_sdk): default-on automatic prompt caching
Adds 'cache_control: {type: ephemeral}' to messages.stream() calls. The API auto-places a cache breakpoint at the longest cacheable prefix (tools + system + accumulated messages) and moves it forward on each turn -- multi-turn agent loops get cache reads on every turn after the first. Default-on because all current Claude models support cache_control and CAPI accepts it (validated end-to-end against claude-mythos-5 via CAPI on 2026-06-12). Callers pointed at proxies that strip / reject cache_control can opt out with 'prompt_caching: false' in model_settings. A string value (e.g. 'prompt_caching: 1h') sets a custom TTL. Local validation against anticomputer/vulnerable-test-app on the same audit pipeline, same model config, only changing prompt_caching: metric | off | on | delta --------------------+-----------+-----------+------------ requests | 60 | 62 | +2 (noise) input tokens fresh | 909,806 | 124 | -99.99% cache read tokens | 0 | 728,079 | new cache write tokens | 0 | 210,261 | new output tokens | 42,300 | 44,933 | similar vulnerabilities | 4 | 5 | +1 est. mythos cost | $11.21 | $5.60 | -50% Same or better audit quality, half the token cost. Real audits with larger system prompts + more tool definitions amortize the cache writes over more reads, so production savings are typically larger than 50%. Tests added: - prompt_caching default-on emits cache_control - prompt_caching=False suppresses cache_control (opt-out) - prompt_caching='1h' includes the ttl field 23 tests pass total, hatch fmt clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 4eea127 commit c97ab6a

2 files changed

Lines changed: 170 additions & 0 deletions

File tree

src/seclab_taskflow_agent/sdk/anthropic_sdk/backend.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,22 @@ async def run_streamed(
202202
create_kwargs["thinking"] = {"type": "adaptive"}
203203
create_kwargs["output_config"] = {"effort": effort}
204204

205+
# Automatic prompt caching: place an ephemeral cache breakpoint at
206+
# the longest cacheable prefix (tools + system + accumulated
207+
# messages). The breakpoint moves forward on each turn, so
208+
# multi-turn agent loops get cache reads on every turn after the
209+
# first -- typically 50%+ cost reduction on token-heavy audits.
210+
# All current Claude models (and the Anthropic-compatible CAPI
211+
# proxy) support cache_control. Default on; explicit opt-out for
212+
# callers pointed at proxies that don't support it.
213+
prompt_caching = handle.model_settings.get("prompt_caching", True)
214+
if prompt_caching:
215+
ttl = prompt_caching if isinstance(prompt_caching, str) else "5m"
216+
cache_block: dict[str, Any] = {"type": "ephemeral"}
217+
if ttl != "5m":
218+
cache_block["ttl"] = ttl
219+
create_kwargs["cache_control"] = cache_block
220+
205221
import anthropic
206222

207223
for turn in range(max_turns):

tests/test_sdk_anthropic_adapter.py

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,3 +227,157 @@ async def _run():
227227

228228
with pytest.raises(BackendBadRequestError, match="invalid reasoning effort"):
229229
asyncio.run(_run())
230+
231+
232+
# -- prompt caching --
233+
234+
235+
def test_prompt_caching_enabled_by_default():
236+
"""All Claude models support cache_control; default to on so callers
237+
get the cost savings without explicit opt-in. Explicit opt-out via
238+
prompt_caching=False remains available for proxies that don't support
239+
cache_control."""
240+
import asyncio
241+
242+
from seclab_taskflow_agent.sdk.anthropic_sdk.backend import _AnthropicHandle
243+
244+
captured = {}
245+
246+
class _FakeStreamCtx:
247+
async def __aenter__(self): return self
248+
async def __aexit__(self, *exc): return False
249+
def __aiter__(self):
250+
async def _gen():
251+
return
252+
yield
253+
return _gen()
254+
async def get_final_message(self):
255+
return type("M", (), {"stop_reason": "end_turn", "content": []})()
256+
257+
class _FakeMessages:
258+
def stream(self, **kwargs):
259+
captured.update(kwargs)
260+
return _FakeStreamCtx()
261+
262+
class _FakeClient:
263+
def __init__(self):
264+
self.messages = _FakeMessages()
265+
266+
handle = _AnthropicHandle(
267+
client=_FakeClient(),
268+
system_prompt="",
269+
model="claude-mythos-5",
270+
max_tokens=100,
271+
tools=[],
272+
mcp_server_map={},
273+
model_settings={},
274+
)
275+
backend = AnthropicSDKBackend()
276+
277+
async def _run():
278+
async for _ in backend.run_streamed(handle, "hi", max_turns=1):
279+
pass
280+
281+
asyncio.run(_run())
282+
assert captured.get("cache_control") == {"type": "ephemeral"}, (
283+
f"expected default cache_control={{type: ephemeral}}, got {captured.get('cache_control')!r}"
284+
)
285+
286+
287+
def test_prompt_caching_explicit_opt_out():
288+
"""prompt_caching=False must suppress cache_control entirely (for
289+
callers pointed at proxies that don't support it)."""
290+
import asyncio
291+
292+
from seclab_taskflow_agent.sdk.anthropic_sdk.backend import _AnthropicHandle
293+
294+
captured = {}
295+
296+
class _FakeStreamCtx:
297+
async def __aenter__(self): return self
298+
async def __aexit__(self, *exc): return False
299+
def __aiter__(self):
300+
async def _gen():
301+
return
302+
yield
303+
return _gen()
304+
async def get_final_message(self):
305+
return type("M", (), {"stop_reason": "end_turn", "content": []})()
306+
307+
class _FakeMessages:
308+
def stream(self, **kwargs):
309+
captured.update(kwargs)
310+
return _FakeStreamCtx()
311+
312+
class _FakeClient:
313+
def __init__(self):
314+
self.messages = _FakeMessages()
315+
316+
handle = _AnthropicHandle(
317+
client=_FakeClient(),
318+
system_prompt="",
319+
model="claude-mythos-5",
320+
max_tokens=100,
321+
tools=[],
322+
mcp_server_map={},
323+
model_settings={"prompt_caching": False},
324+
)
325+
backend = AnthropicSDKBackend()
326+
327+
async def _run():
328+
async for _ in backend.run_streamed(handle, "hi", max_turns=1):
329+
pass
330+
331+
asyncio.run(_run())
332+
assert "cache_control" not in captured, (
333+
f"cache_control should be absent when explicitly opted out, got {captured}"
334+
)
335+
336+
337+
def test_prompt_caching_1h_ttl_passes_ttl_field():
338+
"""When prompt_caching='1h', cache_control must include the 1h ttl."""
339+
import asyncio
340+
341+
from seclab_taskflow_agent.sdk.anthropic_sdk.backend import _AnthropicHandle
342+
343+
captured = {}
344+
345+
class _FakeStreamCtx:
346+
async def __aenter__(self): return self
347+
async def __aexit__(self, *exc): return False
348+
def __aiter__(self):
349+
async def _gen():
350+
return
351+
yield
352+
return _gen()
353+
async def get_final_message(self):
354+
return type("M", (), {"stop_reason": "end_turn", "content": []})()
355+
356+
class _FakeMessages:
357+
def stream(self, **kwargs):
358+
captured.update(kwargs)
359+
return _FakeStreamCtx()
360+
361+
class _FakeClient:
362+
def __init__(self):
363+
self.messages = _FakeMessages()
364+
365+
handle = _AnthropicHandle(
366+
client=_FakeClient(),
367+
system_prompt="",
368+
model="claude-mythos-5",
369+
max_tokens=100,
370+
tools=[],
371+
mcp_server_map={},
372+
model_settings={"prompt_caching": "1h"},
373+
)
374+
backend = AnthropicSDKBackend()
375+
376+
async def _run():
377+
async for _ in backend.run_streamed(handle, "hi", max_turns=1):
378+
pass
379+
380+
asyncio.run(_run())
381+
assert captured.get("cache_control") == {"type": "ephemeral", "ttl": "1h"}, (
382+
f"expected cache_control with 1h ttl, got {captured.get('cache_control')!r}"
383+
)

0 commit comments

Comments
 (0)