refactor(providers): always-on prompt caching via a directly-tested helper

- Remove the PROMPT_CACHE_DISABLED kill switch — prompt caching is always on.
- Extract the Anthropic tagging into applyAnthropicPromptCache(payload, tools,
  systemPrompt) in anthropic/utils.ts: one place that gates and mutates the
  system block + last tool, replacing the two inline blocks in core.ts.
- Add direct unit tests for the helper (system→cached block, last-tool tagged,
  relocated/blanked-system still tags tools, below-threshold and tool-less cases
  untouched) so the actual payload mutation is verified, not just the gate.

No behavior change to outputs; verified on vitest 4.1.8 (CI's version).

Author: waleedlatif1

apps/sim/providers/anthropic/core.ts

@@ -6,6 +6,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
 import type { BlockTokens, IterationToolCall, StreamingExecution } from '@/executor/types'
 import { MAX_TOOL_ITERATIONS } from '@/providers'
 import {
+  applyAnthropicPromptCache,
   checkForForcedToolUsage,
   createReadableStreamFromAnthropicStream,
 } from '@/providers/anthropic/utils'
@@ -16,7 +17,6 @@ import {
   supportsNativeStructuredOutputs,
   supportsTemperature,
 } from '@/providers/models'
-import { shouldCacheStaticPrefix } from '@/providers/prompt-cache'
 import { createStreamingExecution } from '@/providers/streaming-execution'
 import { adaptAnthropicToolSchema } from '@/providers/tool-schema-adapter'
 import { enrichLastModelSegment } from '@/providers/trace-enrichment'
@@ -327,21 +327,9 @@ export async function executeAnthropicProviderRequest(
 
   // Prompt caching: mark the static prefix (system + tools) with an ephemeral
   // cache breakpoint so repeated calls (agent tool-loops, multi-turn) reuse it.
-  // Must run after the structured-output block above, which assumes `system` is
-  // still a string. Tools are tagged at their assignment below.
-  // Gate on the original request system prompt, not payload.system: when there
-  // are no context/chat messages the system text is relocated into a user
-  // message and payload.system is blanked (see above), but the prefix is still
-  // worth caching (the tools, at least).
-  const cacheStaticPrefix = shouldCacheStaticPrefix({
-    systemPrompt: request.systemPrompt,
-    hasTools: !!anthropicTools?.length,
-    toolsApproxChars: anthropicTools ? JSON.stringify(anthropicTools).length : 0,
-  })
-
-  if (cacheStaticPrefix && typeof payload.system === 'string' && payload.system.length > 0) {
-    payload.system = [{ type: 'text', text: payload.system, cache_control: { type: 'ephemeral' } }]
-  }
+  // Runs after the structured-output block above, which assumes `system` is still
+  // a string. Mutates payload.system and the last tool in place.
+  applyAnthropicPromptCache(payload, anthropicTools, request.systemPrompt)
 
   // Add extended thinking configuration if supported and requested
   // The 'none' sentinel means "disable thinking" — skip configuration entirely.
@@ -385,13 +373,6 @@ export async function executeAnthropicProviderRequest(
   }
 
   if (anthropicTools?.length) {
-    if (cacheStaticPrefix) {
-      const lastIndex = anthropicTools.length - 1
-      anthropicTools[lastIndex] = {
-        ...anthropicTools[lastIndex],
-        cache_control: { type: 'ephemeral' },
-      }
-    }
     payload.tools = anthropicTools
     // Per Anthropic docs: forced tool_choice (type: "tool" or "any") is incompatible with
     // thinking. Only auto and none are supported when thinking is enabled.

apps/sim/providers/anthropic/utils.test.ts

@@ -0,0 +1,76 @@
+/**
+ * @vitest-environment node
+ */
+import type { TextBlockParam, Tool } from '@anthropic-ai/sdk/resources'
+import { describe, expect, it } from 'vitest'
+import { applyAnthropicPromptCache } from '@/providers/anthropic/utils'
+
+const LARGE = 'x'.repeat(8_000) // ~2,000 est. tokens, above the 1,024 gate
+const SMALL = 'x'.repeat(400) // ~100 est. tokens, below the gate
+
+const tool = (name: string): Tool => ({
+  name,
+  description: 'does a thing',
+  input_schema: { type: 'object', properties: {} },
+})
+
+describe('applyAnthropicPromptCache', () => {
+  it('converts a large system prompt to a cached text block and tags the last tool', () => {
+    const payload: { system?: string | TextBlockParam[] } = { system: LARGE }
+    const tools = [tool('a'), tool('b')]
+
+    applyAnthropicPromptCache(payload, tools, LARGE)
+
+    expect(Array.isArray(payload.system)).toBe(true)
+    const blocks = payload.system as TextBlockParam[]
+    expect(blocks).toHaveLength(1)
+    expect(blocks[0]).toMatchObject({
+      type: 'text',
+      text: LARGE,
+      cache_control: { type: 'ephemeral' },
+    })
+    // Only the LAST tool carries the breakpoint; earlier tools are untouched.
+    expect(tools[0].cache_control).toBeUndefined()
+    expect(tools[1].cache_control).toEqual({ type: 'ephemeral' })
+  })
+
+  it('tags the system block when the system alone is large and there are no tools', () => {
+    const payload: { system?: string | TextBlockParam[] } = { system: LARGE }
+
+    applyAnthropicPromptCache(payload, undefined, LARGE)
+
+    const blocks = payload.system as TextBlockParam[]
+    expect(blocks[0].cache_control).toEqual({ type: 'ephemeral' })
+  })
+
+  it('tags the tools even when payload.system was relocated/blanked (gate uses the request prompt)', () => {
+    // No-messages path: the provider moves the system text into a user message
+    // and blanks payload.system, but the original prompt is large, so the tools
+    // prefix is still worth caching.
+    const payload: { system?: string | TextBlockParam[] } = { system: '' }
+    const tools = [tool('a')]
+
+    applyAnthropicPromptCache(payload, tools, LARGE)
+
+    expect(payload.system).toBe('') // empty system is never converted
+    expect(tools[0].cache_control).toEqual({ type: 'ephemeral' })
+  })
+
+  it('leaves a small, tool-less prefix untouched (no write surcharge on one-shot calls)', () => {
+    const payload: { system?: string | TextBlockParam[] } = { system: SMALL }
+
+    applyAnthropicPromptCache(payload, undefined, SMALL)
+
+    expect(payload.system).toBe(SMALL)
+  })
+
+  it('does nothing when the combined prefix is below the threshold', () => {
+    const payload: { system?: string | TextBlockParam[] } = { system: SMALL }
+    const tools = [tool('a')]
+
+    applyAnthropicPromptCache(payload, tools, SMALL)
+
+    expect(payload.system).toBe(SMALL)
+    expect(tools[0].cache_control).toBeUndefined()
+  })
+})

apps/sim/providers/anthropic/utils.ts

@@ -2,14 +2,57 @@ import type {
   RawMessageDeltaEvent,
   RawMessageStartEvent,
   RawMessageStreamEvent,
+  TextBlockParam,
+  Tool,
   Usage,
 } from '@anthropic-ai/sdk/resources'
 import { createLogger } from '@sim/logger'
 import { randomFloat } from '@sim/utils/random'
+import { shouldCacheStaticPrefix } from '@/providers/prompt-cache'
 import { trackForcedToolUsage } from '@/providers/utils'
 
 const logger = createLogger('AnthropicUtils')
 
+/** Mutable view of the parts of the Anthropic payload that carry cache breakpoints. */
+interface AnthropicCacheablePayload {
+  system?: string | Array<TextBlockParam>
+}
+
+/**
+ * Marks the static request prefix (system prompt + tools) with an ephemeral
+ * cache breakpoint when {@link shouldCacheStaticPrefix} deems it worthwhile, so
+ * repeated calls reuse the cached prefix. Mutates `payload.system` (string → a
+ * single cached text block) and the last entry of `tools` in place.
+ *
+ * `systemPrompt` is the ORIGINAL request system prompt, used only for the
+ * worthiness gate: on the no-messages path the provider relocates the system
+ * text into a user message and blanks `payload.system`, but the tools prefix is
+ * still worth caching there.
+ */
+export function applyAnthropicPromptCache(
+  payload: AnthropicCacheablePayload,
+  tools: Tool[] | undefined,
+  systemPrompt: string | null | undefined
+): void {
+  const shouldCache = shouldCacheStaticPrefix({
+    systemPrompt,
+    hasTools: !!tools?.length,
+    toolsApproxChars: tools ? JSON.stringify(tools).length : 0,
+  })
+  if (!shouldCache) {
+    return
+  }
+
+  if (typeof payload.system === 'string' && payload.system.length > 0) {
+    payload.system = [{ type: 'text', text: payload.system, cache_control: { type: 'ephemeral' } }]
+  }
+
+  if (tools?.length) {
+    const lastIndex = tools.length - 1
+    tools[lastIndex] = { ...tools[lastIndex], cache_control: { type: 'ephemeral' } }
+  }
+}
+
 export interface AnthropicStreamUsage {
   input_tokens: number
   output_tokens: number

apps/sim/providers/prompt-cache.test.ts

@@ -1,24 +1,13 @@
 /**
  * @vitest-environment node
  */
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { describe, expect, it } from 'vitest'
 import { shouldCacheStaticPrefix } from '@/providers/prompt-cache'
 
 const LARGE = 'x'.repeat(8_000) // ~2,000 est. tokens, above the 1,024 gate
 const SMALL = 'x'.repeat(400) // ~100 est. tokens, below the gate
 
 describe('shouldCacheStaticPrefix', () => {
-  // vi.stubEnv cleanly sets/restores the kill switch without `delete` (which
-  // biome rewrites) or assigning `undefined` (which coerces to the string
-  // "undefined" and leaks to other tests in the worker).
-  beforeEach(() => {
-    vi.stubEnv('PROMPT_CACHE_DISABLED', '')
-  })
-
-  afterEach(() => {
-    vi.unstubAllEnvs()
-  })
-
   it('caches a large system prompt that has tools (agent loop)', () => {
     expect(shouldCacheStaticPrefix({ systemPrompt: LARGE, hasTools: true })).toBe(true)
   })
@@ -51,9 +40,4 @@ describe('shouldCacheStaticPrefix', () => {
     ).toBe(false)
     expect(shouldCacheStaticPrefix({ systemPrompt: null, hasTools: true })).toBe(false)
   })
-
-  it('is disabled by the PROMPT_CACHE_DISABLED kill switch', () => {
-    vi.stubEnv('PROMPT_CACHE_DISABLED', 'true')
-    expect(shouldCacheStaticPrefix({ systemPrompt: LARGE, hasTools: true })).toBe(false)
-  })
 })

apps/sim/providers/prompt-cache.ts

@@ -1,5 +1,3 @@
-import { getEnv, isTruthy } from '@/lib/core/config/env'
-
 /**
  * Minimum estimated static-prefix size (system + tool definitions) before it is
  * worth marking a prompt-cache breakpoint. This is a rough lower bound across
@@ -23,18 +21,12 @@ function estimateTokens(text: string): number {
  * a large system prompt is typically reused across runs within the cache TTL.
  * A small, tool-less prompt is intentionally skipped so a one-shot call never
  * pays the cache-write surcharge for a prefix that is never read back.
- *
- * Set `PROMPT_CACHE_DISABLED=true` to turn this off globally (kill switch).
  */
 export function shouldCacheStaticPrefix(params: {
   systemPrompt: string | null | undefined
   hasTools: boolean
   toolsApproxChars?: number
 }): boolean {
-  if (isTruthy(getEnv('PROMPT_CACHE_DISABLED'))) {
-    return false
-  }
-
   const system = params.systemPrompt ?? ''
   if (!system) {
     return false