fix(cache): defer variant-change flush on providers whose cache ignores request params (#257)

A reasoning-variant flip maps to a thinking-config change. On the Anthropic
family the provider renders that config into the prompt and busts message
cache itself, so draining queued ops rides a natural bust. On implicit-prefix
providers the config is a request parameter outside the cache key, so MC's
flush was the only bust — gratuitous (104 ops drained at 54% usage, 156K
uncached tokens in the report). Gate the flush on a provider predicate;
deferred ops ride the next natural bust. Fixes #257.

Author: ualtinok

packages/plugin/src/hooks/magic-context/hook-handlers.test.ts

@@ -13,7 +13,7 @@ import {
 } from "../../features/magic-context/storage-meta-persisted";
 import { Database } from "../../shared/sqlite";
 import { closeQuietly } from "../../shared/sqlite-helpers";
-import { createEventHook, createToolExecuteAfterHook } from "./hook-handlers";
+import { createChatMessageHook, createEventHook, createToolExecuteAfterHook } from "./hook-handlers";
 
 function createTestDb(): Database {
     const db = new Database(":memory:");
@@ -309,3 +309,189 @@ describe("createEventHook mid-session model switch clears overflow state", () =>
         }
     });
 });
+
+// Variant-change flush must be provider-aware (#257). On providers whose wire
+// renders the thinking config into the prompt (Anthropic family), the
+// provider itself invalidates message blocks on a variant flip, so our flush
+// rides a bust that happens regardless. On implicit-prefix-caching providers
+// (OpenAI-compatible et al.), reasoning_effort/budget is a request parameter
+// outside the cache key, so a variant flip is a full cache HIT and our flush
+// would be the ONLY bust — a gratuitous one that drains queued ops for no
+// provider-side reason. The gate defers the flush on the latter.
+describe("createChatMessageHook variant-change flush is provider-aware", () => {
+    type Sets = {
+        historyRefreshSessions: Set<string>;
+        systemPromptRefreshSessions: Set<string>;
+        pendingMaterializationSessions: Set<string>;
+        lastHeuristicsTurnId: Map<string, string>;
+    };
+
+    function makeHook(sets: Sets, liveModelBySession = new Map<string, { providerID: string; modelID: string }>()) {
+        const db = createTestDb();
+        const hook = createChatMessageHook({
+            db,
+            liveModelBySession,
+            variantBySession: new Map<string, string | undefined>(),
+            agentBySession: new Map<string, string>(),
+            historyRefreshSessions: sets.historyRefreshSessions,
+            systemPromptRefreshSessions: sets.systemPromptRefreshSessions,
+            pendingMaterializationSessions: sets.pendingMaterializationSessions,
+            lastHeuristicsTurnId: sets.lastHeuristicsTurnId,
+        });
+        return { hook, db };
+    }
+
+    function freshSets(): Sets {
+        return {
+            historyRefreshSessions: new Set<string>(),
+            systemPromptRefreshSessions: new Set<string>(),
+            pendingMaterializationSessions: new Set<string>(),
+            lastHeuristicsTurnId: new Map<string, string>(),
+        };
+    }
+
+    // Pins current behavior on the Anthropic family. Deleting the predicate
+    // call (so the gate always takes the TRUE arm) must leave this test green.
+    test("anthropic provider: variant flip signals all three sets + clears lastHeuristicsTurnId", async () => {
+        const sets = freshSets();
+        sets.lastHeuristicsTurnId.set("ses", "turn-1");
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "low", model: { providerID: "anthropic", modelID: "claude" } });
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "anthropic", modelID: "claude" } });
+
+            expect(sets.historyRefreshSessions.has("ses")).toBe(true);
+            expect(sets.systemPromptRefreshSessions.has("ses")).toBe(true);
+            expect(sets.pendingMaterializationSessions.has("ses")).toBe(true);
+            expect(sets.lastHeuristicsTurnId.has("ses")).toBe(false);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("bedrock provider: variant flip signals all three sets (thinking config rendered into prompt)", async () => {
+        const sets = freshSets();
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "low", model: { providerID: "bedrock", modelID: "claude" } });
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "bedrock", modelID: "claude" } });
+
+            expect(sets.historyRefreshSessions.has("ses")).toBe(true);
+            expect(sets.systemPromptRefreshSessions.has("ses")).toBe(true);
+            expect(sets.pendingMaterializationSessions.has("ses")).toBe(true);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("google-vertex-anthropic provider: variant flip signals all three sets", async () => {
+        const sets = freshSets();
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({
+                sessionID: "ses",
+                variant: "low",
+                model: { providerID: "google-vertex-anthropic", modelID: "claude" },
+            });
+            await hook({
+                sessionID: "ses",
+                variant: "high",
+                model: { providerID: "google-vertex-anthropic", modelID: "claude" },
+            });
+
+            expect(sets.historyRefreshSessions.has("ses")).toBe(true);
+            expect(sets.systemPromptRefreshSessions.has("ses")).toBe(true);
+            expect(sets.pendingMaterializationSessions.has("ses")).toBe(true);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    // Mutation-sensitive: this test MUST FAIL if the predicate is hardcoded
+    // true or the gate is removed. We assert the sets are EMPTY (not merely
+    // "not containing extra members") to kill the deleted-effect mutant.
+    test("openai provider: variant flip signals NOTHING and leaves lastHeuristicsTurnId untouched", async () => {
+        const sets = freshSets();
+        sets.lastHeuristicsTurnId.set("ses", "turn-1");
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "low", model: { providerID: "openai", modelID: "gpt-4o" } });
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "openai", modelID: "gpt-4o" } });
+
+            expect(sets.historyRefreshSessions.size).toBe(0);
+            expect(sets.systemPromptRefreshSessions.size).toBe(0);
+            expect(sets.pendingMaterializationSessions.size).toBe(0);
+            expect(sets.lastHeuristicsTurnId.get("ses")).toBe("turn-1");
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("fireworks provider: variant flip signals NOTHING (implicit-prefix cache, request param outside cache key)", async () => {
+        const sets = freshSets();
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "low", model: { providerID: "fireworks", modelID: "fwm" } });
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "fireworks", modelID: "fwm" } });
+
+            expect(sets.historyRefreshSessions.size).toBe(0);
+            expect(sets.systemPromptRefreshSessions.size).toBe(0);
+            expect(sets.pendingMaterializationSessions.size).toBe(0);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    // Unknown provider (no model info on the hook input) takes the
+    // conservative TRUE arm = today's behavior.
+    test("unknown provider (no model info): variant flip takes the TRUE arm (all three sets signaled)", async () => {
+        const sets = freshSets();
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "low" });
+            await hook({ sessionID: "ses", variant: "high" });
+
+            expect(sets.historyRefreshSessions.has("ses")).toBe(true);
+            expect(sets.systemPromptRefreshSessions.has("ses")).toBe(true);
+            expect(sets.pendingMaterializationSessions.has("ses")).toBe(true);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    // The liveModelBySession fallback: when the hook input has no model but a
+    // prior event recorded the provider, the fallback providerID governs the
+    // gate. An OpenAI-compatible provider recorded earlier must still defer.
+    test("liveModelBySession fallback: openai recorded earlier, no model on input → defer (FALSE arm)", async () => {
+        const sets = freshSets();
+        const liveModelBySession = new Map<string, { providerID: string; modelID: string }>([
+            ["ses", { providerID: "openai", modelID: "gpt-4o" }],
+        ]);
+        const { hook, db } = makeHook(sets, liveModelBySession);
+        try {
+            await hook({ sessionID: "ses", variant: "low" });
+            await hook({ sessionID: "ses", variant: "high" });
+
+            expect(sets.historyRefreshSessions.size).toBe(0);
+            expect(sets.systemPromptRefreshSessions.size).toBe(0);
+            expect(sets.pendingMaterializationSessions.size).toBe(0);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("no variant change: no sets signaled regardless of provider", async () => {
+        const sets = freshSets();
+        const { hook, db } = makeHook(sets);
+        try {
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "openai", modelID: "gpt-4o" } });
+            await hook({ sessionID: "ses", variant: "high", model: { providerID: "openai", modelID: "gpt-4o" } });
+
+            expect(sets.historyRefreshSessions.size).toBe(0);
+            expect(sets.systemPromptRefreshSessions.size).toBe(0);
+            expect(sets.pendingMaterializationSessions.size).toBe(0);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+});

packages/plugin/src/hooks/magic-context/hook-handlers.ts

@@ -45,6 +45,7 @@ import {
     resetNoteNudgeCooldownOnly,
 } from "./note-nudger";
 import { readRawSessionMessageById, readRawSessionMessages } from "./read-session-chunk";
+import { variantChangeBustsProviderCache } from "./sentinel";
 import { normalizeTodoStateJson } from "./todo-view";
 
 export type LiveModelBySession = Map<string, { providerID: string; modelID: string }>;
@@ -203,14 +204,49 @@ export function createChatMessageHook(args: {
             input.variant !== undefined &&
             previousVariant !== input.variant
         ) {
-            sessionLog(
-                sessionId,
-                `variant changed (${previousVariant} -> ${input.variant}), triggering flush`,
-            );
-            args.historyRefreshSessions.add(sessionId);
-            args.systemPromptRefreshSessions.add(sessionId);
-            args.pendingMaterializationSessions.add(sessionId);
-            args.lastHeuristicsTurnId.delete(sessionId);
+            // A reasoning-variant change maps to a thinking-config change
+            // (effort / budget_tokens / toggle — see OpenCode's
+            // `reasoningVariants`). Whether that busts the provider's prompt
+            // cache on its own depends on the provider's cache model: the
+            // Anthropic family renders the thinking config into the prompt
+            // (so the provider itself invalidates message blocks on a change
+            // and our queued ops drain on that natural bust), while
+            // OpenAI-compatible providers carry reasoning_effort / budget as
+            // a request parameter outside the cache key, so a variant flip
+            // is a full cache HIT and our flush would be the ONLY bust — a
+            // gratuitous one. See `variantChangeBustsProviderCache` for the
+            // full rationale and the safety asymmetry.
+            //
+            // providerID comes from the hook input (the live request's
+            // model) with `liveModelBySession` as a fallback for sessions
+            // whose first chat.message predates a model-bearing event. When
+            // no provider is known yet we take the conservative TRUE arm
+            // (today's behavior) so we never silently drop a needed drain.
+            const providerID =
+                input.model?.providerID ??
+                args.liveModelBySession.get(sessionId)?.providerID;
+            if (variantChangeBustsProviderCache(providerID)) {
+                sessionLog(
+                    sessionId,
+                    `variant changed (${previousVariant} -> ${input.variant}), triggering flush`,
+                );
+                args.historyRefreshSessions.add(sessionId);
+                args.systemPromptRefreshSessions.add(sessionId);
+                args.pendingMaterializationSessions.add(sessionId);
+                args.lastHeuristicsTurnId.delete(sessionId);
+            } else {
+                // The provider's cache ignores request params, so a variant
+                // flip is a cache HIT. Defer the queued ops to the next
+                // natural bust (fold / threshold / TTL / flush) exactly as
+                // historian publications do — do NOT manufacture a bust here.
+                // This log line also answers the dashboard-mislabeling
+                // complaint at the log level: the variant change was observed
+                // but the flush was deferred, not triggered.
+                sessionLog(
+                    sessionId,
+                    `variant changed (${previousVariant} -> ${input.variant}) on provider ${providerID} whose cache ignores request params; deferring flush to next natural bust`,
+                );
+            }
         }
     };
 }

packages/plugin/src/hooks/magic-context/sentinel.ts

@@ -36,6 +36,52 @@ export function modelAcceptsEmptyContent(providerID?: string): boolean {
     return providerID === "anthropic";
 }
 
+/**
+ * Decide whether a reasoning-variant change (effort / budget_tokens / toggle)
+ * busts the provider's prompt cache on its own.
+ *
+ * Two provider cache models govern this:
+ *
+ *  - Thinking-config-in-prompt (Anthropic family): the reasoning
+ *    configuration is rendered into the prompt itself, so changing it
+ *    re-serializes the affected message blocks. Anthropic's docs state that
+ *    changing the thinking config "always invalidates message blocks"; the
+ *    same holds on Bedrock. The bust therefore happens regardless of our
+ *    flush, and our queued ops drain on that natural bust.
+ *
+ *  - Request-param-outside-cache-key (OpenAI-compatible et al.):
+ *    reasoning_effort / budget is a request parameter that lives outside the
+ *    cache key. A variant flip does NOT re-serialize any cached bytes, so the
+ *    provider serves the whole prefix as a cache HIT. Our flush would be the
+ *    ONLY bust — manufacturing a gratuitous one that re-runs heuristics and
+ *    re-materializes drops for no provider-side reason (reporter's evidence:
+ *    104 pending ops drained at 54% usage, 156K uncached tokens, while a
+ *    no-op variant flip on the same provider was a full cache HIT).
+ *
+ * Safety asymmetry: misclassifying toward TRUE keeps today's behavior
+ * (wasteful but correct — the ops still drain). Misclassifying toward FALSE
+ * only defers queued ops to the next natural bust (fold / threshold / TTL /
+ * flush), which is the already-designed steady-state for historian publishes
+ * (ARCHITECTURE.md invariant 3: deferred work rides the next bust cycle, it
+ * never forces its own). So the conservative default for unknown providers is
+ * TRUE (today's behavior).
+ *
+ * Provider IDs matched (per OpenCode's `provider/transform.ts`):
+ *   - `anthropic`                     — canonical @ai-sdk/anthropic
+ *   - `bedrock`                        — @ai-sdk/amazon-bedrock (reported as "bedrock")
+ *   - `google-vertex-anthropic`       — @ai-sdk/google-vertex/anthropic
+ * The `bedrock` check uses `includes` so any future bedrock-derived providerID
+ * (e.g. a mantle variant) is also covered, matching how OpenCode's own
+ * `useMessageLevelOptions` gate matches bedrock.
+ */
+export function variantChangeBustsProviderCache(providerID?: string): boolean {
+    if (providerID === undefined) return true;
+    if (providerID === "anthropic") return true;
+    if (providerID === "google-vertex-anthropic") return true;
+    if (providerID.includes("bedrock")) return true;
+    return false;
+}
+
 /**
  * Create an empty-text sentinel to replace a stripped message PART (not a
  * whole message) while preserving the array's length and index positions