fix(tokenizer): real Claude tokenizer now loads; accurate sidebar math and per-message cache

The previous `eval("require")` dynamic import of `ai-tokenizer` silently threw
in Bun's ESM runtime, so every token count since the integration fell through
to `Math.ceil(text.length / 3.5)`. Long sessions showed wildly inflated "Tool
Defs + Overhead" residuals (91K vs real ~22K) and misattributed conversation
tokens.

Key changes:
- `read-session-formatting.ts`: replace `eval("require")` fallback with a
  static ESM import of `ai-tokenizer` + `claudeEncoding`; no heuristic
  fallback anymore.
- `rpc-handlers.ts`: switch 5 remaining chars/3.5 call sites in sidebar /
  status snapshots to `estimateTokens()` so compartment, fact, and memory
  token math subtracts from `conversationTokens` in the same units.
- `transform.ts`: per-message token counter now covers text, reasoning,
  thinking, redacted_thinking, file (with real image token estimation), and
  all tool-call variants; persists `conversation_tokens` and
  `tool_call_tokens` separately so the sidebar splits "Tool Calls" (reducible)
  from "Tool Defs + Overhead" (residual).
- `system-prompt-hash.ts`: self-heal `system_prompt_tokens` when stored value
  drifts >50 tokens from a fresh `estimateTokens()` count so existing sessions
  recover from the old heuristic without migration.
- `storage-meta-*`: add `conversation_tokens` and `tool_call_tokens` columns
  via `ensureColumn`; include them in the `getOrCreateSessionMeta` SELECT and
  `SessionMetaRow` validator; keep `toSessionMeta` defensive with NULL
  fallbacks so upgraded DBs never crash.
- `storage-db.ts`: add `healNullIntegerColumns` companion to
  `healNullTextColumns`; normalize NULL INTEGER columns (added via
  ensureColumn against pre-existing rows) to 0 at startup. Guards against the
  scheduler-reset cascade on old DBs.
- `key-files/read-stats.ts` + `identify-key-files.ts`: byte-based `/3.5`
  estimate is kept only where the file content isn't loaded (pre-filter
  bucketing); marked `Intentional:` so audits don't re-raise.
- Sidebar UI: rename "Tools" to "Tool Defs + Overhead" to reflect what the
  residual actually represents.

Verified on a 340K-input live session: plugin-stored conv/tool/sys totals now
match the Anthropic wire body within 0.1%.

Tests updated to include the new columns in their seeded schemas.

Author: ualtinok

packages/plugin/src/features/magic-context/key-files/identify-key-files.ts

@@ -94,7 +94,7 @@ export function getKeyFileCandidates(
     const maxPerFileTokens = Math.min(tokenBudget / 2, 5000);
     // Filter to files within the project directory — long-running sessions may have
     // read files from other repos, which should not be pinned as key files.
-    const projectPrefix = projectDirectory ? projectDirectory.replace(/\/$/, "") + "/" : undefined;
+    const projectPrefix = projectDirectory ? `${projectDirectory.replace(/\/$/, "")}/` : undefined;
     return stats.filter(
         (s) =>
             s.latestReadTokens > 0 &&

packages/plugin/src/features/magic-context/key-files/read-stats.ts

@@ -100,6 +100,9 @@ export function getSessionReadStats(
         spreadAcrossCompartments: 0, // TODO: compute from compartment boundaries if needed
         editCount: editCounts.get(row.file_path) ?? 0,
         latestReadBytes: row.latest_read_bytes ?? 0,
+        // Rough estimate from byte size — we don't have the file content at
+        // this layer. Used only for dreamer's key-file budget-fit filtering,
+        // where approximate bucketing is sufficient. Intentional: bytes/3.5.
         latestReadTokens: Math.ceil((row.latest_read_bytes ?? 0) / 3.5),
     }));
 }

packages/plugin/src/features/magic-context/storage-db.ts

@@ -304,6 +304,83 @@ CREATE INDEX IF NOT EXISTS idx_dream_queue_pending ON dream_queue(started_at, en
     // segments, since Anthropic's usage data rolls system + tools + messages
     // together into cache.write but we want to attribute them separately.
     ensureColumn(db, "session_meta", "conversation_tokens", "INTEGER DEFAULT 0");
+    // Token estimate of tool-call parts (tool_use, tool_result, tool, tool-invocation)
+    // inside messages. Separate from conversation_tokens so the sidebar can show an
+    // actionable "Tool Calls" slice that users can reduce via ctx_reduce.
+    ensureColumn(db, "session_meta", "tool_call_tokens", "INTEGER DEFAULT 0");
+
+    // One-time heal: when ensureColumn adds a new TEXT DEFAULT '' or
+    // INTEGER DEFAULT N column, SQLite leaves pre-existing rows with NULL
+    // instead of applying the DEFAULT. isSessionMetaRow used to check
+    // `typeof === "string"` / `"number"` strictly (null fails), so rows
+    // with NULL columns were rejected → getOrCreateSessionMeta returned
+    // defaults (lastResponseTime=0, cacheTtl="5m") → scheduler always
+    // returned "execute" → applyPendingOperations re-ran forever → each
+    // execute pass mutated message content → sustained cache-bust cascade.
+    // The defensive validator now accepts null, but we also heal the data
+    // for good measure (belt and suspenders — failures here are non-fatal).
+    healNullTextColumns(db);
+    healNullIntegerColumns(db);
+}
+
+function healNullTextColumns(db: Database): void {
+    const columns: Array<[string, string]> = [
+        ["cache_ttl", ""],
+        ["last_nudge_band", ""],
+        ["last_transform_error", ""],
+        ["nudge_anchor_message_id", ""],
+        ["nudge_anchor_text", ""],
+        ["sticky_turn_reminder_text", ""],
+        ["sticky_turn_reminder_message_id", ""],
+        ["note_nudge_trigger_message_id", ""],
+        ["note_nudge_sticky_text", ""],
+        ["note_nudge_sticky_message_id", ""],
+        ["system_prompt_hash", ""],
+        ["stripped_placeholder_ids", ""],
+        ["memory_block_cache", ""],
+        ["compaction_marker_state", ""],
+        ["key_files", ""],
+    ];
+    for (const [column, fallback] of columns) {
+        try {
+            db.prepare(`UPDATE session_meta SET ${column} = ? WHERE ${column} IS NULL`).run(
+                fallback,
+            );
+        } catch (_error) {
+            // Ignore — the column may not exist yet on a brand-new DB that
+            // hasn't gone through all ensureColumn calls yet. The heal runs
+            // again on next startup.
+        }
+    }
+}
+
+function healNullIntegerColumns(db: Database): void {
+    // INTEGER columns added via ensureColumn against pre-existing rows.
+    // SQLite does not backfill the DEFAULT on ALTER TABLE, so old rows have
+    // NULL. The validator tolerates null as of this release, but we still
+    // normalize to 0 so subsequent reads from any path (including paths
+    // that bypass toSessionMeta) see a well-formed row.
+    const columns: Array<[string, number]> = [
+        ["times_execute_threshold_reached", 0],
+        ["compartment_in_progress", 0],
+        ["historian_failure_count", 0],
+        ["cleared_reasoning_through_tag", 0],
+        ["memory_block_count", 0],
+        ["system_prompt_tokens", 0],
+        ["conversation_tokens", 0],
+        ["tool_call_tokens", 0],
+        ["note_nudge_trigger_pending", 0],
+    ];
+    for (const [column, fallback] of columns) {
+        try {
+            db.prepare(`UPDATE session_meta SET ${column} = ? WHERE ${column} IS NULL`).run(
+                fallback,
+            );
+        } catch (_error) {
+            // Same rationale as the text heal — swallow missing-column errors
+            // on brand-new DBs; next startup reruns this.
+        }
+    }
 }
 
 // Intentional: the definition regex allows single quotes and parens because SQLite column

packages/plugin/src/features/magic-context/storage-meta-session.ts

@@ -14,7 +14,7 @@ import type { SessionMeta } from "./types";
 export function getOrCreateSessionMeta(db: Database, sessionId: string): SessionMeta {
     const result = db
         .prepare(
-            "SELECT session_id, last_response_time, cache_ttl, counter, last_nudge_tokens, last_nudge_band, last_transform_error, is_subagent, last_context_percentage, last_input_tokens, times_execute_threshold_reached, compartment_in_progress, system_prompt_hash, system_prompt_tokens, cleared_reasoning_through_tag FROM session_meta WHERE session_id = ?",
+            "SELECT session_id, last_response_time, cache_ttl, counter, last_nudge_tokens, last_nudge_band, last_transform_error, is_subagent, last_context_percentage, last_input_tokens, times_execute_threshold_reached, compartment_in_progress, system_prompt_hash, system_prompt_tokens, conversation_tokens, tool_call_tokens, cleared_reasoning_through_tag FROM session_meta WHERE session_id = ?",
         )
         .get(sessionId);
 

packages/plugin/src/features/magic-context/storage-meta-shared.ts

@@ -19,6 +19,7 @@ export interface SessionMetaRow {
     system_prompt_hash: string | number;
     system_prompt_tokens: number;
     conversation_tokens: number;
+    tool_call_tokens: number;
     cleared_reasoning_through_tag: number;
 }
 
@@ -37,30 +38,54 @@ export const META_COLUMNS: Record<string, string> = {
     systemPromptHash: "system_prompt_hash",
     systemPromptTokens: "system_prompt_tokens",
     conversationTokens: "conversation_tokens",
+    toolCallTokens: "tool_call_tokens",
     clearedReasoningThroughTag: "cleared_reasoning_through_tag",
 };
 
 export const BOOLEAN_META_KEYS = new Set(["isSubagent", "compartmentInProgress"]);
 
+// Defensive typeof checks: columns may be NULL in DB when a row was seeded
+// before a column was added with ensureColumn (SQLite sets existing rows to
+// NULL, not to the DEFAULT). Treat null as "absent/empty" rather than
+// rejecting the whole row — falling back to defaults silently loses the real
+// lastResponseTime, cacheTtl, lastContextPercentage, etc., causing the
+// scheduler to always return "execute" and pending ops to re-apply across
+// every turn (cache bust cascade).
+function isStringOrNull(value: unknown): boolean {
+    return value === null || typeof value === "string";
+}
+
+function isNumberOrNull(value: unknown): boolean {
+    return value === null || typeof value === "number";
+}
+
 export function isSessionMetaRow(row: unknown): row is SessionMetaRow {
     if (row === null || typeof row !== "object") return false;
     const r = row as Record<string, unknown>;
     return (
         typeof r.session_id === "string" &&
         typeof r.last_response_time === "number" &&
-        typeof r.cache_ttl === "string" &&
+        isStringOrNull(r.cache_ttl) &&
         typeof r.counter === "number" &&
         typeof r.last_nudge_tokens === "number" &&
-        typeof r.last_nudge_band === "string" &&
-        typeof r.last_transform_error === "string" &&
+        isStringOrNull(r.last_nudge_band) &&
+        isStringOrNull(r.last_transform_error) &&
         typeof r.is_subagent === "number" &&
         typeof r.last_context_percentage === "number" &&
         typeof r.last_input_tokens === "number" &&
-        typeof r.times_execute_threshold_reached === "number" &&
-        typeof r.compartment_in_progress === "number" &&
-        (typeof r.system_prompt_hash === "string" || typeof r.system_prompt_hash === "number") &&
-        typeof r.system_prompt_tokens === "number" &&
-        typeof r.cleared_reasoning_through_tag === "number"
+        // INTEGER columns added via ensureColumn: pre-existing rows get NULL
+        // instead of DEFAULT. Strict typeof "number" would reject those rows
+        // and trigger the scheduler-reset cascade described above. toSessionMeta
+        // falls back to 0 for NULL.
+        isNumberOrNull(r.times_execute_threshold_reached) &&
+        isNumberOrNull(r.compartment_in_progress) &&
+        (r.system_prompt_hash === null ||
+            typeof r.system_prompt_hash === "string" ||
+            typeof r.system_prompt_hash === "number") &&
+        isNumberOrNull(r.system_prompt_tokens) &&
+        isNumberOrNull(r.conversation_tokens) &&
+        isNumberOrNull(r.tool_call_tokens) &&
+        isNumberOrNull(r.cleared_reasoning_through_tag)
     );
 }
 
@@ -81,6 +106,7 @@ export function getDefaultSessionMeta(sessionId: string): SessionMeta {
         systemPromptHash: "",
         systemPromptTokens: 0,
         conversationTokens: 0,
+        toolCallTokens: 0,
         clearedReasoningThroughTag: 0,
     };
 }
@@ -110,26 +136,38 @@ export function ensureSessionMetaRow(db: Database, sessionId: string): void {
 }
 
 export function toSessionMeta(row: SessionMetaRow): SessionMeta {
+    // Defensive: NULL text columns (e.g. seeded rows pre-ensureColumn) must not
+    // crash with `.length on null`. Treat null/empty as absent and map to the
+    // SessionMeta representation.
+    const nudgeBandRaw = typeof row.last_nudge_band === "string" ? row.last_nudge_band : "";
+    const transformErrorRaw =
+        typeof row.last_transform_error === "string" ? row.last_transform_error : "";
+    const cacheTtlRaw =
+        typeof row.cache_ttl === "string" && row.cache_ttl.length > 0 ? row.cache_ttl : "5m";
+    const systemPromptHashRaw = row.system_prompt_hash == null ? "" : row.system_prompt_hash;
+    // Defensive numeric fallbacks: when isSessionMetaRow accepts NULL for
+    // INTEGER columns added via ensureColumn, the raw row may have `null`
+    // here. Coerce to 0 so callers see a usable SessionMeta without having
+    // to null-check every scalar field.
+    const numOrZero = (value: unknown): number => (typeof value === "number" ? value : 0);
     return {
         sessionId: row.session_id,
         lastResponseTime: row.last_response_time,
-        cacheTtl: row.cache_ttl,
+        cacheTtl: cacheTtlRaw,
         counter: row.counter,
         lastNudgeTokens: row.last_nudge_tokens,
         lastNudgeBand:
-            row.last_nudge_band.length > 0
-                ? (row.last_nudge_band as SessionMeta["lastNudgeBand"])
-                : null,
-        lastTransformError: row.last_transform_error.length > 0 ? row.last_transform_error : null,
+            nudgeBandRaw.length > 0 ? (nudgeBandRaw as SessionMeta["lastNudgeBand"]) : null,
+        lastTransformError: transformErrorRaw.length > 0 ? transformErrorRaw : null,
         isSubagent: row.is_subagent === 1,
         lastContextPercentage: row.last_context_percentage,
         lastInputTokens: row.last_input_tokens,
-        timesExecuteThresholdReached: row.times_execute_threshold_reached,
+        timesExecuteThresholdReached: numOrZero(row.times_execute_threshold_reached),
         compartmentInProgress: row.compartment_in_progress === 1,
-        systemPromptHash: String(row.system_prompt_hash),
-        systemPromptTokens: row.system_prompt_tokens,
-        conversationTokens:
-            typeof row.conversation_tokens === "number" ? row.conversation_tokens : 0,
-        clearedReasoningThroughTag: row.cleared_reasoning_through_tag,
+        systemPromptHash: String(systemPromptHashRaw),
+        systemPromptTokens: numOrZero(row.system_prompt_tokens),
+        conversationTokens: numOrZero(row.conversation_tokens),
+        toolCallTokens: numOrZero(row.tool_call_tokens),
+        clearedReasoningThroughTag: numOrZero(row.cleared_reasoning_through_tag),
     };
 }

packages/plugin/src/features/magic-context/storage-tags.test.ts

@@ -55,6 +55,8 @@ function makeMemoryDatabase(): Database {
       historian_last_failure_at INTEGER DEFAULT NULL,
       system_prompt_hash INTEGER DEFAULT 0,
       system_prompt_tokens INTEGER DEFAULT 0,
+      conversation_tokens INTEGER DEFAULT 0,
+      tool_call_tokens INTEGER DEFAULT 0,
       cleared_reasoning_through_tag INTEGER DEFAULT 0
     );
   `);

packages/plugin/src/features/magic-context/storage.test.ts

@@ -390,6 +390,49 @@ describe("magic-context storage", () => {
         db.close(false);
     });
 
+    it("preserves numeric columns when text columns are NULL (regression: cache bust cascade)", () => {
+        //#given
+        // Simulates an older row seeded before ensureColumn added last_transform_error
+        // with a DEFAULT. SQLite sets existing rows to NULL, not the default, so the
+        // text column is NULL but the numeric columns (last_response_time,
+        // last_context_percentage, etc.) carry real cumulative state.
+        // Pre-fix: validator rejected the row, getOrCreateSessionMeta returned
+        // defaults with lastResponseTime=0, scheduler thought TTL had elapsed on
+        // every pass, applyPendingOperations re-ran forever, and each execute
+        // mutation busted cache.
+        const db = makeMemoryDatabase();
+        const realResponseTime = 1_700_000_000_000;
+        // First seed the row so subsequent UPDATE matches.
+        db.prepare(
+            `INSERT INTO session_meta (session_id, last_response_time, cache_ttl, counter)
+             VALUES (?, ?, '59m', 42)`,
+        ).run("ses-nullish", realResponseTime);
+        db.prepare(
+            `UPDATE session_meta SET
+                last_response_time = ?,
+                cache_ttl = '59m',
+                counter = 42,
+                last_nudge_tokens = 100,
+                last_context_percentage = 25.5,
+                last_input_tokens = 250000,
+                last_transform_error = NULL
+             WHERE session_id = ?`,
+        ).run(realResponseTime, "ses-nullish");
+
+        //#when
+        const meta = getOrCreateSessionMeta(db, "ses-nullish");
+
+        //#then — real cumulative state must survive, not be reset to defaults
+        expect(meta.sessionId).toBe("ses-nullish");
+        expect(meta.lastResponseTime).toBe(realResponseTime);
+        expect(meta.cacheTtl).toBe("59m");
+        expect(meta.counter).toBe(42);
+        expect(meta.lastContextPercentage).toBe(25.5);
+        expect(meta.lastInputTokens).toBe(250000);
+        expect(meta.lastTransformError).toBeNull();
+        db.close(false);
+    });
+
     it("getTopNBySize only returns tags with active status", () => {
         //#given
         const db = makeMemoryDatabase();

packages/plugin/src/features/magic-context/types.ts

@@ -35,6 +35,7 @@ export interface SessionMeta {
     systemPromptHash: string;
     systemPromptTokens: number;
     conversationTokens: number;
+    toolCallTokens: number;
     clearedReasoningThroughTag: number;
 }
 

packages/plugin/src/hooks/magic-context/command-handler.test.ts

@@ -70,6 +70,8 @@ function createTestDb(): Database {
       historian_last_failure_at INTEGER DEFAULT NULL,
       system_prompt_hash INTEGER DEFAULT 0,
       system_prompt_tokens INTEGER DEFAULT 0,
+      conversation_tokens INTEGER DEFAULT 0,
+      tool_call_tokens INTEGER DEFAULT 0,
       cleared_reasoning_through_tag INTEGER DEFAULT 0
     );
   `);

packages/plugin/src/hooks/magic-context/compartment-runner.test.ts

@@ -384,7 +384,11 @@ describe("executeContextRecomp", () => {
             client,
             db,
             sessionId: "ses-recomp-full-state",
-            historianChunkTokens: 7,
+            // Budget sized so chunking packs ~2 messages per pass with the real
+            // Claude tokenizer (ai-tokenizer). The previous value of 7 relied on
+            // the `/3.5` heuristic fallback and no longer reproduces a 2-pass
+            // split with accurate tokenization.
+            historianChunkTokens: 13,
             directory: "/tmp",
         });