feat(core): union-find context compaction for AgentHistoryProvider

Add union-find clustering as an alternative compression strategy,
enabled via COMPRESSION_STRATEGY experiment flag ('union-find').

Messages graduate from a hot buffer into a cold forest where
semantically similar messages merge into equivalence classes.
Cluster summaries replace raw messages while preserving provenance
through parent pointers.

New files:
- contextWindow.ts: Forest (union-find with path compression) +
  ContextWindow (hot/cold partitioning with overlap window)
- embeddingService.ts: TF-IDF embedder (no external model)
- clusterSummarizer.ts: async LLM cluster summarization

Integration:
- chatCompressionService.ts: branches on experiment flag
- config.ts: async getCompressionStrategy() with ensureExperimentsLoaded
- agentHistoryProvider.ts: prompt injection sanitization
- toolDistillationService.ts: AbortSignal propagation, 64K distillation cap

80 new tests across contextWindow, embeddingService, clusterSummarizer,
agentHistoryProvider, and toolDistillationService.

Experiment: 7 pre-registered trials, UF leads flat by 8-18pp on factual
recall at high compression pressure (p=0.039 in trial 2).
See github.com/kimjune01/union-find-compaction

Resolves #22877

Author: kimjune01

packages/core/src/code_assist/experiments/flagNames.ts

@@ -20,6 +20,7 @@ export const ExperimentFlags = {
   PRO_MODEL_NO_ACCESS: 45768879,
   GEMINI_3_1_FLASH_LITE_LAUNCHED: 45771641,
   DEFAULT_REQUEST_TIMEOUT: 45773134,
+  COMPRESSION_STRATEGY: 45768880,
 } as const;
 
 export type ExperimentFlagName =

packages/core/src/config/config.ts

@@ -2350,9 +2350,8 @@ export class Config implements McpContext, AgentLoopContext {
     if (this.experimentalJitContext && this.memoryContextManager) {
       await this.memoryContextManager.refresh();
     } else {
-      const { refreshServerHierarchicalMemory } = await import(
-        '../utils/memoryDiscovery.js'
-      );
+      const { refreshServerHierarchicalMemory } =
+        await import('../utils/memoryDiscovery.js');
       await refreshServerHierarchicalMemory(this);
     }
     if (this._geminiClient?.isInitialized()) {
@@ -3025,6 +3024,17 @@ export class Config implements McpContext, AgentLoopContext {
     return remoteThreshold;
   }
 
+  async getCompressionStrategy(): Promise<string> {
+    await this.ensureExperimentsLoaded();
+    const remoteStrategy =
+      this.experiments?.flags[ExperimentFlags.COMPRESSION_STRATEGY]
+        ?.stringValue;
+    if (remoteStrategy === 'union-find' || remoteStrategy === 'flat') {
+      return remoteStrategy;
+    }
+    return 'flat';
+  }
+
   async getUserCaching(): Promise<boolean | undefined> {
     await this.ensureExperimentsLoaded();
 

packages/core/src/context/agentHistoryProvider.ts

@@ -20,6 +20,15 @@ import {
   normalizeFunctionResponse,
 } from './truncation.js';
 
+function sanitizePromptInput(value: unknown): string {
+  return (JSON.stringify(value) ?? '')
+    .replace(/\\[rn]/g, ' ')
+    .replace(/[\r\n\u2028\u2029]+/g, ' ')
+    .replace(/```/g, "'''")
+    .replace(/[<>]/g, (char) => (char === '<' ? '&lt;' : '&gt;'))
+    .replace(/[\x00-\x1f\x7f]/g, ''); // eslint-disable-line no-control-regex
+}
+
 export class AgentHistoryProvider {
   // TODO(joshualitt): just pass the BaseLlmClient instead of the whole Config.
   constructor(
@@ -379,10 +388,10 @@ Distill these into a high-density Markdown block that orientates the agent on th
 - **Brevity:** Maximum 15 lines. No conversational preamble.
 
 ${hasPreviousSummary ? 'PREVIOUS SUMMARY AND TRUNCATED HISTORY:' : 'TRUNCATED HISTORY:'}
-${JSON.stringify(messagesToTruncate)}
+${sanitizePromptInput(messagesToTruncate)}
 
 ACTIVE BRIDGE (LOOKAHEAD):
-${JSON.stringify(bridge)}`;
+${sanitizePromptInput(bridge)}`;
 
     const summaryResponse = await this.config
       .getBaseLlmClient()

packages/core/src/context/chatCompressionService.test.ts

@@ -193,6 +193,7 @@ describe('ChatCompressionService', () => {
         getProjectTempDir: vi.fn().mockReturnValue(testTempDir),
       },
       getApprovedPlanPath: vi.fn().mockReturnValue('/path/to/plan.md'),
+      getCompressionStrategy: vi.fn().mockReturnValue('flat'),
     } as unknown as Config;
 
     vi.mocked(getInitialChatHistory).mockImplementation(
@@ -897,4 +898,144 @@ describe('ChatCompressionService', () => {
       );
     });
   });
+
+  describe('Compression strategy dispatch', () => {
+    it('should route to flat compression when strategy is flat', async () => {
+      vi.mocked(
+        mockConfig as unknown as { getCompressionStrategy: () => string },
+      ).getCompressionStrategy = vi.fn().mockReturnValue('flat');
+
+      const history: Content[] = [
+        { role: 'user', parts: [{ text: 'msg1' }] },
+        { role: 'model', parts: [{ text: 'msg2' }] },
+        { role: 'user', parts: [{ text: 'msg3' }] },
+        { role: 'model', parts: [{ text: 'msg4' }] },
+      ];
+      vi.mocked(mockChat.getHistory).mockReturnValue(history);
+      vi.mocked(mockChat.getLastPromptTokenCount).mockReturnValue(600000);
+
+      const result = await service.compress(
+        mockChat,
+        mockPromptId,
+        false,
+        mockModel,
+        mockConfig,
+        false,
+      );
+
+      expect(result.info.compressionStatus).toBe(CompressionStatus.COMPRESSED);
+      // Flat uses 2 LLM calls (generate + verify)
+      expect(
+        mockConfig.getBaseLlmClient().generateContent,
+      ).toHaveBeenCalledTimes(2);
+    });
+
+    it('should route to union-find compression when strategy is union-find', async () => {
+      vi.mocked(
+        mockConfig as unknown as { getCompressionStrategy: () => string },
+      ).getCompressionStrategy = vi.fn().mockReturnValue('union-find');
+
+      // Mock for cluster summarization
+      const mockLlmClient = {
+        generateContent: vi.fn().mockResolvedValue({
+          candidates: [
+            {
+              content: {
+                parts: [{ text: 'Cluster summary' }],
+              },
+            },
+          ],
+        } as unknown as GenerateContentResponse),
+      };
+      vi.mocked(mockConfig.getBaseLlmClient).mockReturnValue(
+        mockLlmClient as unknown as BaseLlmClient,
+      );
+
+      // Need enough messages to trigger graduation (> UNION_FIND_HOT_SIZE = 30)
+      const history: Content[] = [];
+      for (let i = 0; i < 35; i++) {
+        history.push({
+          role: i % 2 === 0 ? 'user' : 'model',
+          parts: [{ text: `message ${i}` }],
+        });
+      }
+      vi.mocked(mockChat.getHistory).mockReturnValue(history);
+      vi.mocked(mockChat.getLastPromptTokenCount).mockReturnValue(600000);
+
+      const result = await service.compress(
+        mockChat,
+        mockPromptId,
+        false,
+        mockModel,
+        mockConfig,
+        false,
+      );
+
+      expect(result.info.compressionStatus).toBe(CompressionStatus.COMPRESSED);
+      expect(result.newHistory).not.toBeNull();
+    });
+
+    it('should return valid ChatCompressionInfo for union-find path', async () => {
+      vi.mocked(
+        mockConfig as unknown as { getCompressionStrategy: () => string },
+      ).getCompressionStrategy = vi.fn().mockReturnValue('union-find');
+
+      const mockLlmClient = {
+        generateContent: vi.fn().mockResolvedValue({
+          candidates: [
+            {
+              content: {
+                parts: [{ text: 'Summary' }],
+              },
+            },
+          ],
+        } as unknown as GenerateContentResponse),
+      };
+      vi.mocked(mockConfig.getBaseLlmClient).mockReturnValue(
+        mockLlmClient as unknown as BaseLlmClient,
+      );
+
+      const history: Content[] = [];
+      for (let i = 0; i < 35; i++) {
+        history.push({
+          role: i % 2 === 0 ? 'user' : 'model',
+          parts: [{ text: `msg ${i}` }],
+        });
+      }
+      vi.mocked(mockChat.getHistory).mockReturnValue(history);
+      vi.mocked(mockChat.getLastPromptTokenCount).mockReturnValue(600000);
+
+      const result = await service.compress(
+        mockChat,
+        mockPromptId,
+        false,
+        mockModel,
+        mockConfig,
+        false,
+      );
+
+      expect(result.info.originalTokenCount).toBe(600000);
+      expect(result.info.newTokenCount).toBeDefined();
+      expect(typeof result.info.newTokenCount).toBe('number');
+    });
+
+    it('should return NOOP for union-find with empty history', async () => {
+      vi.mocked(
+        mockConfig as unknown as { getCompressionStrategy: () => string },
+      ).getCompressionStrategy = vi.fn().mockReturnValue('union-find');
+      vi.mocked(mockChat.getHistory).mockReturnValue([]);
+
+      const result = await service.compress(
+        mockChat,
+        mockPromptId,
+        false,
+        mockModel,
+        mockConfig,
+        false,
+      );
+
+      expect(result.info.compressionStatus).toBe(CompressionStatus.NOOP);
+      expect(result.newHistory).toBeNull();
+    });
+  });
 });