diff --git a/packages/core/src/services/chatRecordingTypes.ts b/packages/core/src/services/chatRecordingTypes.ts index 2ddc218bdcb..6ad23645bf3 100644 --- a/packages/core/src/services/chatRecordingTypes.ts +++ b/packages/core/src/services/chatRecordingTypes.ts @@ -83,6 +83,8 @@ export interface ConversationRecord { lastUpdated: string; messages: MessageRecord[]; summary?: string; + /** Raw memory notes extracted from the session (markdown) */ + memoryScratchpad?: string; /** Workspace directories added during the session via /dir add */ directories?: string[]; /** The kind of conversation (main agent or subagent) */ diff --git a/packages/core/src/services/sessionSummaryService.test.ts b/packages/core/src/services/sessionSummaryService.test.ts index 1e16c6c1203..ec901525580 100644 --- a/packages/core/src/services/sessionSummaryService.test.ts +++ b/packages/core/src/services/sessionSummaryService.test.ts @@ -9,6 +9,7 @@ import { SessionSummaryService } from './sessionSummaryService.js'; import type { BaseLlmClient } from '../core/baseLlmClient.js'; import type { MessageRecord } from './chatRecordingService.js'; import type { GenerateContentResponse } from '@google/genai'; +import { CoreToolCallStatus } from '../scheduler/types.js'; describe('SessionSummaryService', () => { let service: SessionSummaryService; @@ -940,3 +941,333 @@ describe('SessionSummaryService', () => { }); }); }); + +describe('SessionSummaryService - generateMemoryExtraction', () => { + let service: SessionSummaryService; + let mockBaseLlmClient: BaseLlmClient; + let mockGenerateContent: ReturnType; + + const sampleExtraction = `# Fix auth token refresh bug + +cwd: ~/projects/my-app +outcome: success +keywords: tokenManager, 401, race condition + +## What was done +Debugged intermittent 401 errors caused by concurrent token refresh calls. + +## How the user works +- Prefers seeing a proposed fix before any edits are made + +## What we learned +- Token refresh lives in src/auth/tokenManager.ts`; + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + + mockGenerateContent = vi.fn().mockResolvedValue({ + candidates: [ + { + content: { + parts: [{ text: sampleExtraction }], + }, + }, + ], + } as unknown as GenerateContentResponse); + + mockBaseLlmClient = { + generateContent: mockGenerateContent, + } as unknown as BaseLlmClient; + + service = new SessionSummaryService(mockBaseLlmClient); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('should return summary parsed from heading and full scratchpad', async () => { + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Fix the auth token refresh bug' }], + }, + { + id: '2', + timestamp: '2026-01-01T00:01:00Z', + type: 'gemini', + content: [{ text: 'I found a race condition in tokenManager.ts' }], + }, + ]; + + const result = await service.generateMemoryExtraction({ messages }); + + expect(result).not.toBeNull(); + expect(result!.summary).toBe('Fix auth token refresh bug'); + expect(result!.memoryScratchpad).toBe(sampleExtraction); + }); + + it('should use promptId session-memory-extraction', async () => { + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Hello' }], + }, + ]; + + await service.generateMemoryExtraction({ messages }); + + expect(mockGenerateContent).toHaveBeenCalledWith( + expect.objectContaining({ + promptId: 'session-memory-extraction', + }), + ); + }); + + it('should fall back to first line when no heading found', async () => { + mockGenerateContent.mockResolvedValue({ + candidates: [ + { + content: { + parts: [{ text: 'No heading here\n\nSome content' }], + }, + }, + ], + } as unknown as GenerateContentResponse); + + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Hello' }], + }, + ]; + + const result = await service.generateMemoryExtraction({ messages }); + + expect(result).not.toBeNull(); + expect(result!.summary).toBe('No heading here'); + }); + + it('should return null for empty messages', async () => { + const result = await service.generateMemoryExtraction({ messages: [] }); + + expect(result).toBeNull(); + expect(mockGenerateContent).not.toHaveBeenCalled(); + }); + + it('should return null when LLM returns empty text', async () => { + mockGenerateContent.mockResolvedValue({ + candidates: [ + { + content: { + parts: [{ text: '' }], + }, + }, + ], + } as unknown as GenerateContentResponse); + + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Hello' }], + }, + ]; + + const result = await service.generateMemoryExtraction({ messages }); + + expect(result).toBeNull(); + }); + + it('should return null on timeout', async () => { + mockGenerateContent.mockImplementation( + ({ abortSignal }) => + new Promise((resolve, reject) => { + const timeoutId = setTimeout( + () => + resolve({ + candidates: [{ content: { parts: [{ text: 'Too late' }] } }], + }), + 60000, + ); + + abortSignal?.addEventListener( + 'abort', + () => { + clearTimeout(timeoutId); + const abortError = new Error('Aborted'); + abortError.name = 'AbortError'; + reject(abortError); + }, + { once: true }, + ); + }), + ); + + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Hello' }], + }, + ]; + + const resultPromise = service.generateMemoryExtraction({ + messages, + timeout: 100, + }); + + await vi.advanceTimersByTimeAsync(100); + + const result = await resultPromise; + expect(result).toBeNull(); + }); + + it('should return null on API error', async () => { + mockGenerateContent.mockRejectedValue(new Error('API Error')); + + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Hello' }], + }, + ]; + + const result = await service.generateMemoryExtraction({ messages }); + + expect(result).toBeNull(); + }); + + it('should use larger message window than generateSummary', async () => { + const messages: MessageRecord[] = Array.from({ length: 60 }, (_, i) => ({ + id: `${i}`, + timestamp: '2026-01-01T00:00:00Z', + type: i % 2 === 0 ? ('user' as const) : ('gemini' as const), + content: [{ text: `Message ${i}` }], + })); + + await service.generateMemoryExtraction({ messages }); + + expect(mockGenerateContent).toHaveBeenCalledTimes(1); + const callArgs = mockGenerateContent.mock.calls[0][0]; + const promptText = callArgs.contents[0].parts[0].text; + + // Should include 50 messages (25 first + 25 last), not 20 + const messageCount = (promptText.match(/Message \d+/g) || []).length; + expect(messageCount).toBe(50); + }); + + it('should truncate messages at 2000 chars instead of 500', async () => { + const longMessage = 'x'.repeat(2500); + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: longMessage }], + }, + ]; + + await service.generateMemoryExtraction({ messages }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + const promptText = callArgs.contents[0].parts[0].text; + + // Should contain 2000 x's + '...' but not the full 2500 + expect(promptText).toContain('x'.repeat(2000)); + expect(promptText).toContain('...'); + expect(promptText).not.toContain('x'.repeat(2001)); + }); + + it('should remove quotes from parsed summary', async () => { + mockGenerateContent.mockResolvedValue({ + candidates: [ + { + content: { + parts: [{ text: '# "Fix the bug"\n\nSome content' }], + }, + }, + ], + } as unknown as GenerateContentResponse); + + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Fix the bug' }], + }, + ]; + + const result = await service.generateMemoryExtraction({ messages }); + + expect(result).not.toBeNull(); + expect(result!.summary).toBe('Fix the bug'); + }); + + it('should include recorded tool calls when gemini content is empty', async () => { + const messages: MessageRecord[] = [ + { + id: '1', + timestamp: '2026-01-01T00:00:00Z', + type: 'user', + content: [{ text: 'Figure out why session memory is missing context' }], + }, + { + id: '2', + timestamp: '2026-01-01T00:01:00Z', + type: 'gemini', + content: [{ text: '' }], + toolCalls: [ + { + id: 'tool-1', + name: 'run_shell_command', + args: { + command: + 'cat packages/core/src/services/sessionSummaryService.ts', + }, + result: [ + { + functionResponse: { + id: 'tool-1', + name: 'run_shell_command', + response: { + output: + 'packages/core/src/services/sessionSummaryService.ts', + }, + }, + }, + ], + status: CoreToolCallStatus.Success, + timestamp: '2026-01-01T00:01:05Z', + }, + ], + }, + ]; + + await service.generateMemoryExtraction({ messages }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + const promptText = callArgs.contents[0].parts[0].text as string; + + expect(promptText).toContain('Tool: run_shell_command'); + expect(promptText).toContain( + '"command":"cat packages/core/src/services/sessionSummaryService.ts"', + ); + expect(promptText).toContain( + 'packages/core/src/services/sessionSummaryService.ts', + ); + }); +}); diff --git a/packages/core/src/services/sessionSummaryService.ts b/packages/core/src/services/sessionSummaryService.ts index 09c60a2e310..128e2dd634f 100644 --- a/packages/core/src/services/sessionSummaryService.ts +++ b/packages/core/src/services/sessionSummaryService.ts @@ -4,18 +4,23 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { MessageRecord } from './chatRecordingService.js'; +import type { MessageRecord, ToolCallRecord } from './chatRecordingService.js'; import type { BaseLlmClient } from '../core/baseLlmClient.js'; import { partListUnionToString } from '../core/geminiRequest.js'; import { debugLogger } from '../utils/debugLogger.js'; import type { Content } from '@google/genai'; import { getResponseText } from '../utils/partUtils.js'; import { LlmRole } from '../telemetry/types.js'; +import { safeJsonStringify } from '../utils/safeJsonStringify.js'; const DEFAULT_MAX_MESSAGES = 20; const DEFAULT_TIMEOUT_MS = 5000; const MAX_MESSAGE_LENGTH = 500; +const EXTRACTION_MAX_MESSAGES = 50; +const EXTRACTION_TIMEOUT_MS = 30000; +const EXTRACTION_MAX_MESSAGE_LENGTH = 2000; + const SUMMARY_PROMPT = `Summarize the user's primary intent or goal in this conversation in ONE sentence (max 80 characters). Focus on what the user was trying to accomplish. @@ -31,6 +36,52 @@ Conversation: Summary (max 80 chars):`; +const MEMORY_EXTRACTION_PROMPT = `You are reading a past conversation between a user and an AI assistant. Your job is to extract notes that would help in future similar conversations. + +IMPORTANT: The heading must describe the user's actual goal in the conversation, NOT the task of extracting notes. For example: "Debug auth token refresh bug" or "Add pagination to the API". + +Return a markdown document with this exact structure: + +# <1-line description of what the user was working on, max 80 chars> + +outcome: +keywords: + +## What was done + + +## How the user works + + +## What we learned + + +## What went wrong + + +Rules: +- Be evidence-based. Do not invent facts or claim verification that did not happen. +- Redact secrets: never store tokens, keys, or passwords. Replace with [REDACTED]. +- Prefer the user's own wording when capturing preferences. +- Keep it concise but useful. A future agent should be able to understand what happened without re-reading the full conversation. +- If the conversation has no meaningful reusable signal (trivial questions, one-off tasks), return only the heading line and metadata with empty sections. +- Omit sections that have no content rather than writing placeholder text. + +Conversation: +{conversation} + +Memory notes (markdown):`; + +/** + * Result of memory extraction containing both a summary and full notes. + */ +export interface MemoryExtractionResult { + /** 1-line summary parsed from the markdown heading */ + summary: string; + /** Full markdown memory notes */ + memoryScratchpad: string; +} + /** * Options for generating a session summary. */ @@ -61,49 +112,17 @@ export class SessionSummaryService { } = options; try { - // Filter to user/gemini messages only (exclude system messages) - const filteredMessages = messages.filter((msg) => { - // Skip system messages (info, error, warning) - if (msg.type !== 'user' && msg.type !== 'gemini') { - return false; - } - const content = partListUnionToString(msg.content); - return content.trim().length > 0; - }); - - // Apply sliding window selection: first N + last N messages - let relevantMessages: MessageRecord[]; - if (filteredMessages.length <= maxMessages) { - // If fewer messages than max, include all - relevantMessages = filteredMessages; - } else { - // Sliding window: take the first and last messages. - const firstWindowSize = Math.ceil(maxMessages / 2); - const lastWindowSize = Math.floor(maxMessages / 2); - const firstMessages = filteredMessages.slice(0, firstWindowSize); - const lastMessages = filteredMessages.slice(-lastWindowSize); - relevantMessages = firstMessages.concat(lastMessages); - } + const conversationText = this.formatConversation( + messages, + maxMessages, + MAX_MESSAGE_LENGTH, + ); - if (relevantMessages.length === 0) { + if (!conversationText) { debugLogger.debug('[SessionSummary] No messages to summarize'); return null; } - // Format conversation for the prompt - const conversationText = relevantMessages - .map((msg) => { - const role = msg.type === 'user' ? 'User' : 'Assistant'; - const content = partListUnionToString(msg.content); - // Truncate very long messages to avoid token limit - const truncated = - content.length > MAX_MESSAGE_LENGTH - ? content.slice(0, MAX_MESSAGE_LENGTH) + '...' - : content; - return `${role}: ${truncated}`; - }) - .join('\n\n'); - const prompt = SUMMARY_PROMPT.replace('{conversation}', conversationText); // Create abort controller with timeout @@ -161,4 +180,194 @@ export class SessionSummaryService { return null; } } + + /** + * Extract structured memory notes from a chat session. + * Returns both a 1-line summary (parsed from the heading) and the full + * markdown scratchpad, or null if extraction fails. + */ + async generateMemoryExtraction( + options: GenerateSummaryOptions, + ): Promise { + const { + messages, + maxMessages = EXTRACTION_MAX_MESSAGES, + timeout = EXTRACTION_TIMEOUT_MS, + } = options; + + try { + const conversationText = this.formatConversation( + messages, + maxMessages, + EXTRACTION_MAX_MESSAGE_LENGTH, + ); + + if (!conversationText) { + debugLogger.debug('[SessionSummary] No messages for memory extraction'); + return null; + } + + const prompt = MEMORY_EXTRACTION_PROMPT.replace( + '{conversation}', + conversationText, + ); + + const abortController = new AbortController(); + const timeoutId = setTimeout(() => { + abortController.abort(); + }, timeout); + + try { + const contents: Content[] = [ + { + role: 'user', + parts: [{ text: prompt }], + }, + ]; + + const response = await this.baseLlmClient.generateContent({ + modelConfigKey: { model: 'summarizer-default' }, + contents, + abortSignal: abortController.signal, + promptId: 'session-memory-extraction', + role: LlmRole.UTILITY_SUMMARIZER, + }); + + const rawText = getResponseText(response); + + if (!rawText || rawText.trim().length === 0) { + debugLogger.debug( + '[SessionSummary] Empty memory extraction returned', + ); + return null; + } + + const memoryScratchpad = rawText.trim(); + + // Parse the summary from the first markdown heading + const headingMatch = memoryScratchpad.match(/^#\s+(.+)$/m); + let summary = headingMatch + ? headingMatch[1].trim() + : memoryScratchpad.split('\n')[0].trim(); + + // Clean the summary: remove quotes, normalize whitespace + summary = summary.replace(/^["']|["']$/g, '').trim(); + + debugLogger.debug( + `[SessionSummary] Memory extraction generated, summary: "${summary}"`, + ); + return { summary, memoryScratchpad }; + } finally { + clearTimeout(timeoutId); + } + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + debugLogger.debug('[SessionSummary] Timeout during memory extraction'); + } else { + debugLogger.debug( + `[SessionSummary] Error during memory extraction: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return null; + } + } + + /** + * Filter, window, and format messages into conversation text. + * Returns null if no relevant messages remain after filtering. + */ + private formatConversation( + messages: MessageRecord[], + maxMessages: number, + maxMessageLength: number, + ): string | null { + const filteredMessages = messages + .filter((msg) => msg.type === 'user' || msg.type === 'gemini') + .map((msg) => ({ + msg, + content: this.formatMessageForConversation(msg), + })) + .filter(({ content }) => content.length > 0); + + let relevantMessages: typeof filteredMessages; + if (filteredMessages.length <= maxMessages) { + relevantMessages = filteredMessages; + } else { + const firstWindowSize = Math.ceil(maxMessages / 2); + const lastWindowSize = Math.floor(maxMessages / 2); + const firstMessages = filteredMessages.slice(0, firstWindowSize); + const lastMessages = filteredMessages.slice(-lastWindowSize); + relevantMessages = firstMessages.concat(lastMessages); + } + + if (relevantMessages.length === 0) { + return null; + } + + return relevantMessages + .map(({ msg, content }) => { + const role = msg.type === 'user' ? 'User' : 'Assistant'; + const characters = Array.from(content); + const truncated = + characters.length > maxMessageLength + ? characters.slice(0, maxMessageLength).join('') + '...' + : content; + return `${role}: ${truncated}`; + }) + .join('\n\n'); + } + + private formatMessageForConversation(msg: MessageRecord): string { + const sections: string[] = []; + const content = partListUnionToString(msg.content).trim(); + + if (content) { + sections.push(content); + } + + if (msg.type === 'gemini' && msg.toolCalls?.length) { + sections.push( + ...msg.toolCalls.map((toolCall) => + this.formatToolCallForConversation(toolCall), + ), + ); + } + + return sections.join('\n'); + } + + private formatToolCallForConversation(toolCall: ToolCallRecord): string { + const lines = [`Tool: ${toolCall.name}`, `Status: ${toolCall.status}`]; + + if (Object.keys(toolCall.args).length > 0) { + lines.push(`Args: ${this.stringifyForConversation(toolCall.args)}`); + } + + const result = this.stringifyToolCallResult(toolCall); + if (result) { + lines.push(`Result: ${result}`); + } + + return lines.join('\n'); + } + + private stringifyToolCallResult(toolCall: ToolCallRecord): string { + if (!toolCall.result) { + return ''; + } + + return this.stringifyForConversation(toolCall.result); + } + + private stringifyForConversation(value: unknown): string { + if (value === undefined || value === null) { + return ''; + } + + if (typeof value === 'string') { + return value; + } + + return safeJsonStringify(value); + } } diff --git a/packages/core/src/services/sessionSummaryUtils.test.ts b/packages/core/src/services/sessionSummaryUtils.test.ts index 2314b7ca066..dfca2fe315a 100644 --- a/packages/core/src/services/sessionSummaryUtils.test.ts +++ b/packages/core/src/services/sessionSummaryUtils.test.ts @@ -19,6 +19,7 @@ const mockReaddir = fs.readdir as unknown as ReturnType; vi.mock('./sessionSummaryService.js', () => ({ SessionSummaryService: vi.fn().mockImplementation(() => ({ generateSummary: vi.fn(), + generateMemoryExtraction: vi.fn(), })), })); @@ -27,14 +28,26 @@ vi.mock('../core/baseLlmClient.js', () => ({ BaseLlmClient: vi.fn(), })); +// Mock the ClearcutLogger module +vi.mock('../telemetry/clearcut-logger/clearcut-logger.js', () => ({ + ClearcutLogger: { + getInstance: vi.fn().mockReturnValue(undefined), + }, +})); + // Helper to create a session with N user messages function createSessionWithUserMessages( count: number, - options: { summary?: string; sessionId?: string } = {}, + options: { + summary?: string; + memoryScratchpad?: string; + sessionId?: string; + } = {}, ) { return JSON.stringify({ sessionId: options.sessionId ?? 'session-id', summary: options.summary, + memoryScratchpad: options.memoryScratchpad, messages: Array.from({ length: count }, (_, i) => ({ id: String(i + 1), type: 'user', @@ -47,6 +60,7 @@ describe('sessionSummaryUtils', () => { let mockConfig: Config; let mockContentGenerator: ContentGenerator; let mockGenerateSummary: ReturnType; + let mockGenerateMemoryExtraction: ReturnType; beforeEach(async () => { vi.clearAllMocks(); @@ -62,8 +76,12 @@ describe('sessionSummaryUtils', () => { }, } as unknown as Config; - // Setup mock generateSummary function + // Setup mock functions mockGenerateSummary = vi.fn().mockResolvedValue('Add dark mode to the app'); + mockGenerateMemoryExtraction = vi.fn().mockResolvedValue({ + summary: 'Add dark mode to the app', + memoryScratchpad: '# Add dark mode to the app\n\noutcome: success', + }); // Import the mocked module to access the constructor const { SessionSummaryService } = await import( @@ -73,6 +91,7 @@ describe('sessionSummaryUtils', () => { SessionSummaryService as unknown as ReturnType ).mockImplementation(() => ({ generateSummary: mockGenerateSummary, + generateMemoryExtraction: mockGenerateMemoryExtraction, })); }); @@ -102,7 +121,9 @@ describe('sessionSummaryUtils', () => { vi.mocked(fs.access).mockResolvedValue(undefined); mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(5, { summary: 'Existing summary' }), + createSessionWithUserMessages(5, { + summary: 'Existing summary', + }), ); const result = await getPreviousSession(mockConfig); @@ -179,7 +200,7 @@ describe('sessionSummaryUtils', () => { await expect(generateSummary(mockConfig)).resolves.not.toThrow(); }); - it('should generate and save summary for session needing one', async () => { + it('should generate and save memory extraction for session needing one', async () => { const sessionPath = path.join( '/tmp/project', 'chats', @@ -195,12 +216,16 @@ describe('sessionSummaryUtils', () => { await generateSummary(mockConfig); - expect(mockGenerateSummary).toHaveBeenCalledTimes(1); + expect(mockGenerateMemoryExtraction).toHaveBeenCalledTimes(1); expect(fs.writeFile).toHaveBeenCalledTimes(1); expect(fs.writeFile).toHaveBeenCalledWith( sessionPath, expect.stringContaining('Add dark mode to the app'), ); + expect(fs.writeFile).toHaveBeenCalledWith( + sessionPath, + expect.stringContaining('memoryScratchpad'), + ); }); it('should handle errors gracefully without throwing', async () => { diff --git a/packages/core/src/services/sessionSummaryUtils.ts b/packages/core/src/services/sessionSummaryUtils.ts index c64f19870d3..cc624264bdb 100644 --- a/packages/core/src/services/sessionSummaryUtils.ts +++ b/packages/core/src/services/sessionSummaryUtils.ts @@ -12,13 +12,19 @@ import { SESSION_FILE_PREFIX, type ConversationRecord, } from './chatRecordingService.js'; +import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js'; +import { + MemoryExtractionEvent, + MemoryExtractionSkippedEvent, +} from '../telemetry/types.js'; import fs from 'node:fs/promises'; import path from 'node:path'; const MIN_MESSAGES_FOR_SUMMARY = 1; /** - * Generates and saves a summary for a session file. + * Generates and saves a summary and memory scratchpad for a session file. + * Uses a single LLM call to produce both outputs. */ async function generateAndSaveSummary( config: Config, @@ -55,16 +61,34 @@ async function generateAndSaveSummary( } const baseLlmClient = new BaseLlmClient(contentGenerator, config); const summaryService = new SessionSummaryService(baseLlmClient); + const logger = ClearcutLogger.getInstance(config); + const messageCount = conversation.messages.length; - // Generate summary - const summary = await summaryService.generateSummary({ + // Generate memory extraction (produces both summary and scratchpad) + const startTime = Date.now(); + const result = await summaryService.generateMemoryExtraction({ messages: conversation.messages, }); - - if (!summary) { - debugLogger.warn( - `[SessionSummary] Failed to generate summary for ${sessionPath}`, - ); + const durationMs = Date.now() - startTime; + + if (!result) { + // Fall back to simple summary if extraction fails + const summary = await summaryService.generateSummary({ + messages: conversation.messages, + }); + if (summary) { + await saveSummaryOnly(sessionPath, summary); + logger?.logMemoryExtractionEvent( + new MemoryExtractionEvent(false, durationMs, messageCount, 0, true), + ); + } else { + logger?.logMemoryExtractionEvent( + new MemoryExtractionEvent(false, durationMs, messageCount, 0, false), + ); + debugLogger.warn( + `[SessionSummary] Failed to generate summary for ${sessionPath}`, + ); + } return; } @@ -73,7 +97,7 @@ async function generateAndSaveSummary( // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const freshConversation: ConversationRecord = JSON.parse(freshContent); - // Check if summary was added by another process + // Check if extraction was added by another process if (freshConversation.summary) { debugLogger.debug( `[SessionSummary] Summary was added by another process for ${sessionPath}`, @@ -81,12 +105,47 @@ async function generateAndSaveSummary( return; } - // Add summary and write back + // Add both summary and scratchpad, then write back + freshConversation.summary = result.summary; + freshConversation.memoryScratchpad = result.memoryScratchpad; + freshConversation.lastUpdated = new Date().toISOString(); + await fs.writeFile(sessionPath, JSON.stringify(freshConversation, null, 2)); + + logger?.logMemoryExtractionEvent( + new MemoryExtractionEvent( + true, + durationMs, + messageCount, + result.memoryScratchpad.length, + false, + ), + ); + + debugLogger.debug( + `[SessionSummary] Saved memory scratchpad for ${sessionPath}: "${result.summary}"`, + ); +} + +/** + * Saves only the summary (fallback when memory extraction fails). + */ +async function saveSummaryOnly( + sessionPath: string, + summary: string, +): Promise { + const freshContent = await fs.readFile(sessionPath, 'utf-8'); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const freshConversation: ConversationRecord = JSON.parse(freshContent); + + if (freshConversation.summary || freshConversation.memoryScratchpad) { + return; + } + freshConversation.summary = summary; freshConversation.lastUpdated = new Date().toISOString(); await fs.writeFile(sessionPath, JSON.stringify(freshConversation, null, 2)); debugLogger.debug( - `[SessionSummary] Saved summary for ${sessionPath}: "${summary}"`, + `[SessionSummary] Saved summary (fallback) for ${sessionPath}: "${summary}"`, ); } @@ -123,38 +182,41 @@ export async function getPreviousSession( // Filename format: session-YYYY-MM-DDTHH-MM-XXXXXXXX.json sessionFiles.sort((a, b) => b.localeCompare(a)); - // Check the most recently created session - const mostRecentFile = sessionFiles[0]; - const filePath = path.join(chatsDir, mostRecentFile); - - try { - const content = await fs.readFile(filePath, 'utf-8'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const conversation: ConversationRecord = JSON.parse(content); - - if (conversation.summary) { - debugLogger.debug( - '[SessionSummary] Most recent session already has summary', - ); - return null; - } - - // Only generate summaries for sessions with more than 1 user message - const userMessageCount = conversation.messages.filter( - (m) => m.type === 'user', - ).length; - if (userMessageCount <= MIN_MESSAGES_FOR_SUMMARY) { - debugLogger.debug( - `[SessionSummary] Most recent session has ${userMessageCount} user message(s), skipping (need more than ${MIN_MESSAGES_FOR_SUMMARY})`, - ); - return null; + // Iterate through sessions to find the first eligible one. + // The most recent file is typically the current active session (few messages), + // so we skip past ineligible sessions. + for (const file of sessionFiles.slice(0, 20)) { + const filePath = path.join(chatsDir, file); + + try { + const content = await fs.readFile(filePath, 'utf-8'); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const conversation: ConversationRecord = JSON.parse(content); + + // Skip if summary already exists + if (conversation.summary) { + continue; + } + + // Skip sessions with too few user messages + const userMessageCount = conversation.messages.filter( + (m) => m.type === 'user', + ).length; + if (userMessageCount <= MIN_MESSAGES_FOR_SUMMARY) { + continue; + } + + return filePath; + } catch { + // Skip unreadable files + continue; } - - return filePath; - } catch { - debugLogger.debug('[SessionSummary] Could not read most recent session'); - return null; } + + debugLogger.debug( + '[SessionSummary] No eligible session found for memory extraction', + ); + return null; } catch (error) { debugLogger.debug( `[SessionSummary] Error finding previous session: ${error instanceof Error ? error.message : String(error)}`, @@ -172,6 +234,10 @@ export async function generateSummary(config: Config): Promise { const sessionPath = await getPreviousSession(config); if (sessionPath) { await generateAndSaveSummary(config, sessionPath); + } else { + ClearcutLogger.getInstance(config)?.logMemoryExtractionSkippedEvent( + new MemoryExtractionSkippedEvent('no_eligible_session'), + ); } } catch (error) { // Log but don't throw - we want graceful degradation diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index a5896d57f34..0a4b19f81ec 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -53,6 +53,8 @@ import type { StartupStatsEvent, OnboardingStartEvent, OnboardingSuccessEvent, + MemoryExtractionEvent, + MemoryExtractionSkippedEvent, } from '../types.js'; import type { CreditsUsedEvent, @@ -139,6 +141,8 @@ export enum EventNames { BROWSER_AGENT_VISION_STATUS = 'browser_agent_vision_status', BROWSER_AGENT_TASK_OUTCOME = 'browser_agent_task_outcome', BROWSER_AGENT_CLEANUP = 'browser_agent_cleanup', + MEMORY_EXTRACTION = 'memory_extraction', + MEMORY_EXTRACTION_SKIPPED = 'memory_extraction_skipped', } export interface LogResponse { @@ -2079,6 +2083,58 @@ export class ClearcutLogger { this.flushIfNeeded(); } + // ========================================================================== + // Memory Extraction Events + // ========================================================================== + + logMemoryExtractionEvent(event: MemoryExtractionEvent): void { + const data: EventValue[] = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_SUCCESS, + value: event.success.toString(), + }, + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_DURATION_MS, + value: event.duration_ms.toString(), + }, + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_MESSAGE_COUNT, + value: event.message_count.toString(), + }, + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_SCRATCHPAD_LENGTH, + value: event.scratchpad_length.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_FALLBACK, + value: event.fallback.toString(), + }, + ]; + + this.enqueueLogEvent( + this.createLogEvent(EventNames.MEMORY_EXTRACTION, data), + ); + this.flushIfNeeded(); + } + + logMemoryExtractionSkippedEvent(event: MemoryExtractionSkippedEvent): void { + const data: EventValue[] = [ + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_MEMORY_EXTRACTION_SKIP_REASON, + value: event.skip_reason, + }, + ]; + + this.enqueueLogEvent( + this.createLogEvent(EventNames.MEMORY_EXTRACTION_SKIPPED, data), + ); + this.flushIfNeeded(); + } + /** * Adds default fields to data, and returns a new data array. This fields * should exist on all log events. diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index b9e7b2b75ca..a0187f85c66 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -7,7 +7,7 @@ // Defines valid event metadata keys for Clearcut logging. export enum EventMetadataKey { // Deleted enums: 24 - // Next ID: 203 + // Next ID: 209 GEMINI_CLI_KEY_UNKNOWN = 0, @@ -753,4 +753,26 @@ export enum EventMetadataKey { // Logs the number of tools discovered from the MCP server. GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT = 202, + + // ========================================================================== + // Memory Extraction Event Keys + // ========================================================================== + + // Logs whether the memory extraction succeeded. + GEMINI_CLI_MEMORY_EXTRACTION_SUCCESS = 203, + + // Logs the duration of the memory extraction LLM call in milliseconds. + GEMINI_CLI_MEMORY_EXTRACTION_DURATION_MS = 204, + + // Logs the number of messages in the session being extracted. + GEMINI_CLI_MEMORY_EXTRACTION_MESSAGE_COUNT = 205, + + // Logs the character length of the generated scratchpad. + GEMINI_CLI_MEMORY_EXTRACTION_SCRATCHPAD_LENGTH = 206, + + // Logs whether the extraction fell back to simple summary. + GEMINI_CLI_MEMORY_EXTRACTION_FALLBACK = 207, + + // Logs the reason memory extraction was skipped. + GEMINI_CLI_MEMORY_EXTRACTION_SKIP_REASON = 208, } diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 9d6cd08c72e..b89495fe3cb 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -2443,3 +2443,74 @@ export class TokenStorageInitializationEvent implements BaseTelemetryEvent { return `Token storage initialized. Type: ${this.type}. Forced: ${this.forced}`; } } + +export const EVENT_MEMORY_EXTRACTION = 'gemini_cli.memory.extraction'; +export class MemoryExtractionEvent implements BaseTelemetryEvent { + 'event.name': 'memory_extraction'; + 'event.timestamp': string; + success: boolean; + duration_ms: number; + message_count: number; + scratchpad_length: number; + fallback: boolean; + + constructor( + success: boolean, + duration_ms: number, + message_count: number, + scratchpad_length: number, + fallback: boolean, + ) { + this['event.name'] = 'memory_extraction'; + this['event.timestamp'] = new Date().toISOString(); + this.success = success; + this.duration_ms = duration_ms; + this.message_count = message_count; + this.scratchpad_length = scratchpad_length; + this.fallback = fallback; + } + + toOpenTelemetryAttributes(config: Config): LogAttributes { + return { + ...getCommonAttributes(config), + 'event.name': EVENT_MEMORY_EXTRACTION, + 'event.timestamp': this['event.timestamp'], + success: this.success, + duration_ms: this.duration_ms, + message_count: this.message_count, + scratchpad_length: this.scratchpad_length, + fallback: this.fallback, + }; + } + + toLogBody(): string { + return `Memory extraction ${this.success ? 'succeeded' : 'failed'}. Duration: ${this.duration_ms}ms. Messages: ${this.message_count}. Scratchpad: ${this.scratchpad_length} chars. Fallback: ${this.fallback}`; + } +} + +export const EVENT_MEMORY_EXTRACTION_SKIPPED = + 'gemini_cli.memory.extraction_skipped'; +export class MemoryExtractionSkippedEvent implements BaseTelemetryEvent { + 'event.name': 'memory_extraction_skipped'; + 'event.timestamp': string; + skip_reason: string; + + constructor(skip_reason: string) { + this['event.name'] = 'memory_extraction_skipped'; + this['event.timestamp'] = new Date().toISOString(); + this.skip_reason = skip_reason; + } + + toOpenTelemetryAttributes(config: Config): LogAttributes { + return { + ...getCommonAttributes(config), + 'event.name': EVENT_MEMORY_EXTRACTION_SKIPPED, + 'event.timestamp': this['event.timestamp'], + skip_reason: this.skip_reason, + }; + } + + toLogBody(): string { + return `Memory extraction skipped. Reason: ${this.skip_reason}`; + } +}