diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c59e466b1a..d87a6d38ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,12 +11,14 @@ jobs: name: TypeScript runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.9" - name: Cache Bun dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.bun/install/cache key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} @@ -39,14 +41,14 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" - name: Install linter - run: pip install ruff + run: pip install ruff==0.9.10 - name: Lint run: ruff check src @@ -59,9 +61,9 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index efbda3b883..2e7e9b1d45 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,10 +20,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: "pip" @@ -36,10 +36,10 @@ jobs: run: mkdocs build -f docs/mkdocs.yml -d site - name: Setup Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 with: path: docs/site @@ -52,4 +52,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/publish-engine.yml b/.github/workflows/publish-engine.yml index d07b6298f2..3d7647fc3b 100644 --- a/.github/workflows/publish-engine.yml +++ b/.github/workflows/publish-engine.yml @@ -13,22 +13,23 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: "pip" cache-dependency-path: packages/altimate-engine/pyproject.toml - name: Install build tools - run: pip install build + run: pip install build==1.2.2 - name: Build package run: python -m build working-directory: packages/altimate-engine - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 with: packages-dir: packages/altimate-engine/dist/ + skip-existing: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1bee2e6eb6..a8c78ec72b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,10 +9,6 @@ concurrency: group: release cancel-in-progress: false -permissions: - contents: write - id-token: write - env: GH_REPO: AltimateAI/altimate-code @@ -20,17 +16,21 @@ jobs: build: name: Build (${{ matrix.os }}) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: os: [linux, darwin, win32] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.9" - name: Cache Bun dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.bun/install/cache key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} @@ -50,7 +50,7 @@ jobs: MODELS_DEV_API_JSON: test/tool/fixtures/models-api.json - name: Upload build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: dist-${{ matrix.os }} path: packages/altimate-code/dist/ @@ -59,13 +59,17 @@ jobs: name: Publish to npm needs: build runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2 + with: + bun-version: "1.3.9" - name: Cache Bun dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.bun/install/cache key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} @@ -76,7 +80,7 @@ jobs: run: bun install - name: Download all build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: dist-* path: packages/altimate-code/dist/ @@ -124,23 +128,23 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" cache: 'pip' cache-dependency-path: packages/altimate-engine/pyproject.toml - name: Install build tools - run: pip install build + run: pip install build==1.2.2 - name: Build package run: python -m build working-directory: packages/altimate-engine - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 with: packages-dir: packages/altimate-engine/dist/ skip-existing: true @@ -152,7 +156,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 @@ -206,14 +210,14 @@ jobs: CURRENT_TAG: ${{ github.ref_name }} - name: Download all build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: dist-* path: packages/altimate-code/dist/ merge-multiple: true - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 with: body_path: notes.md draft: false diff --git a/packages/altimate-code/src/cli/cmd/auth.ts b/packages/altimate-code/src/cli/cmd/auth.ts index 308b4ff48f..5a03c501db 100644 --- a/packages/altimate-code/src/cli/cmd/auth.ts +++ b/packages/altimate-code/src/cli/cmd/auth.ts @@ -314,9 +314,23 @@ export const AuthLoginCommand = cmd({ prompts.intro("Add credential") if (args.url) { const wellknown = await fetch(`${args.url}/.well-known/altimate-code`).then((x) => x.json() as any) - prompts.log.info(`Running \`${wellknown.auth.command.join(" ")}\``) + const raw = wellknown?.auth?.command + if (!Array.isArray(raw) || !raw.every((c: unknown) => typeof c === 'string')) { + prompts.log.warn('Invalid auth command from server') + prompts.outro('Done') + return + } + const cmd = raw as string[] + const confirm = await prompts.confirm({ + message: `The server requests to run: ${cmd.join(" ")}. Allow?`, + }) + if (prompts.isCancel(confirm) || !confirm) { + prompts.log.warn("Aborted.") + prompts.outro("Done") + return + } const proc = Bun.spawn({ - cmd: wellknown.auth.command, + cmd, stdout: "pipe", }) const exit = await proc.exited diff --git a/packages/altimate-code/src/mcp/index.ts b/packages/altimate-code/src/mcp/index.ts index 672fc5ad1e..dfc9b53bf0 100644 --- a/packages/altimate-code/src/mcp/index.ts +++ b/packages/altimate-code/src/mcp/index.ts @@ -188,7 +188,10 @@ export namespace MCP { return } - const result = await create(key, mcp).catch(() => undefined) + const result = await create(key, mcp).catch((e) => { + log.warn("failed to initialize MCP server", { key, error: e instanceof Error ? e.message : String(e) }) + return undefined + }) if (!result) return status[key] = result.status @@ -244,7 +247,7 @@ export namespace MCP { async function fetchResourcesForClient(clientName: string, client: Client) { const resources = await withTimeout(client.listResources(), DEFAULT_TIMEOUT).catch((e) => { - log.error("failed to get prompts", { clientName, error: e.message }) + log.error("failed to get resources", { clientName, error: e.message }) return undefined }) @@ -380,7 +383,7 @@ export namespace MCP { }) // Census: collect tool and resource counts (fire-and-forget, never block connect) const remoteTransport = name === "SSE" ? "sse" as const : "streamable-http" as const - Promise.all([ + void Promise.all([ client.listTools().catch(() => ({ tools: [] })), client.listResources().catch(() => ({ resources: [] })), ]).then(([toolsList, resourcesList]) => { @@ -496,7 +499,7 @@ export namespace MCP { duration_ms: Date.now() - localConnectStart, }) // Census: collect tool and resource counts (fire-and-forget, never block connect) - Promise.all([ + void Promise.all([ client.listTools().catch(() => ({ tools: [] })), client.listResources().catch(() => ({ resources: [] })), ]).then(([toolsList, resourcesList]) => { @@ -781,7 +784,7 @@ export namespace MCP { const client = clientsSnapshot[clientName] if (!client) { - log.warn("client not found for prompt", { + log.warn("client not found for resource", { clientName: clientName, }) return undefined @@ -792,7 +795,7 @@ export namespace MCP { uri: resourceUri, }) .catch((e) => { - log.error("failed to get prompt from MCP server", { + log.error("failed to read resource from MCP server", { clientName: clientName, resourceUri: resourceUri, error: e.message, diff --git a/packages/altimate-code/src/session/compaction.ts b/packages/altimate-code/src/session/compaction.ts index f9227d16ba..2ebbfb451c 100644 --- a/packages/altimate-code/src/session/compaction.ts +++ b/packages/altimate-code/src/session/compaction.ts @@ -164,6 +164,9 @@ export namespace SessionCompaction { }) { const attempt = (compactionAttempts.get(input.sessionID) ?? 0) + 1 compactionAttempts.set(input.sessionID, attempt) + input.abort.addEventListener("abort", () => { + compactionAttempts.delete(input.sessionID) + }, { once: true }) Telemetry.track({ type: "compaction_triggered", timestamp: Date.now(), diff --git a/packages/altimate-code/src/telemetry/index.ts b/packages/altimate-code/src/telemetry/index.ts index 59f7d20dc2..187e041fb3 100644 --- a/packages/altimate-code/src/telemetry/index.ts +++ b/packages/altimate-code/src/telemetry/index.ts @@ -474,7 +474,13 @@ export namespace Telemetry { log.debug("telemetry flush failed", { status: response.status }) } } catch { - // Silently drop on failure — telemetry must never break the CLI + // Re-add events that haven't been retried yet to avoid data loss + const retriable = events.filter((e) => !(e as any)._retried) + for (const e of retriable) { + ;(e as any)._retried = true + } + const space = Math.max(0, MAX_BUFFER_SIZE - buffer.length) + buffer.unshift(...retriable.slice(0, space)) } finally { clearTimeout(timeout) } diff --git a/packages/altimate-code/test/mcp/mcp.test.ts b/packages/altimate-code/test/mcp/mcp.test.ts new file mode 100644 index 0000000000..a89dd2aa6e --- /dev/null +++ b/packages/altimate-code/test/mcp/mcp.test.ts @@ -0,0 +1,648 @@ +import { describe, test, expect, beforeEach, mock, afterEach } from "bun:test" + +// --------------------------------------------------------------------------- +// Mocks — set up before importing the MCP module +// --------------------------------------------------------------------------- + +// Track telemetry events +let trackedEvents: Array<{ type: string; [key: string]: any }> = [] + +// Track client operations +const mockClients: Record< + string, + { + connected: boolean + listToolsCalls: number + listToolsResult?: { tools: Array<{ name: string; inputSchema: any; description?: string }> } + listToolsError?: Error + closeCalls: number + } +> = {} + +// Mock transport tracking +const transportAttempts: Array<{ + type: "streamable" | "sse" | "stdio" + error?: string +}> = [] + +// Mock the MCP SDK client +mock.module("@modelcontextprotocol/sdk/client/index.js", () => ({ + Client: class MockClient { + name: string + constructor(opts: any) { + this.name = opts?.name ?? "test" + } + async connect(_transport: any) { + // Connect always succeeds in our mock + } + async listTools() { + return { tools: [] } + } + async listResources() { + return { resources: [] } + } + async listPrompts() { + return { prompts: [] } + } + setNotificationHandler() {} + async close() {} + }, +})) + +mock.module("@modelcontextprotocol/sdk/client/streamableHttp.js", () => ({ + StreamableHTTPClientTransport: class { + constructor() { + transportAttempts.push({ type: "streamable" }) + } + }, +})) + +mock.module("@modelcontextprotocol/sdk/client/sse.js", () => ({ + SSEClientTransport: class { + constructor() { + transportAttempts.push({ type: "sse" }) + } + }, +})) + +mock.module("@modelcontextprotocol/sdk/client/stdio.js", () => ({ + StdioClientTransport: class { + stderr: any = null + constructor() { + transportAttempts.push({ type: "stdio" }) + } + }, +})) + +mock.module("@modelcontextprotocol/sdk/client/auth.js", () => ({ + UnauthorizedError: class UnauthorizedError extends Error { + constructor(msg?: string) { + super(msg ?? "Unauthorized") + this.name = "UnauthorizedError" + } + }, +})) + +mock.module("@modelcontextprotocol/sdk/types.js", () => ({ + CallToolResultSchema: {}, + ToolListChangedNotificationSchema: {}, +})) + +beforeEach(() => { + trackedEvents = [] + transportAttempts.length = 0 + for (const key of Object.keys(mockClients)) { + delete mockClients[key] + } +}) + +// --------------------------------------------------------------------------- +// These tests verify MCP behaviors by testing the patterns and logic used +// in mcp/index.ts. Since the MCP module has complex dependencies (Instance, +// Config, transport SDKs), we test the critical code patterns in isolation, +// similar to how command-resilience.test.ts tests loading patterns. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// 1. MCP error recovery +// --------------------------------------------------------------------------- +describe("MCP error recovery", () => { + /** + * Simulates the listTools error handling from mcp/index.ts lines 681-689. + * When client.listTools() fails, the status is set to "failed" and the + * client is removed from the clients map. + */ + function simulateToolsListFailure( + clientName: string, + clients: Record, + status: Record, + error: Error, + ) { + const failedStatus = { + status: "failed" as const, + error: error instanceof Error ? error.message : String(error), + } + status[clientName] = failedStatus + delete clients[clientName] + return failedStatus + } + + test("listTools failure sets status to 'failed'", () => { + const clients: Record = { "test-server": { mock: true } } + const status: Record = { "test-server": { status: "connected" } } + + const result = simulateToolsListFailure( + "test-server", + clients, + status, + new Error("Connection refused"), + ) + + expect(result.status).toBe("failed") + expect(result.error).toBe("Connection refused") + expect(status["test-server"].status).toBe("failed") + }) + + test("listTools failure removes client from clients map", () => { + const clients: Record = { "test-server": { mock: true } } + const status: Record = { "test-server": { status: "connected" } } + + simulateToolsListFailure( + "test-server", + clients, + status, + new Error("Timeout"), + ) + + expect(clients["test-server"]).toBeUndefined() + }) + + test("listTools failure on one server doesn't affect others", () => { + const clients: Record = { + "server-a": { mock: true }, + "server-b": { mock: true }, + } + const status: Record = { + "server-a": { status: "connected" }, + "server-b": { status: "connected" }, + } + + simulateToolsListFailure( + "server-a", + clients, + status, + new Error("server-a failed"), + ) + + expect(clients["server-a"]).toBeUndefined() + expect(clients["server-b"]).toBeDefined() + expect(status["server-a"].status).toBe("failed") + expect(status["server-b"].status).toBe("connected") + }) +}) + +// --------------------------------------------------------------------------- +// 2. MCP tool registration +// --------------------------------------------------------------------------- +describe("MCP tool registration", () => { + /** + * Simulates the registeredMcpTools logic from mcp/index.ts lines 32-36, 695-705. + */ + function simulateToolRegistration( + toolsResults: Array<{ + clientName: string + tools: Array<{ name: string }> + }>, + ): Set { + const registeredMcpTools = new Set() + + // Mirrors line 695: registeredMcpTools.clear() + registeredMcpTools.clear() + + for (const { clientName, tools } of toolsResults) { + for (const tool of tools) { + const sanitizedClientName = clientName.replace(/[^a-zA-Z0-9_-]/g, "_") + const sanitizedToolName = tool.name.replace(/[^a-zA-Z0-9_-]/g, "_") + const toolName = sanitizedClientName + "_" + sanitizedToolName + registeredMcpTools.add(toolName) + } + } + + return registeredMcpTools + } + + test("registeredMcpTools is populated from connected servers", () => { + const registered = simulateToolRegistration([ + { + clientName: "server-a", + tools: [{ name: "tool1" }, { name: "tool2" }], + }, + { + clientName: "server-b", + tools: [{ name: "tool3" }], + }, + ]) + + expect(registered.size).toBe(3) + expect(registered.has("server-a_tool1")).toBe(true) + expect(registered.has("server-a_tool2")).toBe(true) + expect(registered.has("server-b_tool3")).toBe(true) + }) + + test("isMcpTool returns true for registered tools", () => { + const registered = simulateToolRegistration([ + { + clientName: "myserver", + tools: [{ name: "custom_action" }], + }, + ]) + + // Simulates isMcpTool from mcp/index.ts:34-36 + const isMcpTool = (name: string) => registered.has(name) + + expect(isMcpTool("myserver_custom_action")).toBe(true) + expect(isMcpTool("nonexistent_tool")).toBe(false) + }) + + test("tool names are sanitized correctly", () => { + const registered = simulateToolRegistration([ + { + clientName: "my.server@v2", + tools: [{ name: "run query!" }], + }, + ]) + + // Special chars replaced with underscore + expect(registered.has("my_server_v2_run_query_")).toBe(true) + // Original name is NOT registered + expect(registered.has("my.server@v2_run query!")).toBe(false) + }) + + test("registeredMcpTools is cleared before repopulating", () => { + // First registration + const reg1 = simulateToolRegistration([ + { + clientName: "server-a", + tools: [{ name: "old_tool" }], + }, + ]) + expect(reg1.has("server-a_old_tool")).toBe(true) + + // Second registration with different tools + const reg2 = simulateToolRegistration([ + { + clientName: "server-b", + tools: [{ name: "new_tool" }], + }, + ]) + + // Set is fresh (was cleared) + expect(reg2.has("server-a_old_tool")).toBe(false) + expect(reg2.has("server-b_new_tool")).toBe(true) + }) + + test("empty tools list results in empty set", () => { + const registered = simulateToolRegistration([ + { clientName: "server-a", tools: [] }, + ]) + + expect(registered.size).toBe(0) + }) +}) + +// --------------------------------------------------------------------------- +// 3. MCP initialization resilience +// --------------------------------------------------------------------------- +describe("MCP initialization resilience", () => { + /** + * Simulates the initialization pattern from mcp/index.ts lines 178-209. + * Each MCP server initialization is independent and wrapped in .catch() + * so one failure doesn't prevent others from connecting. + */ + async function simulateInit( + servers: Record< + string, + { + shouldFail: boolean + error?: string + } + >, + ) { + const status: Record = {} + const clients: Record = {} + + await Promise.all( + Object.entries(servers).map(async ([key, server]) => { + const result = await (async () => { + if (server.shouldFail) { + throw new Error(server.error ?? "connection failed") + } + return { + mcpClient: { name: key }, + status: { status: "connected" as const }, + } + })().catch((e) => { + // Mirrors line 191-194 in mcp/index.ts + return undefined + }) + + if (!result) return + + status[key] = result.status + if (result.mcpClient) { + clients[key] = result.mcpClient + } + }), + ) + + return { status, clients } + } + + test("one failing MCP server doesn't prevent others from connecting", async () => { + const result = await simulateInit({ + "healthy-server": { shouldFail: false }, + "broken-server": { shouldFail: true, error: "ECONNREFUSED" }, + "another-healthy": { shouldFail: false }, + }) + + // Healthy servers connected + expect(result.status["healthy-server"]?.status).toBe("connected") + expect(result.status["another-healthy"]?.status).toBe("connected") + expect(result.clients["healthy-server"]).toBeDefined() + expect(result.clients["another-healthy"]).toBeDefined() + + // Broken server didn't get registered + expect(result.status["broken-server"]).toBeUndefined() + expect(result.clients["broken-server"]).toBeUndefined() + }) + + test("all failing servers result in empty clients", async () => { + const result = await simulateInit({ + "broken-1": { shouldFail: true }, + "broken-2": { shouldFail: true }, + }) + + expect(Object.keys(result.clients).length).toBe(0) + expect(Object.keys(result.status).length).toBe(0) + }) + + test("all healthy servers all connect", async () => { + const result = await simulateInit({ + "server-1": { shouldFail: false }, + "server-2": { shouldFail: false }, + "server-3": { shouldFail: false }, + }) + + expect(Object.keys(result.clients).length).toBe(3) + for (const key of ["server-1", "server-2", "server-3"]) { + expect(result.status[key]?.status).toBe("connected") + } + }) + + test("disabled servers are marked as disabled", async () => { + // Simulates mcp/index.ts lines 186-189 + function simulateDisabledCheck( + servers: Record, + ) { + const status: Record = {} + + for (const [key, mcp] of Object.entries(servers)) { + if (mcp.enabled === false) { + status[key] = { status: "disabled" } + continue + } + status[key] = { status: "connected" } + } + + return status + } + + const result = simulateDisabledCheck({ + "active-server": { enabled: true }, + "disabled-server": { enabled: false }, + "default-server": {}, + }) + + expect(result["active-server"].status).toBe("connected") + expect(result["disabled-server"].status).toBe("disabled") + expect(result["default-server"].status).toBe("connected") + }) +}) + +// --------------------------------------------------------------------------- +// 4. MCP timeout handling +// --------------------------------------------------------------------------- +describe("MCP timeout handling", () => { + test("timeout falls back to DEFAULT_TIMEOUT when not configured", () => { + const DEFAULT_TIMEOUT = 30_000 + + const mcp1 = { timeout: undefined } + const mcp2 = { timeout: 60_000 } + + const timeout1 = mcp1.timeout ?? DEFAULT_TIMEOUT + const timeout2 = mcp2.timeout ?? DEFAULT_TIMEOUT + + expect(timeout1).toBe(30_000) + expect(timeout2).toBe(60_000) + }) +}) + +// --------------------------------------------------------------------------- +// 5. MCP status transitions +// --------------------------------------------------------------------------- +describe("MCP status transitions", () => { + test("status type literals are correct", () => { + const statuses: Array<{ status: string }> = [ + { status: "connected" }, + { status: "disabled" }, + { status: "failed" }, + { status: "needs_auth" }, + { status: "needs_client_registration" }, + ] + + const validStatuses = new Set([ + "connected", + "disabled", + "failed", + "needs_auth", + "needs_client_registration", + ]) + + for (const s of statuses) { + expect(validStatuses.has(s.status)).toBe(true) + } + }) + + test("failed status includes error message", () => { + const status = { + status: "failed" as const, + error: "Connection timeout after 30s", + } + + expect(status.status).toBe("failed") + expect(status.error).toContain("timeout") + }) + + test("disconnect sets status to disabled", () => { + // Simulates mcp/index.ts lines 645-664 + const status: Record = { + "my-server": { status: "connected" }, + } + const clients: Record = { + "my-server": { mock: true }, + } + + // Simulate disconnect + delete clients["my-server"] + status["my-server"] = { status: "disabled" } + + expect(status["my-server"].status).toBe("disabled") + expect(clients["my-server"]).toBeUndefined() + }) +}) + +// --------------------------------------------------------------------------- +// 6. MCP telemetry events +// --------------------------------------------------------------------------- +describe("MCP telemetry events", () => { + test("mcp_server_status event structure for connected", () => { + const event: any = { + type: "mcp_server_status", + timestamp: Date.now(), + session_id: "sess-1", + server_name: "test-server", + transport: "stdio", + status: "connected", + duration_ms: 500, + } + + expect(event.type).toBe("mcp_server_status") + expect(event.transport).toBe("stdio") + expect(event.status).toBe("connected") + expect(event.duration_ms).toBe(500) + }) + + test("mcp_server_status event structure for error", () => { + const event: any = { + type: "mcp_server_status", + timestamp: Date.now(), + session_id: "sess-1", + server_name: "test-server", + transport: "streamable-http", + status: "error", + error: "Connection refused".slice(0, 500), + duration_ms: 100, + } + + expect(event.status).toBe("error") + expect(event.error).toBe("Connection refused") + }) + + test("mcp_server_status event for disconnect", () => { + const event: any = { + type: "mcp_server_status", + timestamp: Date.now(), + session_id: "sess-1", + server_name: "test-server", + transport: "stdio", + status: "disconnected", + } + + expect(event.status).toBe("disconnected") + expect(event.duration_ms).toBeUndefined() + }) + + test("mcp_server_census event structure", () => { + const event: any = { + type: "mcp_server_census", + timestamp: Date.now(), + session_id: "sess-1", + server_name: "test-server", + transport: "stdio", + tool_count: 5, + resource_count: 3, + } + + expect(event.type).toBe("mcp_server_census") + expect(event.tool_count).toBe(5) + expect(event.resource_count).toBe(3) + }) + + test("error messages in telemetry are truncated to 500 chars", () => { + const longError = "e".repeat(1000) + const truncated = longError.slice(0, 500) + + expect(truncated.length).toBe(500) + }) +}) + +// --------------------------------------------------------------------------- +// 7. MCP client cleanup +// --------------------------------------------------------------------------- +describe("MCP client cleanup", () => { + test("existing client is closed before replacing on reconnect", async () => { + // Simulates mcp/index.ts lines 289-295 + let closeCalled = false + const existingClient = { + close: async () => { + closeCalled = true + }, + } + const newClient = { mock: true } + + const clients: Record = { "my-server": existingClient } + + // Simulate add() replacing existing client + const existing = clients["my-server"] + if (existing) { + await existing.close().catch(() => {}) + } + clients["my-server"] = newClient + + expect(closeCalled).toBe(true) + expect(clients["my-server"]).toBe(newClient) + }) + + test("close error does not prevent new client from being set", async () => { + const existingClient = { + close: async () => { + throw new Error("close failed") + }, + } + const newClient = { mock: true } + const clients: Record = { "my-server": existingClient } + + const existing = clients["my-server"] + if (existing) { + await existing.close().catch(() => {}) + } + clients["my-server"] = newClient + + expect(clients["my-server"]).toBe(newClient) + }) +}) + +// --------------------------------------------------------------------------- +// 8. MCP listTools failure → client cleanup +// --------------------------------------------------------------------------- +describe("MCP listTools failure and cleanup", () => { + /** + * Simulates mcp/index.ts lines 555-576: after connecting, if listTools + * fails, the client is closed and status is set to "failed". + */ + async function simulatePostConnectListToolsFailure(opts: { + listToolsError: Error + }) { + let clientClosed = false + const mcpClient = { + close: async () => { + clientClosed = true + }, + } + + // Simulate withTimeout(mcpClient.listTools(), ...).catch(...) + const result = await Promise.resolve(undefined) // simulates failed listTools returning undefined + + if (!result) { + await mcpClient.close().catch(() => {}) + return { + mcpClient: undefined, + status: { status: "failed" as const, error: "Failed to get tools" }, + clientClosed, + } + } + + return { mcpClient, status: { status: "connected" as const }, clientClosed } + } + + test("failed listTools closes client and returns failed status", async () => { + const result = await simulatePostConnectListToolsFailure({ + listToolsError: new Error("timeout"), + }) + + expect(result.mcpClient).toBeUndefined() + expect(result.status.status).toBe("failed") + expect((result.status as any).error).toBe("Failed to get tools") + expect(result.clientClosed).toBe(true) + }) +}) diff --git a/packages/altimate-code/test/session/processor.test.ts b/packages/altimate-code/test/session/processor.test.ts new file mode 100644 index 0000000000..ab9f80d538 --- /dev/null +++ b/packages/altimate-code/test/session/processor.test.ts @@ -0,0 +1,730 @@ +import { describe, test, expect, beforeEach, mock } from "bun:test" +import { Telemetry } from "../../src/telemetry" + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +// Capture all telemetry events for assertions +let trackedEvents: Telemetry.Event[] = [] +const originalTrack = Telemetry.track.bind(Telemetry) + +function captureTrack(event: Telemetry.Event) { + trackedEvents.push(event) + // Still call original so shutdown/buffer logic works + originalTrack(event) +} + +beforeEach(async () => { + trackedEvents = [] + await Telemetry.shutdown() +}) + +// --------------------------------------------------------------------------- +// These tests replicate the telemetry instrumentation logic from +// processor.ts, testing it in isolation. This mirrors the approach used in +// command-resilience.test.ts: we duplicate the critical code paths rather +// than trying to stand up the full processor with all its dependencies. +// +// If the instrumentation in processor.ts changes, these tests should be +// updated to match. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// 1. Tool call telemetry +// --------------------------------------------------------------------------- +describe("tool call telemetry", () => { + /** + * Simulates the tool-result telemetry path from processor.ts lines 216-231. + * Returns the event that would be tracked. + */ + function simulateToolResult(opts: { + tool: string + isMcpTool: boolean + sessionID: string + messageID: string + startTime: number + toolCallCounter: number + previousTool: string | null + }): Telemetry.Event { + const toolType = opts.isMcpTool ? ("mcp" as const) : ("standard" as const) + const event: Telemetry.Event = { + type: "tool_call", + timestamp: Date.now(), + session_id: opts.sessionID, + message_id: opts.messageID, + tool_name: opts.tool, + tool_type: toolType, + tool_category: Telemetry.categorizeToolName(opts.tool, toolType), + status: "success", + duration_ms: Date.now() - opts.startTime, + sequence_index: opts.toolCallCounter, + previous_tool: opts.previousTool, + } + return event + } + + /** + * Simulates the tool-error telemetry path from processor.ts lines 252-266. + */ + function simulateToolError(opts: { + tool: string + isMcpTool: boolean + sessionID: string + messageID: string + startTime: number + toolCallCounter: number + previousTool: string | null + error: Error | string + }): Telemetry.Event { + const errToolType = opts.isMcpTool ? ("mcp" as const) : ("standard" as const) + const errorStr = + opts.error instanceof Error ? opts.error.message : String(opts.error) + const event: Telemetry.Event = { + type: "tool_call", + timestamp: Date.now(), + session_id: opts.sessionID, + message_id: opts.messageID, + tool_name: opts.tool, + tool_type: errToolType, + tool_category: Telemetry.categorizeToolName(opts.tool, errToolType), + status: "error", + duration_ms: Date.now() - opts.startTime, + sequence_index: opts.toolCallCounter, + previous_tool: opts.previousTool, + error: errorStr.slice(0, 500), + } + return event + } + + test("tool_call events include tool_name, tool_type, and tool_category", () => { + const event = simulateToolResult({ + tool: "sql_execute", + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 100, + toolCallCounter: 0, + previousTool: null, + }) + + expect(event.type).toBe("tool_call") + expect(event.tool_name).toBe("sql_execute") + expect(event.tool_type).toBe("standard") + expect(event.tool_category).toBe("sql") + expect(event.status).toBe("success") + }) + + test("MCP tools get tool_type 'mcp' and category 'mcp'", () => { + const event = simulateToolResult({ + tool: "myserver_custom_tool", + isMcpTool: true, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 50, + toolCallCounter: 0, + previousTool: null, + }) + + expect(event.tool_type).toBe("mcp") + expect(event.tool_category).toBe("mcp") + }) + + test("sequence_index increments with each tool call", () => { + const events: Telemetry.Event[] = [] + let counter = 0 + let prevTool: string | null = null + + for (const tool of ["read", "edit", "bash"]) { + const event = simulateToolResult({ + tool, + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 50, + toolCallCounter: counter, + previousTool: prevTool, + }) + events.push(event) + counter++ + prevTool = tool + } + + expect((events[0] as any).sequence_index).toBe(0) + expect((events[1] as any).sequence_index).toBe(1) + expect((events[2] as any).sequence_index).toBe(2) + }) + + test("previous_tool tracks the last tool used", () => { + const events: Telemetry.Event[] = [] + let counter = 0 + let prevTool: string | null = null + + for (const tool of ["read", "edit", "bash"]) { + const event = simulateToolResult({ + tool, + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 50, + toolCallCounter: counter, + previousTool: prevTool, + }) + events.push(event) + counter++ + prevTool = tool + } + + expect((events[0] as any).previous_tool).toBeNull() + expect((events[1] as any).previous_tool).toBe("read") + expect((events[2] as any).previous_tool).toBe("edit") + }) + + test("error tool calls include error_message truncated to 500 chars", () => { + const longError = "x".repeat(1000) + const event = simulateToolError({ + tool: "bash", + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 100, + toolCallCounter: 0, + previousTool: null, + error: new Error(longError), + }) + + expect(event.status).toBe("error") + expect(event.error).toBeDefined() + expect(event.error!.length).toBe(500) + }) + + test("error tool calls with short messages preserve full message", () => { + const event = simulateToolError({ + tool: "bash", + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 100, + toolCallCounter: 0, + previousTool: null, + error: new Error("file not found"), + }) + + expect(event.error).toBe("file not found") + }) + + test("error tool calls with string errors work correctly", () => { + const event = simulateToolError({ + tool: "edit", + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime: Date.now() - 100, + toolCallCounter: 0, + previousTool: null, + error: "something went wrong", + }) + + expect(event.error).toBe("something went wrong") + }) + + test("duration_ms is positive", () => { + const startTime = Date.now() - 150 + const event = simulateToolResult({ + tool: "read", + isMcpTool: false, + sessionID: "sess-1", + messageID: "msg-1", + startTime, + toolCallCounter: 0, + previousTool: null, + }) + + expect(event.duration_ms).toBeGreaterThanOrEqual(0) + }) +}) + +// --------------------------------------------------------------------------- +// 2. Tool categorization +// --------------------------------------------------------------------------- +describe("tool categorization", () => { + test("file tools are categorized correctly", () => { + for (const tool of ["read", "write", "edit", "glob", "grep", "bash"]) { + expect(Telemetry.categorizeToolName(tool, "standard")).toBe("file") + } + }) + + test("search tools (grep, glob) are categorized as file", () => { + expect(Telemetry.categorizeToolName("grep", "standard")).toBe("file") + expect(Telemetry.categorizeToolName("glob", "standard")).toBe("file") + }) + + test("shell tools (bash) are categorized as file", () => { + expect(Telemetry.categorizeToolName("bash", "standard")).toBe("file") + }) + + test("MCP tools always return 'mcp' regardless of name", () => { + expect(Telemetry.categorizeToolName("read", "mcp")).toBe("mcp") + expect(Telemetry.categorizeToolName("sql_execute", "mcp")).toBe("mcp") + expect(Telemetry.categorizeToolName("bash", "mcp")).toBe("mcp") + expect(Telemetry.categorizeToolName("dbt_run", "mcp")).toBe("mcp") + }) + + test("SQL tools are categorized as sql", () => { + expect(Telemetry.categorizeToolName("sql_execute", "standard")).toBe("sql") + expect(Telemetry.categorizeToolName("run_query", "standard")).toBe("sql") + }) + + test("schema tools are categorized as schema", () => { + expect(Telemetry.categorizeToolName("schema_inspector", "standard")).toBe("schema") + expect(Telemetry.categorizeToolName("list_columns", "standard")).toBe("schema") + expect(Telemetry.categorizeToolName("describe_table", "standard")).toBe("schema") + }) + + test("dbt tools are categorized as dbt", () => { + expect(Telemetry.categorizeToolName("dbt_build", "standard")).toBe("dbt") + expect(Telemetry.categorizeToolName("dbt_run", "standard")).toBe("dbt") + }) + + test("finops tools are categorized as finops", () => { + expect(Telemetry.categorizeToolName("cost_analysis", "standard")).toBe("finops") + expect(Telemetry.categorizeToolName("finops_report", "standard")).toBe("finops") + expect(Telemetry.categorizeToolName("warehouse_usage_stats", "standard")).toBe("finops") + }) + + test("warehouse tools are categorized as warehouse", () => { + expect(Telemetry.categorizeToolName("warehouse_list", "standard")).toBe("warehouse") + expect(Telemetry.categorizeToolName("connection_test", "standard")).toBe("warehouse") + }) + + test("lineage tools are categorized as lineage", () => { + expect(Telemetry.categorizeToolName("lineage_trace", "standard")).toBe("lineage") + expect(Telemetry.categorizeToolName("dag_viewer", "standard")).toBe("lineage") + }) + + test("unknown tools default to 'standard'", () => { + expect(Telemetry.categorizeToolName("custom_thing", "standard")).toBe("standard") + expect(Telemetry.categorizeToolName("foobar", "standard")).toBe("standard") + }) + + test("categorization is case insensitive", () => { + expect(Telemetry.categorizeToolName("SQL_EXECUTE", "standard")).toBe("sql") + expect(Telemetry.categorizeToolName("DBT_Build", "standard")).toBe("dbt") + expect(Telemetry.categorizeToolName("Read", "standard")).toBe("file") + }) +}) + +// --------------------------------------------------------------------------- +// 3. Error recovery tracking +// --------------------------------------------------------------------------- +describe("error recovery tracking", () => { + /** + * Simulates the error_recovered telemetry path from processor.ts lines 296-308. + * This fires in the "finish-step" case when attempt > 0 and retryErrorType is set. + */ + function simulateErrorRecovery(opts: { + sessionID: string + attempt: number + retryErrorType: string + retryStartTime: number + }): Telemetry.Event { + return { + type: "error_recovered", + timestamp: Date.now(), + session_id: opts.sessionID, + error_type: opts.retryErrorType, + recovery_strategy: "retry", + attempts: opts.attempt, + recovered: true, + duration_ms: Date.now() - opts.retryStartTime, + } + } + + test("error_recovered event fires after retry succeeds", () => { + const event = simulateErrorRecovery({ + sessionID: "sess-1", + attempt: 2, + retryErrorType: "APIError", + retryStartTime: Date.now() - 5000, + }) + + expect(event.type).toBe("error_recovered") + expect(event.error_type).toBe("APIError") + expect(event.recovery_strategy).toBe("retry") + expect(event.attempts).toBe(2) + expect(event.recovered).toBe(true) + expect(event.duration_ms).toBeGreaterThanOrEqual(0) + }) + + test("error_recovered includes correct attempt count after multiple retries", () => { + const event = simulateErrorRecovery({ + sessionID: "sess-1", + attempt: 5, + retryErrorType: "RateLimitError", + retryStartTime: Date.now() - 10000, + }) + + expect(event.attempts).toBe(5) + expect(event.error_type).toBe("RateLimitError") + }) + + test("error_recovered duration reflects time since retry started", () => { + const retryStart = Date.now() - 3000 + const event = simulateErrorRecovery({ + sessionID: "sess-1", + attempt: 1, + retryErrorType: "TimeoutError", + retryStartTime: retryStart, + }) + + // Duration should be at least ~3000ms (allow for timing jitter) + expect(event.duration_ms).toBeGreaterThanOrEqual(2900) + }) + + test("recovery logic: retryErrorType and retryStartTime are reset after recovery", () => { + // Simulate the processor state management from lines 296-308 + let retryErrorType: string | null = "APIError" + let retryStartTime: number | null = Date.now() - 2000 + const attempt = 1 + + // This is the condition check from processor.ts:296 + if (attempt > 0 && retryErrorType) { + // Track the event (would call Telemetry.track in real code) + const event = simulateErrorRecovery({ + sessionID: "sess-1", + attempt, + retryErrorType, + retryStartTime: retryStartTime ?? Date.now(), + }) + // Reset state as processor.ts does at lines 307-308 + retryErrorType = null + retryStartTime = null + + expect(event.type).toBe("error_recovered") + expect(retryErrorType).toBeNull() + expect(retryStartTime).toBeNull() + } + }) + + test("no error_recovered event when attempt is 0", () => { + // Simulate the guard condition from processor.ts:296 + const attempt = 0 + const retryErrorType: string | null = null + let eventFired = false + + if (attempt > 0 && retryErrorType) { + eventFired = true + } + + expect(eventFired).toBe(false) + }) + + test("no error_recovered event when retryErrorType is null", () => { + const attempt = 1 + const retryErrorType: string | null = null + let eventFired = false + + if (attempt > 0 && retryErrorType) { + eventFired = true + } + + expect(eventFired).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// 4. Doom loop detection telemetry +// --------------------------------------------------------------------------- +describe("doom loop detection telemetry", () => { + test("doom_loop_detected event includes tool_name and repeat_count", () => { + const DOOM_LOOP_THRESHOLD = 3 + const event: Telemetry.Event = { + type: "doom_loop_detected", + timestamp: Date.now(), + session_id: "sess-1", + tool_name: "bash", + repeat_count: DOOM_LOOP_THRESHOLD, + } + + Telemetry.track(event) + + expect(event.tool_name).toBe("bash") + expect(event.repeat_count).toBe(3) + }) +}) + +// --------------------------------------------------------------------------- +// 5. Generation telemetry +// --------------------------------------------------------------------------- +describe("generation telemetry", () => { + test("generation event contains all required fields", () => { + const event: Telemetry.Event = { + type: "generation", + timestamp: Date.now(), + session_id: "sess-1", + message_id: "msg-1", + model_id: "claude-opus-4-6", + provider_id: "anthropic", + agent: "builder", + finish_reason: "end_turn", + tokens: { + input: 1000, + output: 500, + reasoning: 200, + cache_read: 800, + cache_write: 100, + }, + cost: 0.05, + duration_ms: 3000, + } + + expect(event.model_id).toBe("claude-opus-4-6") + expect(event.tokens.input).toBe(1000) + expect(event.tokens.cache_read).toBe(800) + expect(event.cost).toBe(0.05) + expect(event.finish_reason).toBe("end_turn") + }) +}) + +// --------------------------------------------------------------------------- +// 6. Context utilization telemetry +// --------------------------------------------------------------------------- +describe("context utilization telemetry", () => { + /** + * Simulates the context utilization tracking from processor.ts lines 348-365. + */ + function simulateContextUtilization(opts: { + sessionID: string + modelID: string + tokensInput: number + tokensOutput: number + cacheRead: number + contextLimit: number + generationCounter: number + }) { + const totalTokens = opts.tokensInput + opts.tokensOutput + opts.cacheRead + const totalInput = opts.cacheRead + opts.tokensInput + + if (opts.contextLimit > 0) { + const event: Telemetry.Event = { + type: "context_utilization", + timestamp: Date.now(), + session_id: opts.sessionID, + model_id: opts.modelID, + tokens_used: totalTokens, + context_limit: opts.contextLimit, + utilization_pct: Math.round((totalTokens / opts.contextLimit) * 1000) / 1000, + generation_number: opts.generationCounter, + cache_hit_ratio: totalInput > 0 ? Math.round((opts.cacheRead / totalInput) * 1000) / 1000 : 0, + } + return event + } + return null + } + + test("context_utilization calculates utilization_pct correctly", () => { + const event = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "claude-opus-4-6", + tokensInput: 5000, + tokensOutput: 1000, + cacheRead: 4000, + contextLimit: 200000, + generationCounter: 1, + }) + + expect(event).not.toBeNull() + // totalTokens = 5000 + 1000 + 4000 = 10000 + // utilization = 10000 / 200000 = 0.05 + expect(event!.utilization_pct).toBe(0.05) + }) + + test("context_utilization calculates cache_hit_ratio correctly", () => { + const event = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "claude-opus-4-6", + tokensInput: 2000, + tokensOutput: 500, + cacheRead: 8000, + contextLimit: 200000, + generationCounter: 1, + }) + + expect(event).not.toBeNull() + // totalInput = 8000 + 2000 = 10000 + // cache_hit_ratio = 8000 / 10000 = 0.8 + expect(event!.cache_hit_ratio).toBe(0.8) + }) + + test("context_utilization returns 0 cache_hit_ratio when no input tokens", () => { + const event = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "claude-opus-4-6", + tokensInput: 0, + tokensOutput: 500, + cacheRead: 0, + contextLimit: 200000, + generationCounter: 1, + }) + + expect(event).not.toBeNull() + expect(event!.cache_hit_ratio).toBe(0) + }) + + test("context_utilization is not emitted when context_limit is 0", () => { + const event = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "some-model", + tokensInput: 5000, + tokensOutput: 1000, + cacheRead: 4000, + contextLimit: 0, + generationCounter: 1, + }) + + expect(event).toBeNull() + }) + + test("generation_number increments correctly", () => { + const event1 = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "claude-opus-4-6", + tokensInput: 1000, + tokensOutput: 500, + cacheRead: 0, + contextLimit: 200000, + generationCounter: 1, + }) + const event2 = simulateContextUtilization({ + sessionID: "sess-1", + modelID: "claude-opus-4-6", + tokensInput: 2000, + tokensOutput: 700, + cacheRead: 1000, + contextLimit: 200000, + generationCounter: 2, + }) + + expect(event1!.generation_number).toBe(1) + expect(event2!.generation_number).toBe(2) + }) +}) + +// --------------------------------------------------------------------------- +// 7. Error telemetry in catch block +// --------------------------------------------------------------------------- +describe("processor error telemetry", () => { + test("error events truncate messages to 500 chars", () => { + const longMessage = "y".repeat(1000) + // Simulates processor.ts lines 457-464 + const event: Telemetry.Event = { + type: "error", + timestamp: Date.now(), + session_id: "sess-1", + error_name: "SomeError", + error_message: longMessage.slice(0, 500), + context: "processor", + } + + expect(event.error_message.length).toBe(500) + expect(event.context).toBe("processor") + }) + + test("error events use error name from exception or fallback", () => { + const e1 = new TypeError("bad type") + const e2 = { message: "plain object" } + + const name1 = e1?.name ?? "UnknownError" + const name2 = (e2 as any)?.name ?? "UnknownError" + + expect(name1).toBe("TypeError") + expect(name2).toBe("UnknownError") + }) + + test("provider_error includes http_status when available", () => { + const error = Object.assign(new Error("rate limited"), { status: 429 }) + + const event: Telemetry.Event = { + type: "provider_error", + timestamp: Date.now(), + session_id: "sess-1", + provider_id: "anthropic", + model_id: "claude-opus-4-6", + error_type: error.name, + error_message: error.message.slice(0, 500), + http_status: (error as any).status, + } + + expect(event.http_status).toBe(429) + expect(event.error_type).toBe("Error") + }) +}) + +// --------------------------------------------------------------------------- +// 8. Processor state machine — counter and state tracking +// --------------------------------------------------------------------------- +describe("processor state tracking", () => { + test("toolCallCounter starts at 0 and increments", () => { + // Simulates processor.ts line 40 and 230 + let toolCallCounter = 0 + expect(toolCallCounter).toBe(0) + + toolCallCounter++ + expect(toolCallCounter).toBe(1) + + toolCallCounter++ + expect(toolCallCounter).toBe(2) + }) + + test("previousTool starts null and tracks last tool", () => { + // Simulates processor.ts lines 41, 231 + let previousTool: string | null = null + expect(previousTool).toBeNull() + + previousTool = "read" + expect(previousTool).toBe("read") + + previousTool = "edit" + expect(previousTool).toBe("edit") + }) + + test("generationCounter starts at 0 and increments on finish-step", () => { + // Simulates processor.ts lines 42, 295 + let generationCounter = 0 + expect(generationCounter).toBe(0) + + // Simulates finish-step handler incrementing + generationCounter++ + expect(generationCounter).toBe(1) + }) + + test("retryErrorType and retryStartTime track retry state", () => { + // Simulates processor.ts lines 43-44, 494-497 + let retryErrorType: string | null = null + let retryStartTime: number | null = null + let attempt = 0 + + expect(retryErrorType).toBeNull() + expect(retryStartTime).toBeNull() + + // First retry + if (attempt === 0) { + retryStartTime = Date.now() + } + retryErrorType = "APIError" + attempt++ + + expect(retryErrorType).toBe("APIError") + expect(retryStartTime).not.toBeNull() + expect(attempt).toBe(1) + + // Second retry (retryStartTime should not be overwritten) + const firstRetryStart = retryStartTime + if (attempt === 0) { + retryStartTime = Date.now() + } + attempt++ + expect(retryStartTime).toBe(firstRetryStart) + expect(attempt).toBe(2) + }) +}) diff --git a/packages/altimate-code/test/telemetry/telemetry.test.ts b/packages/altimate-code/test/telemetry/telemetry.test.ts index a614ca7915..3c2cb1ce99 100644 --- a/packages/altimate-code/test/telemetry/telemetry.test.ts +++ b/packages/altimate-code/test/telemetry/telemetry.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test" +import { describe, expect, test, mock, afterEach, beforeEach, spyOn } from "bun:test" import { Telemetry } from "../../src/telemetry" // --------------------------------------------------------------------------- @@ -341,3 +341,980 @@ describe("telemetry.naming-convention", () => { } }) }) + +// --------------------------------------------------------------------------- +// 8. parseConnectionString (tested indirectly via init + flush) +// --------------------------------------------------------------------------- +describe("telemetry.parseConnectionString (indirect)", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("valid connection string enables telemetry and produces correct endpoint in flush", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchCalls: { url: string; body: string }[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (input: any, init: any) => { + fetchCalls.push({ url: String(input), body: String(init?.body ?? "") }) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=test-key-123;IngestionEndpoint=https://example.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: 1000, + session_id: "s1", + model_id: "m1", + provider_id: "p1", + agent: "a1", + project_id: "proj1", + }) + + await Telemetry.flush() + + expect(fetchCalls.length).toBe(1) + expect(fetchCalls[0].url).toBe("https://example.com/v2/track") + const body = JSON.parse(fetchCalls[0].body) + expect(body[0].iKey).toBe("test-key-123") + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("missing InstrumentationKey in connection string disables telemetry", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "IngestionEndpoint=https://example.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: 1000, + session_id: "s1", + model_id: "m1", + provider_id: "p1", + agent: "a1", + project_id: "proj1", + }) + + await Telemetry.flush() + + // fetch should NOT be called — telemetry disabled due to invalid connection string + expect(fetchMock).not.toHaveBeenCalled() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("trailing slash on IngestionEndpoint is handled correctly", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchCalls: { url: string }[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (input: any) => { + fetchCalls.push({ url: String(input) }) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + // Trailing slash — should not double-up + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=key-456;IngestionEndpoint=https://example.com/" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: 1000, + session_id: "s1", + model_id: "m1", + provider_id: "p1", + agent: "a1", + project_id: "proj1", + }) + + await Telemetry.flush() + + expect(fetchCalls.length).toBe(1) + // Should be /v2/track, not //v2/track + expect(fetchCalls[0].url).toBe("https://example.com/v2/track") + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("empty connection string disables telemetry", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = "" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: 1000, + session_id: "s1", + model_id: "m1", + provider_id: "p1", + agent: "a1", + project_id: "proj1", + }) + + await Telemetry.flush() + expect(fetchMock).not.toHaveBeenCalled() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +}) + +// --------------------------------------------------------------------------- +// 9. toAppInsightsEnvelopes (tested indirectly via track + flush) +// --------------------------------------------------------------------------- +describe("telemetry.toAppInsightsEnvelopes (indirect)", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + async function initWithMockedFetch(): { + fetchCalls: { url: string; body: string }[] + fetchMock: ReturnType + cleanup: () => void + } { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=test-ikey;IngestionEndpoint=https://test.endpoint.com" + + const fetchCalls: { url: string; body: string }[] = [] + const fetchMock = spyOn(global, "fetch").mockImplementation(async (_input: any, init: any) => { + fetchCalls.push({ url: String(_input), body: String(init?.body ?? "") }) + return new Response("", { status: 200 }) + }) + + await Telemetry.init() + + return { + fetchCalls, + fetchMock, + cleanup: () => { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + }, + } + } + + test("basic event is converted to envelope format", async () => { + const { fetchCalls, cleanup } = await initWithMockedFetch() + try { + Telemetry.setContext({ sessionId: "sess-abc", projectId: "proj-xyz" }) + Telemetry.track({ + type: "session_start", + timestamp: 1700000000000, + session_id: "sess-abc", + model_id: "claude-3", + provider_id: "anthropic", + agent: "builder", + project_id: "proj-xyz", + }) + + await Telemetry.flush() + + expect(fetchCalls.length).toBe(1) + const envelopes = JSON.parse(fetchCalls[0].body) + expect(envelopes.length).toBe(1) + + const env = envelopes[0] + expect(env.name).toBe("Microsoft.ApplicationInsights.test-ikey.Event") + expect(env.iKey).toBe("test-ikey") + expect(env.time).toBe(new Date(1700000000000).toISOString()) + expect(env.tags["ai.session.id"]).toBe("sess-abc") + expect(env.tags["ai.cloud.role"]).toBe("altimate") + expect(env.data.baseType).toBe("EventData") + expect(env.data.baseData.ver).toBe(2) + expect(env.data.baseData.name).toBe("session_start") + // String fields go to properties + expect(env.data.baseData.properties.model_id).toBe("claude-3") + expect(env.data.baseData.properties.provider_id).toBe("anthropic") + expect(env.data.baseData.properties.agent).toBe("builder") + } finally { + cleanup() + } + }) + + test("numeric fields go to measurements, string fields go to properties", async () => { + const { fetchCalls, cleanup } = await initWithMockedFetch() + try { + Telemetry.track({ + type: "session_end", + timestamp: 1700000000000, + session_id: "sess-1", + total_cost: 0.05, + total_tokens: 1500, + tool_call_count: 10, + duration_ms: 30000, + }) + + await Telemetry.flush() + + const envelopes = JSON.parse(fetchCalls[0].body) + const baseData = envelopes[0].data.baseData + // Numeric fields in measurements + expect(baseData.measurements.total_cost).toBe(0.05) + expect(baseData.measurements.total_tokens).toBe(1500) + expect(baseData.measurements.tool_call_count).toBe(10) + expect(baseData.measurements.duration_ms).toBe(30000) + } finally { + cleanup() + } + }) + + test("nested tokens object is flattened with tokens_ prefix", async () => { + const { fetchCalls, cleanup } = await initWithMockedFetch() + try { + Telemetry.track({ + type: "generation", + timestamp: 1700000000000, + session_id: "sess-1", + message_id: "msg-1", + model_id: "claude-3", + provider_id: "anthropic", + agent: "builder", + finish_reason: "end_turn", + tokens: { + input: 100, + output: 200, + reasoning: 50, + cache_read: 10, + cache_write: 5, + }, + cost: 0.01, + duration_ms: 2000, + }) + + await Telemetry.flush() + + const envelopes = JSON.parse(fetchCalls[0].body) + const measurements = envelopes[0].data.baseData.measurements + expect(measurements.tokens_input).toBe(100) + expect(measurements.tokens_output).toBe(200) + expect(measurements.tokens_reasoning).toBe(50) + expect(measurements.tokens_cache_read).toBe(10) + expect(measurements.tokens_cache_write).toBe(5) + // Raw "tokens" key should not appear in properties + expect(envelopes[0].data.baseData.properties.tokens).toBeUndefined() + } finally { + cleanup() + } + }) + + test("fallback sessionId when none set on event", async () => { + const { fetchCalls, cleanup } = await initWithMockedFetch() + try { + // Set context with a session ID + Telemetry.setContext({ sessionId: "fallback-sess", projectId: "p" }) + + // Track event without session_id in the actual event payload is not possible + // since all events require session_id. But we can verify the tags use event's session_id. + // If the event session_id is empty, the module falls back to the context sessionId. + Telemetry.track({ + type: "error", + timestamp: 1700000000000, + session_id: "", + error_name: "TestError", + error_message: "test", + context: "test", + }) + + await Telemetry.flush() + + const envelopes = JSON.parse(fetchCalls[0].body) + // Empty session_id falls back to "startup" (since "" is falsy) + // Actually, looking at the code: `const sid = fields.session_id ?? sessionId` + // "" is not null/undefined, so sid = "". Then `sid || "startup"` = "startup" + expect(envelopes[0].tags["ai.session.id"]).toBe("startup") + } finally { + cleanup() + } + }) + + test("timestamp is included as ISO string", async () => { + const { fetchCalls, cleanup } = await initWithMockedFetch() + try { + const ts = 1700000000000 + Telemetry.track({ + type: "session_start", + timestamp: ts, + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + + const envelopes = JSON.parse(fetchCalls[0].body) + expect(envelopes[0].time).toBe(new Date(ts).toISOString()) + } finally { + cleanup() + } + }) +}) + +// --------------------------------------------------------------------------- +// 10. flush +// --------------------------------------------------------------------------- +describe("telemetry.flush", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("flush sends buffered events via fetch", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + let fetchCallCount = 0 + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + fetchCallCount++ + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + expect(fetchCallCount).toBe(1) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("flush clears the buffer after success", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + let fetchCallCount = 0 + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + fetchCallCount++ + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + expect(fetchCallCount).toBe(1) + + // Second flush should not call fetch — buffer is empty + await Telemetry.flush() + expect(fetchCallCount).toBe(1) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("flush re-adds events to buffer on network error", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + let fetchCallCount = 0 + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + fetchCallCount++ + throw new Error("Network error") + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + // First flush — network error, events re-added to buffer + await Telemetry.flush() + expect(fetchCallCount).toBe(1) + + // Second flush should attempt again (events were re-added with _retried flag) + await Telemetry.flush() + expect(fetchCallCount).toBe(2) + + // Third flush — events had _retried=true, so after second failure they are + // filtered out by `events.filter(e => !e._retried)` and NOT re-added. + // Buffer is now empty, so third flush is a no-op. + await Telemetry.flush() + expect(fetchCallCount).toBe(2) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("flush handles AbortController timeout", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (_input: any, init: any) => { + // Verify that signal is passed + expect(init?.signal).toBeDefined() + expect(init.signal).toBeInstanceOf(AbortSignal) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + expect(fetchMock).toHaveBeenCalled() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("droppedEvents counter is included in flush and reset after", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchBodies: string[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (_input: any, init: any) => { + fetchBodies.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + // Fill buffer beyond MAX_BUFFER_SIZE (200) to trigger drops + for (let i = 0; i < 210; i++) { + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + } + + await Telemetry.flush() + + expect(fetchBodies.length).toBe(1) + const envelopes = JSON.parse(fetchBodies[0]) + // Should include a TelemetryBufferOverflow error event + const overflowEvent = envelopes.find( + (e: any) => e.data?.baseData?.name === "error" && + e.data?.baseData?.properties?.error_name === "TelemetryBufferOverflow", + ) + expect(overflowEvent).toBeDefined() + expect(overflowEvent.data.baseData.properties.error_message).toContain("10 events dropped") + + // Second flush — no more dropped events counter + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + await Telemetry.flush() + + const envelopes2 = JSON.parse(fetchBodies[1]) + const overflowEvent2 = envelopes2.find( + (e: any) => e.data?.baseData?.properties?.error_name === "TelemetryBufferOverflow", + ) + expect(overflowEvent2).toBeUndefined() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +}) + +// --------------------------------------------------------------------------- +// 11. shutdown +// --------------------------------------------------------------------------- +describe("telemetry.shutdown", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("shutdown flushes remaining events", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + let fetchCallCount = 0 + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + fetchCallCount++ + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + // shutdown should trigger a flush + await Telemetry.shutdown() + expect(fetchCallCount).toBe(1) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("shutdown resets all state", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + Telemetry.setContext({ sessionId: "sess-1", projectId: "proj-1" }) + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.shutdown() + + // After shutdown, context should be reset + const ctx = Telemetry.getContext() + expect(ctx.sessionId).toBe("") + expect(ctx.projectId).toBe("") + + // After shutdown, track should drop silently (initDone is reset, so it buffers) + // But flush without re-init should do nothing since enabled=false + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s2", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + let fetchCountBefore = fetchMock.mock.calls.length + await Telemetry.flush() + // flush does nothing because enabled=false after shutdown + expect(fetchMock.mock.calls.length).toBe(fetchCountBefore) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("shutdown clears the flush timer (no further periodic flushes)", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + await Telemetry.shutdown() + + // After shutdown, init() promise should be reset (can re-init) + const p1 = Telemetry.init() + // With telemetry disabled env var still unset, this will try to re-init. + // The point is that shutdown fully resets, allowing a fresh init. + await p1 + await Telemetry.shutdown() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +}) + +// --------------------------------------------------------------------------- +// 12. buffer overflow +// --------------------------------------------------------------------------- +describe("telemetry.buffer overflow", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("buffer beyond MAX_BUFFER_SIZE (200) drops oldest events and keeps newest", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchBodies: string[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (_input: any, init: any) => { + fetchBodies.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + // Track 250 events — first 50 should be dropped + for (let i = 0; i < 250; i++) { + Telemetry.track({ + type: "command", + timestamp: i, // use index as timestamp to identify ordering + session_id: "s1", + command_name: `cmd-${i}`, + command_source: "command", + message_id: `msg-${i}`, + }) + } + + await Telemetry.flush() + + expect(fetchBodies.length).toBe(1) + const envelopes = JSON.parse(fetchBodies[0]) + + // 200 buffered events + 1 overflow error event = 201 + expect(envelopes.length).toBe(201) + + // The first event in the buffer should be index 50 (0-49 were dropped) + const firstDataEvent = envelopes[0] + expect(firstDataEvent.data.baseData.properties.command_name).toBe("cmd-50") + + // The last data event should be index 249 + const lastDataEvent = envelopes[199] + expect(lastDataEvent.data.baseData.properties.command_name).toBe("cmd-249") + + // The overflow error event should be the last one + const overflowEvent = envelopes[200] + expect(overflowEvent.data.baseData.name).toBe("error") + expect(overflowEvent.data.baseData.properties.error_name).toBe("TelemetryBufferOverflow") + expect(overflowEvent.data.baseData.properties.error_message).toContain("50 events dropped") + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("droppedEvents counter increments correctly", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchBodies: string[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (_input: any, init: any) => { + fetchBodies.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + // Exactly 205 events — 5 should be dropped + for (let i = 0; i < 205; i++) { + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + } + + await Telemetry.flush() + + const envelopes = JSON.parse(fetchBodies[0]) + const overflowEvent = envelopes.find( + (e: any) => e.data?.baseData?.properties?.error_name === "TelemetryBufferOverflow", + ) + expect(overflowEvent).toBeDefined() + expect(overflowEvent.data.baseData.properties.error_message).toContain("5 events dropped") + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +}) + +// --------------------------------------------------------------------------- +// 13. init with enabled telemetry +// --------------------------------------------------------------------------- +describe("telemetry.init with enabled telemetry", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("init() with telemetry enabled sets up the flush timer", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + await Telemetry.init() + + // If flush timer is set up, tracking + waiting should eventually trigger flush + // We verify indirectly: after init, flush works (meaning enabled=true, appInsights is set) + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + expect(fetchMock).toHaveBeenCalledTimes(1) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("init() parses the connection string and configures App Insights", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const fetchCalls: { url: string; body: string }[] = [] + + const fetchMock = spyOn(global, "fetch").mockImplementation(async (input: any, init: any) => { + fetchCalls.push({ url: String(input), body: String(init?.body ?? "") }) + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=custom-key-abc;IngestionEndpoint=https://custom.endpoint.azure.com" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + + expect(fetchCalls.length).toBe(1) + expect(fetchCalls[0].url).toBe("https://custom.endpoint.azure.com/v2/track") + const body = JSON.parse(fetchCalls[0].body) + expect(body[0].iKey).toBe("custom-key-abc") + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) + + test("init() with ALTIMATE_TELEMETRY_DISABLED=true disables telemetry completely", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + await Telemetry.init() + + Telemetry.track({ + type: "session_start", + timestamp: Date.now(), + session_id: "s1", + model_id: "m", + provider_id: "p", + agent: "a", + project_id: "proj", + }) + + await Telemetry.flush() + expect(fetchMock).not.toHaveBeenCalled() + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + fetchMock.mockRestore() + } + }) + + test("init() is idempotent — returns the same promise on concurrent calls", async () => { + const origEnv = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + + const fetchMock = spyOn(global, "fetch").mockImplementation(async () => { + return new Response("", { status: 200 }) + }) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://e.com" + + const p1 = Telemetry.init() + const p2 = Telemetry.init() + expect(p1).toBe(p2) // Same promise object + await p1 + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origEnv + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +})