diff --git a/src/lib/agent/runner/harness/pi/__tests__/security.test.ts b/src/lib/agent/runner/harness/pi/__tests__/security.test.ts index 9c86f959..0dcce5c6 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/security.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/security.test.ts @@ -6,10 +6,16 @@ import { evaluateToolCall, createSecurityExtension, isScopedFileRemoval, + observeTransportLeak, overwriteShrinkReason, MAX_TOOL_CALLS, type PiExtensionApiLike, } from '../security'; +import { analytics } from '@utils/analytics'; + +vi.mock('@utils/analytics', () => ({ + analytics: { wizardCapture: vi.fn() }, +})); // @posthog/warlock resolves to __mocks__/@posthog/warlock.ts (ESM + WASM can't // load under the CJS test runner). Default: scan matches nothing; tests @@ -687,3 +693,45 @@ describe('pi-security: overwrite shrink guard (destructive whole-file rewrite)', expect(decision.block).toBe(false); }); }); + +describe('observeTransportLeak (passive telemetry)', () => { + // The exact garbage run 9704a73e wrote into a customer's global-error.tsx. + const LEAKED_LINE = + "' }#+#+#+#+.functions.complete_task (commentary json.functions.complete_taskjson>tagger历山大发 { "; + + it('reports which pattern fired and where — never any matched text', () => { + observeTransportLeak('Write', LEAKED_LINE); + const call = vi + .mocked(analytics.wizardCapture) + .mock.calls.find(([e]) => e === 'file content leak observed'); + expect(call?.[1]).toMatchObject({ + tool: 'Write', + leak: 'leaked tool-call tokens', + leak_offset: LEAKED_LINE.indexOf('functions.'), + content_length: LEAKED_LINE.length, + }); + expect(JSON.stringify(call?.[1])).not.toContain('complete_task'); + }); + + it('flags control characters by pattern only', () => { + vi.mocked(analytics.wizardCapture).mockClear(); + observeTransportLeak('Edit', 'trailing\x7f'); + const call = vi + .mocked(analytics.wizardCapture) + .mock.calls.find(([e]) => e === 'file content leak observed'); + expect(call?.[1]).toMatchObject({ leak: 'control characters' }); + }); + + it('stays silent on real source, including Firebase functions.* code', () => { + vi.mocked(analytics.wizardCapture).mockClear(); + observeTransportLeak( + 'Write', + 'const f = functions.https.onRequest(app);\nline\n\ttabbed\r\n', + ); + expect( + vi + .mocked(analytics.wizardCapture) + .mock.calls.some(([e]) => e === 'file content leak observed'), + ).toBe(false); + }); +}); diff --git a/src/lib/agent/runner/harness/pi/security.ts b/src/lib/agent/runner/harness/pi/security.ts index 9c550466..8e87b170 100644 --- a/src/lib/agent/runner/harness/pi/security.ts +++ b/src/lib/agent/runner/harness/pi/security.ts @@ -31,6 +31,7 @@ import { } from '@lib/yara-hooks'; import { scanVerdict, type ScanContext } from '@lib/yara-policy'; import { logToFile } from '@utils/debug'; +import { analytics } from '@utils/analytics'; /** warlock ScanMatch → the report shape `recordExternalScan` expects. */ function toReportViolation(m: ScanMatch) { @@ -45,6 +46,50 @@ function toReportViolation(m: ScanMatch) { /** Runaway backstop: hard cap on tool calls per (sub)agent session. */ export const MAX_TOOL_CALLS = 250; +/** + * Passive telemetry for an unfixed upstream failure: gpt-5.x streaming with + * tools leaks its function-call grammar into string values, and the leak lands + * on disk (run 9704a73e shipped `…functions.complete_task (commentary…` plus a + * DEL byte inside a customer's global-error.tsx). Observe-only — never blocks. + * Prior art: https://github.com/BerriAI/litellm/issues/14260 (closed stale), + * https://community.openai.com/t/1386422 (gpt-5.6-luna, no fix). + */ +const TRANSPORT_LEAK_PATTERNS: readonly { pattern: RegExp; label: string }[] = [ + // The litellm#14260 signature, scoped to the wizard's own tool names — a + // generic `functions.*` would false-positive on real code (Firebase). + { + pattern: /functions\.(?:complete_task|enqueue_task|read_handoffs)/, + label: 'leaked tool-call tokens', + }, + { + pattern: /<\|(?:channel|constrain|message|call|end|start|return)\|>/, + label: 'leaked channel markers', + }, + // C0 controls and DEL minus tab/newline/CR — never valid in source text. + // eslint-disable-next-line no-control-regex + { pattern: /[\0-\x08\x0B\f\x0E-\x1F\x7F]/, label: 'control characters' }, +]; + +/** Capture one event per leaking write/edit: which pattern fired and where it sat — no matched text, ever. */ +export function observeTransportLeak(tool: string, content: string): void { + for (const { pattern, label } of TRANSPORT_LEAK_PATTERNS) { + const match = pattern.exec(content); + if (!match) continue; + analytics.wizardCapture('file content leak observed', { + tool, + leak: label, + leak_offset: match.index, + content_length: content.length, + // End-of-string leaks are the upstream decoder signature. + at_end: content.length - match.index < 80, + }); + logToFile( + `[transport-leak] observed ${tool}: ${label} at ${match.index}/${content.length}`, + ); + return; + } +} + export interface ToolGateContext { disallowedTools?: readonly string[]; /** True while a wizard_ask overlay is open (interactive); blocks Write/Edit. */ @@ -270,6 +315,7 @@ async function preExecutionYaraBlock( return undefined; } if (!content) return undefined; + if (ctx === 'output') observeTransportLeak(tool, content); let matches = await scanAndTriage(content, ctx, triage); if (ctx === 'output' && isWizardDocumentationPath(str(input.path))) {