Merge pull request #97 from AxmeAI/feat/telemetry-client-20260410

feat(telemetry): anonymous Phase 1 + Phase 2 client

Author: George-iam

README.md

@@ -284,6 +284,34 @@ Additional presets available: `production-ready`, `team-collaboration`.
 
 ---
 
+## Telemetry
+
+axme-code sends anonymous usage telemetry to help us improve the product. We collect:
+
+- **Lifecycle events**: install, startup, version update
+- **Product health events**: setup completion, audit completion (counts of memories/decisions/safety extracted, duration, cost, error class)
+- **Errors**: category and bounded error class for failures in audit, setup, hooks, and auto-update
+
+What we **never** send:
+- Hostnames, usernames, file paths, working directories
+- Source code, transcripts, decisions, memories, or any project content
+- IP addresses (stripped at the server)
+- Raw exception messages (we map to a small set of error classes)
+
+Each install gets a random 64-character machine ID stored in `~/.local/share/axme-code/machine-id`. The ID is not derived from hardware and cannot be linked back to you.
+
+**To disable telemetry**, set either of these environment variables:
+
+```bash
+export AXME_TELEMETRY_DISABLED=1
+# or the industry-standard:
+export DO_NOT_TRACK=1
+```
+
+When disabled, no network requests are made and no machine ID is generated.
+
+---
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.

docs/TELEMETRY_TZ.md

@@ -46,6 +46,8 @@ export interface SessionAuditResult {
   chunks?: number;
   /** Estimated prompt tokens for observability. */
   promptTokens?: number;
+  /** Number of extraction blocks the parser dropped due to missing required fields. Used for telemetry. */
+  droppedCount?: number;
 }
 
 /**
@@ -537,6 +539,7 @@ export async function runSessionAudit(opts: {
   let totalCostUsd = 0;
   let totalCostCached: CostInfo | undefined;
   let totalPromptChars = 0;
+  let totalDroppedCount = 0;
 
   for (let i = 0; i < chunks.length; i++) {
     const chunkBlock = chunks[i];
@@ -561,6 +564,7 @@ export async function runSessionAudit(opts: {
     });
 
     totalPromptChars += chunkResult.promptChars;
+    totalDroppedCount += chunkResult.droppedCount ?? 0;
     if (chunkResult.cost) {
       totalCostCached = chunkResult.cost;
       totalCostUsd += chunkResult.cost.costUsd ?? 0;
@@ -594,6 +598,7 @@ export async function runSessionAudit(opts: {
     durationMs: Date.now() - startTime,
     chunks: chunks.length,
     promptTokens: Math.round(totalPromptChars / 4),
+    droppedCount: totalDroppedCount,
   };
 }
 
@@ -639,7 +644,7 @@ async function runSingleAuditCall(opts: {
     // auto-loading the project's .claude/settings.json, but users or CI may
     // register hooks via environment or other means, so the belt-and-braces
     // env check in every hook handler is what actually stops the recursion.
-    env: { ...process.env, AXME_SKIP_HOOKS: "1" },
+    env: { ...process.env, AXME_SKIP_HOOKS: "1", AXME_TELEMETRY_DISABLED: "1" },
   };
 
   const isMultiChunk = opts.totalChunks > 1;
@@ -891,7 +896,7 @@ ${freeTextAnalysis}`;
       "Read", "Grep", "Glob", "Write", "Edit", "NotebookEdit", "Agent",
       "Skill", "TodoWrite", "WebFetch", "WebSearch", "Bash", "ToolSearch",
     ],
-    env: { ...process.env, AXME_SKIP_HOOKS: "1" },
+    env: { ...process.env, AXME_SKIP_HOOKS: "1", AXME_TELEMETRY_DISABLED: "1" },
   };
 
   const q = sdk.query({ prompt: formatPrompt, options: queryOpts });
@@ -925,10 +930,11 @@ ${freeTextAnalysis}`;
  */
 export function parseAuditOutput(output: string | object, sessionId: string): Omit<SessionAuditResult, "cost" | "durationMs"> {
   const today = new Date().toISOString().slice(0, 10);
+  let droppedCount = 0;
   const json = typeof output === "object" ? output : extractJson(output);
   if (!json) {
     process.stderr.write(`AXME auditor: failed to extract JSON from output (${typeof output === "string" ? output.length : 0} chars). First 300: ${typeof output === "string" ? output.slice(0, 300) : JSON.stringify(output).slice(0, 300)}\n`);
-    return { memories: [], decisions: [], safetyRules: [], oracleNeedsRescan: false, questions: [], handoff: null, sessionSummary: null };
+    return { memories: [], decisions: [], safetyRules: [], oracleNeedsRescan: false, questions: [], handoff: null, sessionSummary: null, droppedCount: 0 };
   }
 
   // Parse memories
@@ -945,11 +951,11 @@ export function parseAuditOutput(output: string | object, sessionId: string): Om
       const fieldName = m.body ? "body" : m.summary ? "summary" : "description";
       process.stderr.write(`AXME auditor: memory title recovered from ${fieldName}: ${title.slice(0, 80)}\n`);
     }
-    if (!title) { process.stderr.write(`AXME auditor: memory dropped (no usable content): ${JSON.stringify(m).slice(0, 200)}\n`); continue; }
+    if (!title) { droppedCount++; process.stderr.write(`AXME auditor: memory dropped (no usable content): ${JSON.stringify(m).slice(0, 200)}\n`); continue; }
     const type = m.type;
-    if (type !== "feedback" && type !== "pattern") { process.stderr.write(`AXME auditor: memory "${title.slice(0, 60)}" dropped (invalid type: ${type})\n`); continue; }
+    if (type !== "feedback" && type !== "pattern") { droppedCount++; process.stderr.write(`AXME auditor: memory "${title.slice(0, 60)}" dropped (invalid type: ${type})\n`); continue; }
     const slug = toMemorySlug(m.slug || title);
-    if (!slug) { process.stderr.write(`AXME auditor: memory "${title.slice(0, 60)}" dropped (could not generate slug)\n`); continue; }
+    if (!slug) { droppedCount++; process.stderr.write(`AXME auditor: memory "${title.slice(0, 60)}" dropped (could not generate slug)\n`); continue; }
     const scope = parseScopeField(m.scope);
     memories.push({
       slug, type, title,
@@ -965,10 +971,11 @@ export function parseAuditOutput(output: string | object, sessionId: string): Om
   const decisions: Omit<Decision, "id">[] = [];
   for (const d of (Array.isArray(json.decisions) ? json.decisions : [])) {
     const title = d.title;
-    if (!title) { process.stderr.write(`AXME auditor: decision dropped (no title): ${JSON.stringify(d).slice(0, 200)}\n`); continue; }
+    if (!title) { droppedCount++; process.stderr.write(`AXME auditor: decision dropped (no title): ${JSON.stringify(d).slice(0, 200)}\n`); continue; }
     // Fallback: if decision body is missing, try reasoning or use title
     let decision = d.decision || d.reasoning || "";
     if (!decision) {
+      droppedCount++;
       process.stderr.write(`AXME auditor: decision "${title}" dropped (no decision or reasoning field)\n`);
       continue;
     }
@@ -996,8 +1003,8 @@ export function parseAuditOutput(output: string | object, sessionId: string): Om
   for (const s of (Array.isArray(json.safety) ? json.safety : [])) {
     const ruleType = s.rule_type;
     const value = s.value;
-    if (!ruleType) { process.stderr.write(`AXME auditor: safety dropped (no rule_type): ${JSON.stringify(s).slice(0, 200)}\n`); continue; }
-    if (!value) { process.stderr.write(`AXME auditor: safety dropped (no value, rule_type=${ruleType})\n`); continue; }
+    if (!ruleType) { droppedCount++; process.stderr.write(`AXME auditor: safety dropped (no rule_type): ${JSON.stringify(s).slice(0, 200)}\n`); continue; }
+    if (!value) { droppedCount++; process.stderr.write(`AXME auditor: safety dropped (no value, rule_type=${ruleType})\n`); continue; }
     const scope = parseScopeField(s.scope);
     safetyRules.push({ ruleType, value, ...(scope ? { scope } : {}) });
   }
@@ -1046,7 +1053,7 @@ export function parseAuditOutput(output: string | object, sessionId: string): Om
   const sessionSummary = json.session_summary && typeof json.session_summary === "string" && json.session_summary.trim().length > 10
     ? json.session_summary.trim() : null;
 
-  return { memories, decisions, safetyRules, oracleNeedsRescan, questions, handoff, sessionSummary };
+  return { memories, decisions, safetyRules, oracleNeedsRescan, questions, handoff, sessionSummary, droppedCount };
 }
 
 /**

src/agents/session-auditor.ts

@@ -217,7 +217,12 @@ export async function backgroundAutoUpdate(): Promise<void> {
         updated: false,
       });
     }
-  } catch {
-    // Never crash the server due to update logic
+  } catch (err) {
+    // Never crash the server due to update logic.
+    // Report to telemetry so we can see auto-update failures in aggregate.
+    try {
+      const { reportError, classifyError } = await import("./telemetry.js");
+      reportError("auto_update", classifyError(err), false);
+    } catch { /* swallow */ }
   }
 }

src/auto-update.ts

@@ -266,15 +266,59 @@ After setup, run 'claude' as usual. AXME tools are available automatically.`);
 }
 
 async function main() {
+  // Send anonymous startup telemetry only for user-facing CLI commands.
+  // Hook and audit-session subcommands run as short-lived subprocesses
+  // (Claude Code hooks fire many times per session, audit-session is a
+  // detached background worker), so firing telemetry per invocation would
+  // spam the endpoint and skew startup counts. The `serve` command sends
+  // its own startup event from server.ts after MCP server is up.
+  // We AWAIT this so events flush before heavy work begins — under event
+  // loop pressure (LLM scanners), fire-and-forget setImmediate may stall.
+  const startupCommands = new Set(["setup", "status", "stats", "audit-kb", "cleanup", "help"]);
+  if (command && startupCommands.has(command)) {
+    const { sendStartupEvents } = await import("./telemetry.js");
+    await sendStartupEvents();
+  }
+
   switch (command) {
     case "setup": {
+      const setupStartMs = Date.now();
       const forceSetup = args.includes("--force");
       const pluginMode = args.includes("--plugin") || !!process.env.CLAUDE_PLUGIN_ROOT;
       const setupArgs = args.filter(a => a !== "--force" && a !== "--plugin");
       const projectPath = resolve(setupArgs[1] || ".");
       const hasGitDir = existsSync(join(projectPath, ".git"));
       const ws = detectWorkspace(projectPath);
       const isWorkspace = hasGitDir ? false : ws.type !== "single";
+      const childRepos = isWorkspace
+        ? ws.projects.filter(p => existsSync(join(projectPath, p.path, ".git"))).length
+        : 0;
+      // Telemetry-relevant fields, populated as setup progresses
+      let setupOutcome: "success" | "fallback" | "failed" = "failed";
+      let setupMethod: "llm" | "deterministic" = "deterministic";
+      let setupPhaseFailed: string | null = null;
+      let setupPresetsApplied = 0;
+      let setupScannersRun = 0;
+      let setupScannersFailed = 0;
+      // Use the blocking variant so the event lands BEFORE process.exit() runs.
+      // The fire-and-forget sendTelemetry uses setImmediate, which is killed
+      // by process.exit() before the network request is even started.
+      const sendSetupTelemetry = async () => {
+        try {
+          const { sendTelemetryBlocking } = await import("./telemetry.js");
+          await sendTelemetryBlocking("setup_complete", {
+            outcome: setupOutcome,
+            duration_ms: Date.now() - setupStartMs,
+            method: setupMethod,
+            scanners_run: setupScannersRun,
+            scanners_failed: setupScannersFailed,
+            phase_failed: setupPhaseFailed,
+            presets_applied: setupPresetsApplied,
+            is_workspace: isWorkspace,
+            child_repos: childRepos,
+          });
+        } catch { /* swallow */ }
+      };
 
       if (isWorkspace) {
         console.log(`Initializing AXME Code workspace in ${projectPath} (${ws.type}, ${ws.projects.length} projects)...`);
@@ -289,36 +333,60 @@ async function main() {
         console.error(`To authenticate, run one of:`);
         console.error(`  claude login              (Claude subscription)`);
         console.error(`  export ANTHROPIC_API_KEY=sk-ant-...  (API key)\n`);
+        setupOutcome = "failed";
+        setupPhaseFailed = "auth_check";
+        await sendSetupTelemetry();
         process.exit(1);
       }
 
       // Init with LLM scanners (parallel)
-      if (isWorkspace) {
-        const { workspaceResult, projectResults } = await initWorkspaceWithLLM(projectPath, { onProgress: console.log });
-        const totalCost = workspaceResult.cost.costUsd + projectResults.reduce((s, r) => s + r.cost.costUsd, 0);
-        console.log(`  Workspace: ${workspaceResult.decisions.count} decisions, ${workspaceResult.memories.count} memories`);
-        for (const r of projectResults) {
-          const name = r.projectPath.split("/").pop();
-          console.log(`  ${name}: ${r.decisions.count} decisions (${r.decisions.fromScan} LLM + ${r.decisions.fromPresets} presets)`);
-        }
-        if (totalCost > 0) console.log(`  Total cost: $${totalCost.toFixed(2)}`);
-        for (const e of [...workspaceResult.errors, ...projectResults.flatMap(r => r.errors)]) {
-          console.log(`  Warning: ${e}`);
-        }
-        generateWorkspaceYaml(projectPath, ws);
-      } else {
-        const result = await initProjectWithLLM(projectPath, { onProgress: console.log, force: forceSetup });
-        if (!result.created && result.durationMs === 0) {
-          console.log(`  Already initialized (skipped LLM scan). Use --force to re-scan.`);
-          console.log(`  Decisions: ${result.decisions.count}, Memories: ${result.memories.count}`);
+      try {
+        if (isWorkspace) {
+          const { workspaceResult, projectResults } = await initWorkspaceWithLLM(projectPath, { onProgress: console.log });
+          const totalCost = workspaceResult.cost.costUsd + projectResults.reduce((s, r) => s + r.cost.costUsd, 0);
+          console.log(`  Workspace: ${workspaceResult.decisions.count} decisions, ${workspaceResult.memories.count} memories`);
+          for (const r of projectResults) {
+            const name = r.projectPath.split("/").pop();
+            console.log(`  ${name}: ${r.decisions.count} decisions (${r.decisions.fromScan} LLM + ${r.decisions.fromPresets} presets)`);
+          }
+          if (totalCost > 0) console.log(`  Total cost: $${totalCost.toFixed(2)}`);
+          for (const e of [...workspaceResult.errors, ...projectResults.flatMap(r => r.errors)]) {
+            console.log(`  Warning: ${e}`);
+          }
+          generateWorkspaceYaml(projectPath, ws);
+          // Track telemetry: any LLM scan in any repo means LLM method
+          const anyLlm = projectResults.some(r => r.oracle.llm) || workspaceResult.decisions.fromScan > 0;
+          setupMethod = anyLlm ? "llm" : "deterministic";
+          setupPresetsApplied = projectResults.reduce((s, r) => s + (r.decisions.fromPresets || 0), 0);
+          // Sum scanner counts across workspace + all projects
+          setupScannersRun = workspaceResult.scannersRun + projectResults.reduce((s, r) => s + r.scannersRun, 0);
+          setupScannersFailed = workspaceResult.scannersFailed + projectResults.reduce((s, r) => s + r.scannersFailed, 0);
         } else {
-          console.log(`  Oracle: ${result.oracle.files} files (${result.oracle.llm ? "LLM scan" : "deterministic fallback"})`);
-          console.log(`  Decisions: ${result.decisions.count} (${result.decisions.fromScan} LLM + ${result.decisions.fromPresets} presets)`);
-          console.log(`  Memories: ${result.memories.count} (${result.memories.fromPresets} from presets)`);
-          console.log(`  Safety: ${result.safety.llm ? "LLM scan" : "defaults + presets"}`);
-          if (result.cost.costUsd > 0) console.log(`  Cost: $${result.cost.costUsd.toFixed(2)}, ${(result.durationMs / 1000).toFixed(1)}s`);
-          for (const e of result.errors) console.log(`  Warning: ${e}`);
+          const result = await initProjectWithLLM(projectPath, { onProgress: console.log, force: forceSetup });
+          if (!result.created && result.durationMs === 0) {
+            console.log(`  Already initialized (skipped LLM scan). Use --force to re-scan.`);
+            console.log(`  Decisions: ${result.decisions.count}, Memories: ${result.memories.count}`);
+          } else {
+            console.log(`  Oracle: ${result.oracle.files} files (${result.oracle.llm ? "LLM scan" : "deterministic fallback"})`);
+            console.log(`  Decisions: ${result.decisions.count} (${result.decisions.fromScan} LLM + ${result.decisions.fromPresets} presets)`);
+            console.log(`  Memories: ${result.memories.count} (${result.memories.fromPresets} from presets)`);
+            console.log(`  Safety: ${result.safety.llm ? "LLM scan" : "defaults + presets"}`);
+            if (result.cost.costUsd > 0) console.log(`  Cost: $${result.cost.costUsd.toFixed(2)}, ${(result.durationMs / 1000).toFixed(1)}s`);
+            for (const e of result.errors) console.log(`  Warning: ${e}`);
+          }
+          // Track telemetry: oracle.llm tells us whether LLM path was used
+          setupMethod = result.oracle.llm ? "llm" : "deterministic";
+          setupPresetsApplied = (result.decisions.fromPresets || 0);
+          setupScannersRun = result.scannersRun;
+          setupScannersFailed = result.scannersFailed;
         }
+      } catch (err) {
+        setupOutcome = "failed";
+        setupPhaseFailed = "init_scan";
+        const { classifyError, reportError } = await import("./telemetry.js");
+        try { reportError("setup", classifyError(err), true); } catch { /* swallow */ }
+        await sendSetupTelemetry();
+        throw err;
       }
 
       // Detect plugin context — skip .mcp.json and hooks if running from plugin
@@ -378,6 +446,11 @@ async function main() {
         : 0;
       writeBootstrapToAxmeMemory(projectPath, isWorkspace, repoCount);
 
+      // Setup completed. setupMethod was set above by the init scan.
+      // outcome=success when LLM ran end-to-end, fallback when deterministic was used.
+      setupOutcome = setupMethod === "llm" ? "success" : "fallback";
+      await sendSetupTelemetry();
+
       console.log("\nDone! Run 'claude' to start using AXME tools.");
       break;
     }

src/cli.ts

@@ -68,7 +68,13 @@ export async function runPostToolUseHook(workspacePath?: string): Promise<void>
     for await (const chunk of process.stdin) chunks.push(chunk);
     const input = JSON.parse(Buffer.concat(chunks).toString("utf-8")) as HookInput;
     handlePostToolUse(workspacePath, input);
-  } catch {
-    // Hook failures must be silent
+  } catch (err) {
+    // Hook failures must be silent — but reported to telemetry for visibility.
+    // Use blocking send: hook subprocess exits ms after this catch and would
+    // kill any setImmediate-queued network call.
+    try {
+      const { sendTelemetryBlocking, classifyError } = await import("../telemetry.js");
+      await sendTelemetryBlocking("error", { category: "hook", error_class: classifyError(err), fatal: false });
+    } catch { /* swallow */ }
   }
 }

src/hooks/post-tool-use.ts

@@ -219,7 +219,13 @@ export async function runPreToolUseHook(workspacePath?: string): Promise<void> {
     for await (const chunk of process.stdin) chunks.push(chunk);
     const input = JSON.parse(Buffer.concat(chunks).toString("utf-8")) as HookInput;
     handlePreToolUse(workspacePath, input);
-  } catch {
-    // Hook failures must be silent - fail open for safety
+  } catch (err) {
+    // Hook failures must be silent - fail open for safety.
+    // Reported to telemetry via blocking send so the network call lands
+    // before this short-lived hook subprocess exits.
+    try {
+      const { sendTelemetryBlocking, classifyError } = await import("../telemetry.js");
+      await sendTelemetryBlocking("error", { category: "hook", error_class: classifyError(err), fatal: false });
+    } catch { /* swallow */ }
   }
 }

src/hooks/pre-tool-use.ts

@@ -87,7 +87,12 @@ export async function runSessionEndHook(workspacePath?: string): Promise<void> {
       // Empty/invalid stdin is fine — we'll proceed without transcript attachment
     }
     handleSessionEnd(workspacePath, input);
-  } catch {
-    // Hook failures must be silent
+  } catch (err) {
+    // Hook failures must be silent — but reported to telemetry for visibility.
+    // Use blocking send: hook subprocess exits ms after this catch.
+    try {
+      const { sendTelemetryBlocking, classifyError } = await import("../telemetry.js");
+      await sendTelemetryBlocking("error", { category: "hook", error_class: classifyError(err), fatal: false });
+    } catch { /* swallow */ }
   }
 }