feat: add telemetry to TUI add paths via withAddTelemetry

Hweinstock · Hweinstock · commit a290c00ce201 · 2026-04-30T13:04:11.000Z
diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts
@@ -1,4 +1,5 @@
 import { getErrorMessage } from '../errors';
+import type { AddResult } from '../primitives/types.js';
 import { TelemetryClientAccessor } from './client-accessor.js';
 import type { Command, CommandAttrs } from './schemas/command-run.js';
 
@@ -35,3 +36,32 @@ export async function cliCommandRun<C extends Command>(
     process.exit(1);
   }
 }
+
+/**
+ * Wrap a primitive .add() call with telemetry — used by TUI paths.
+ * CLI paths use {@link cliCommandRun} instead.
+ */
+export async function withAddTelemetry<C extends Command, T extends Record<string, unknown>>(
+  command: C,
+  attrs: CommandAttrs<C>,
+  fn: () => Promise<AddResult<T>>
+): Promise<AddResult<T>> {
+  let client;
+  try {
+    client = await TelemetryClientAccessor.get();
+  } catch {
+    return fn();
+  }
+
+  let result: AddResult<T> | undefined;
+  try {
+    await client.withCommandRun(command, async () => {
+      result = await fn();
+      if (!result.success) throw new Error(result.error);
+      return attrs;
+    });
+  } catch {
+    if (!result) return fn();
+  }
+  return result!;
+}
diff --git a/src/cli/tui/hooks/useCreateEvaluator.ts b/src/cli/tui/hooks/useCreateEvaluator.ts
@@ -1,5 +1,7 @@
 import type { EvaluatorConfig } from '../../../schema';
 import { evaluatorPrimitive } from '../../primitives/registry';
+import { withAddTelemetry } from '../../telemetry/cli-command-run.js';
+import { Level, standardize } from '../../telemetry/schemas/common-shapes.js';
 import { useCallback, useEffect, useState } from 'react';
 
 interface CreateEvaluatorConfig {
@@ -16,11 +18,19 @@ export function useCreateEvaluator() {
   const create = useCallback(async (config: CreateEvaluatorConfig) => {
     setStatus({ state: 'loading' });
     try {
-      const addResult = await evaluatorPrimitive.add({
-        name: config.name,
-        level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
-        config: config.config,
-      });
+      const addResult = await withAddTelemetry(
+        'add.evaluator',
+        {
+          evaluator_type: config.config.codeBased ? 'code-based' : 'llm-as-a-judge',
+          level: standardize(Level, config.level),
+        },
+        () =>
+          evaluatorPrimitive.add({
+            name: config.name,
+            level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
+            config: config.config,
+          })
+      );
       if (!addResult.success) {
         throw new Error(addResult.error ?? 'Failed to create evaluator');
       }
diff --git a/src/cli/tui/hooks/useCreateMcp.ts b/src/cli/tui/hooks/useCreateMcp.ts
@@ -4,6 +4,8 @@ import {
   gatewayTargetPrimitive,
   policyEnginePrimitive,
 } from '../../primitives/registry';
+import { withAddTelemetry } from '../../telemetry/cli-command-run.js';
+import { AuthorizerType, PolicyEngineMode, standardize } from '../../telemetry/schemas/common-shapes.js';
 import type { AddGatewayConfig } from '../screens/mcp/types';
 import { useCallback, useEffect, useState } from 'react';
 
@@ -23,22 +25,33 @@ export function useCreateGateway() {
   const createGateway = useCallback(async (config: AddGatewayConfig) => {
     setStatus({ state: 'loading' });
     try {
-      const addResult = await gatewayPrimitive.add({
-        name: config.name,
-        description: config.description,
-        authorizerType: config.authorizerType,
-        discoveryUrl: config.jwtConfig?.discoveryUrl,
-        allowedAudience: config.jwtConfig?.allowedAudience?.join(','),
-        allowedClients: config.jwtConfig?.allowedClients?.join(','),
-        allowedScopes: config.jwtConfig?.allowedScopes?.join(','),
-        customClaims: config.jwtConfig?.customClaims,
-        clientId: config.jwtConfig?.clientId,
-        clientSecret: config.jwtConfig?.clientSecret,
-        enableSemanticSearch: config.enableSemanticSearch,
-        exceptionLevel: config.exceptionLevel,
-        policyEngine: config.policyEngineConfiguration?.policyEngineName,
-        policyEngineMode: config.policyEngineConfiguration?.mode,
-      });
+      const addResult = await withAddTelemetry(
+        'add.gateway',
+        {
+          authorizer_type: standardize(AuthorizerType, config.authorizerType ?? 'NONE'),
+          has_policy_engine: !!config.policyEngineConfiguration?.policyEngineName,
+          policy_engine_mode: standardize(PolicyEngineMode, config.policyEngineConfiguration?.mode ?? 'log_only'),
+          semantic_search: config.enableSemanticSearch !== false,
+          runtime_count: 0,
+        },
+        () =>
+          gatewayPrimitive.add({
+            name: config.name,
+            description: config.description,
+            authorizerType: config.authorizerType,
+            discoveryUrl: config.jwtConfig?.discoveryUrl,
+            allowedAudience: config.jwtConfig?.allowedAudience?.join(','),
+            allowedClients: config.jwtConfig?.allowedClients?.join(','),
+            allowedScopes: config.jwtConfig?.allowedScopes?.join(','),
+            customClaims: config.jwtConfig?.customClaims,
+            clientId: config.jwtConfig?.clientId,
+            clientSecret: config.jwtConfig?.clientSecret,
+            enableSemanticSearch: config.enableSemanticSearch,
+            exceptionLevel: config.exceptionLevel,
+            policyEngine: config.policyEngineConfiguration?.policyEngineName,
+            policyEngineMode: config.policyEngineConfiguration?.mode,
+          })
+      );
       if (!addResult.success) {
         throw new Error(addResult.error ?? 'Failed to create gateway');
       }
diff --git a/src/cli/tui/hooks/useCreateMemory.ts b/src/cli/tui/hooks/useCreateMemory.ts
@@ -2,6 +2,7 @@ import { ConfigIO } from '../../../lib';
 import type { Memory } from '../../../schema';
 import { getAvailableAgents } from '../../operations/attach';
 import { memoryPrimitive } from '../../primitives/registry';
+import { withAddTelemetry } from '../../telemetry/cli-command-run.js';
 import { useCallback, useEffect, useState } from 'react';
 
 interface CreateMemoryConfig {
@@ -24,13 +25,25 @@ export function useCreateMemory() {
     setStatus({ state: 'loading' });
     try {
       const strategiesStr = config.strategies.map(s => s.type).join(',');
-      const addResult = await memoryPrimitive.add({
-        name: config.name,
-        expiry: config.eventExpiryDuration,
-        strategies: strategiesStr || undefined,
-        dataStreamArn: config.streaming?.dataStreamArn,
-        contentLevel: config.streaming?.contentLevel,
-      });
+      const strategyList = strategiesStr ? strategiesStr.split(',').map(s => s.trim().toUpperCase()) : [];
+      const addResult = await withAddTelemetry(
+        'add.memory',
+        {
+          strategy_count: strategyList.length,
+          strategy_semantic: strategyList.includes('SEMANTIC'),
+          strategy_summarization: strategyList.includes('SUMMARIZATION'),
+          strategy_user_preference: strategyList.includes('USER_PREFERENCE'),
+          strategy_episodic: strategyList.includes('EPISODIC'),
+        },
+        () =>
+          memoryPrimitive.add({
+            name: config.name,
+            expiry: config.eventExpiryDuration,
+            strategies: strategiesStr || undefined,
+            dataStreamArn: config.streaming?.dataStreamArn,
+            contentLevel: config.streaming?.contentLevel,
+          })
+      );
       if (!addResult.success) {
         throw new Error(addResult.error ?? 'Failed to create memory');
       }
diff --git a/src/cli/tui/hooks/useCreateOnlineEval.ts b/src/cli/tui/hooks/useCreateOnlineEval.ts
@@ -1,4 +1,5 @@
 import { onlineEvalConfigPrimitive } from '../../primitives/registry';
+import { withAddTelemetry } from '../../telemetry/cli-command-run.js';
 import { useCallback, useEffect, useState } from 'react';
 
 interface CreateOnlineEvalConfig {
@@ -17,13 +18,21 @@ export function useCreateOnlineEval() {
   const create = useCallback(async (config: CreateOnlineEvalConfig) => {
     setStatus({ state: 'loading' });
     try {
-      const addResult = await onlineEvalConfigPrimitive.add({
-        name: config.name,
-        agent: config.agent,
-        evaluators: config.evaluators,
-        samplingRate: config.samplingRate,
-        enableOnCreate: config.enableOnCreate,
-      });
+      const addResult = await withAddTelemetry(
+        'add.online-eval',
+        {
+          evaluator_count: config.evaluators.length,
+          enable_on_create: config.enableOnCreate ?? false,
+        },
+        () =>
+          onlineEvalConfigPrimitive.add({
+            name: config.name,
+            agent: config.agent,
+            evaluators: config.evaluators,
+            samplingRate: config.samplingRate,
+            enableOnCreate: config.enableOnCreate,
+          })
+      );
       if (!addResult.success) {
         throw new Error(addResult.error ?? 'Failed to create online eval config');
       }
diff --git a/src/cli/tui/screens/identity/useCreateIdentity.ts b/src/cli/tui/screens/identity/useCreateIdentity.ts
@@ -2,6 +2,7 @@ import { ConfigIO } from '../../../../lib';
 import type { Credential } from '../../../../schema';
 import type { AddCredentialOptions } from '../../../primitives/CredentialPrimitive';
 import { credentialPrimitive } from '../../../primitives/registry';
+import { withAddTelemetry } from '../../../telemetry/cli-command-run.js';
 import { useCallback, useEffect, useState } from 'react';
 
 interface CreateStatus<T> {
@@ -16,7 +17,13 @@ export function useCreateIdentity() {
   const create = useCallback(async (config: AddCredentialOptions) => {
     setStatus({ state: 'loading' });
     try {
-      const result = await credentialPrimitive.add(config);
+      const result = await withAddTelemetry(
+        'add.credential',
+        {
+          credential_type: config.authorizerType === 'OAuthCredentialProvider' ? 'oauth' : 'api-key',
+        },
+        () => credentialPrimitive.add(config)
+      );
       if (!result.success) {
         throw new Error(result.error ?? 'Failed to create credential');
       }
diff --git a/src/cli/tui/screens/policy/AddPolicyFlow.tsx b/src/cli/tui/screens/policy/AddPolicyFlow.tsx
@@ -1,4 +1,6 @@
 import { policyEnginePrimitive, policyPrimitive } from '../../../primitives/registry';
+import { withAddTelemetry } from '../../../telemetry/cli-command-run.js';
+import { ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js';
 import {
   ErrorPrompt,
   Panel,
@@ -128,7 +130,14 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD
   }, []);
 
   const commitEngine = useCallback(async (engineName: string, gateways?: string[], mode?: 'LOG_ONLY' | 'ENFORCE') => {
-    const result = await policyEnginePrimitive.add({ name: engineName });
+    const result = await withAddTelemetry(
+      'add.policy-engine',
+      {
+        attach_gateway_count: 0,
+        attach_mode: 'log_only',
+      },
+      () => policyEnginePrimitive.add({ name: engineName })
+    );
     if (!result.success) {
       setFlow({ name: 'error', message: result.error });
       return;
@@ -155,13 +164,21 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD
   );
 
   const handlePolicyComplete = useCallback(async (config: AddPolicyConfig) => {
-    const result = await policyPrimitive.add({
-      name: config.name,
-      engine: config.engine,
-      statement: config.statement,
-      source: config.sourceFile || undefined,
-      validationMode: config.validationMode,
-    });
+    const result = await withAddTelemetry(
+      'add.policy',
+      {
+        source_type: config.sourceFile ? 'file' : 'statement',
+        validation_mode: standardize(ValidationMode, config.validationMode ?? 'FAIL_ON_ANY_FINDINGS'),
+      },
+      () =>
+        policyPrimitive.add({
+          name: config.name,
+          engine: config.engine,
+          statement: config.statement,
+          source: config.sourceFile || undefined,
+          validationMode: config.validationMode,
+        })
+    );
 
     if (result.success) {
       setPolicyNames(prev => [...prev, config.name]);
diff --git a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx
@@ -1,5 +1,6 @@
 import { ConfigIO } from '../../../../lib';
 import { runtimeEndpointPrimitive } from '../../../primitives/registry';
+import { withAddTelemetry } from '../../../telemetry/cli-command-run.js';
 import { ErrorPrompt } from '../../components';
 import { AddSuccessScreen } from '../add/AddSuccessScreen';
 import { AddRuntimeEndpointScreen } from './AddRuntimeEndpointScreen';
@@ -78,24 +79,24 @@ export function AddRuntimeEndpointFlow({
   }, [isInteractive, flow.name, onExit]);
 
   const handleCreateComplete = useCallback((config: RuntimeEndpointWizardConfig) => {
-    void runtimeEndpointPrimitive
-      .add({
+    void withAddTelemetry('add.runtime-endpoint', {}, () =>
+      runtimeEndpointPrimitive.add({
         runtime: config.runtimeName,
         endpoint: config.endpointName,
         version: config.version,
         description: config.description,
       })
-      .then(result => {
-        if (result.success) {
-          setFlow({
-            name: 'create-success',
-            endpointName: config.endpointName,
-            runtimeName: config.runtimeName,
-          });
-          return;
-        }
-        setFlow({ name: 'error', message: result.error ?? 'Unknown error' });
-      });
+    ).then(result => {
+      if (result.success) {
+        setFlow({
+          name: 'create-success',
+          endpointName: config.endpointName,
+          runtimeName: config.runtimeName,
+        });
+        return;
+      }
+      setFlow({ name: 'error', message: result.error ?? 'Unknown error' });
+    });
   }, []);
 
   if (flow.name === 'loading') {
