feat: record command attrs on telemetry failure via fallbackAttrs (#1204)

Hweinstock · web-flow · commit f69e252473e9 · 2026-05-12T17:38:56.000-04:00
* feat: record command attrs on telemetry failure via fallbackAttrs

Add optional fallbackAttrs parameter to client.withCommandRun so
command-specific attributes are recorded even when the callback throws.

- client.ts: accept fallbackAttrs, use on failure instead of {}
- client.ts: run resilientParse on all non-empty attrs (not just success)
- cli-command-run.ts: withCommandRunTelemetry passes attrs as fallbackAttrs
- cli-command-run.ts: runCliCommand accepts optional knownAttrs param
- command.tsx: extract knownAttrs upfront, pass to runCliCommand
- client.test.ts: add unit tests for fallbackAttrs behavior
- create-edge-cases.test.ts: assert attrs present on failure entry

* chore: rebase onto mainline
diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts
@@ -38,6 +38,8 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
       telemetry.assertMetricEmitted({
         command: 'create',
         exit_reason: 'failure',
+        language: 'python',
+        has_agent: 'true',
       });
     });
 
diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx
@@ -114,92 +114,99 @@ async function handleCreateCLI(options: CreateOptions): Promise<void> {
     process.exit(0);
   }
 
-  await runCliCommand('create', !!options.json, async () => {
-    const validation = validateCreateOptions(options, cwd);
-    if (!validation.valid) {
-      throw new Error(validation.error);
-    }
-    const green = '\x1b[32m';
-    const reset = '\x1b[0m';
+  const knownAttrs = {
+    language: standardize(Language, options.language),
+    framework: standardize(Framework, options.framework),
+    model_provider: standardize(ModelProviderEnum, options.modelProvider),
+    memory: standardize(Memory, options.memory ?? 'none'),
+    protocol: standardize(Protocol, options.protocol ?? 'http'),
+    build: standardize(Build, options.build ?? 'codezip'),
+    agent_type: standardize(AgentType, options.type ?? 'create'),
+    network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'),
+    has_agent: options.agent !== false,
+  };
 
-    // Progress callback for real-time output
-    const onProgress: ProgressCallback | undefined = options.json
-      ? undefined
-      : (step, status) => {
-          if (status === 'done') {
-            console.log(`${green}[done]${reset}  ${step}`);
-          } else if (status === 'error') {
-            console.log(`\x1b[31m[error]${reset} ${step}`);
-          }
-          // 'start' is silent - we only show when done
-        };
+  await runCliCommand(
+    'create',
+    !!options.json,
+    async () => {
+      const validation = validateCreateOptions(options, cwd);
+      if (!validation.valid) {
+        throw new Error(validation.error);
+      }
+      const green = '\x1b[32m';
+      const reset = '\x1b[0m';
 
-    // Commander.js --no-agent sets agent=false, not noAgent=true
-    const skipAgent = options.agent === false;
+      // Progress callback for real-time output
+      const onProgress: ProgressCallback | undefined = options.json
+        ? undefined
+        : (step, status) => {
+            if (status === 'done') {
+              console.log(`${green}[done]${reset}  ${step}`);
+            } else if (status === 'error') {
+              console.log(`\x1b[31m[error]${reset} ${step}`);
+            }
+            // 'start' is silent - we only show when done
+          };
 
-    const result = skipAgent
-      ? await createProject({
-          name: projectName!,
-          cwd,
-          skipGit: options.skipGit,
-          skipInstall: options.skipInstall,
-          onProgress,
-        })
-      : await createProjectWithAgent({
-          name: name!,
-          projectName,
-          cwd,
-          type: options.type as 'create' | 'import' | undefined,
-          buildType: (options.build as BuildType) ?? 'CodeZip',
-          language: (options.language as TargetLanguage) ?? (options.type === 'import' ? 'Python' : undefined),
-          framework: options.framework as SDKFramework | undefined,
-          modelProvider: options.modelProvider as ModelProvider | undefined,
-          apiKey: options.apiKey,
-          memory: (options.memory as 'none' | 'shortTerm' | 'longAndShortTerm') ?? 'none',
-          protocol: options.protocol as ProtocolMode | undefined,
-          agentId: options.agentId,
-          agentAliasId: options.agentAliasId,
-          region: options.region,
-          networkMode: options.networkMode as NetworkMode | undefined,
-          subnets: parseCommaSeparatedList(options.subnets),
-          securityGroups: parseCommaSeparatedList(options.securityGroups),
-          idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
-          maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
-          sessionStorageMountPath: options.sessionStorageMountPath,
-          withConfigBundle: options.withConfigBundle,
-          skipGit: options.skipGit,
-          skipInstall: options.skipInstall,
-          skipPythonSetup: options.skipPythonSetup,
-          onProgress,
-        });
+      // Commander.js --no-agent sets agent=false, not noAgent=true
+      const skipAgent = options.agent === false;
 
-    if (!result.success) {
-      throw result.error;
-    }
+      const result = skipAgent
+        ? await createProject({
+            name: projectName!,
+            cwd,
+            skipGit: options.skipGit,
+            skipInstall: options.skipInstall,
+            onProgress,
+          })
+        : await createProjectWithAgent({
+            name: name!,
+            projectName,
+            cwd,
+            type: options.type as 'create' | 'import' | undefined,
+            buildType: (options.build as BuildType) ?? 'CodeZip',
+            language: (options.language as TargetLanguage) ?? (options.type === 'import' ? 'Python' : undefined),
+            framework: options.framework as SDKFramework | undefined,
+            modelProvider: options.modelProvider as ModelProvider | undefined,
+            apiKey: options.apiKey,
+            memory: (options.memory as 'none' | 'shortTerm' | 'longAndShortTerm') ?? 'none',
+            protocol: options.protocol as ProtocolMode | undefined,
+            agentId: options.agentId,
+            agentAliasId: options.agentAliasId,
+            region: options.region,
+            networkMode: options.networkMode as NetworkMode | undefined,
+            subnets: parseCommaSeparatedList(options.subnets),
+            securityGroups: parseCommaSeparatedList(options.securityGroups),
+            idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined,
+            maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined,
+            sessionStorageMountPath: options.sessionStorageMountPath,
+            withConfigBundle: options.withConfigBundle,
+            skipGit: options.skipGit,
+            skipInstall: options.skipInstall,
+            skipPythonSetup: options.skipPythonSetup,
+            onProgress,
+          });
 
-    if (options.json) {
-      console.log(JSON.stringify(serializeResult(result)));
-    } else {
-      printCreateSummary(projectName!, result.agentName, options.language, options.framework);
-      if (options.skipInstall) {
-        console.log(
-          "\nDependency installation was skipped. Run 'npm install' in agentcore/cdk/ and 'uv sync' in your agent directory manually."
-        );
+      if (!result.success) {
+        throw result.error;
       }
-    }
 
-    return {
-      language: standardize(Language, options.language),
-      framework: standardize(Framework, options.framework),
-      model_provider: standardize(ModelProviderEnum, options.modelProvider),
-      memory: standardize(Memory, options.memory ?? 'none'),
-      protocol: standardize(Protocol, options.protocol ?? 'http'),
-      build: standardize(Build, options.build ?? 'codezip'),
-      agent_type: standardize(AgentType, options.type ?? 'create'),
-      network_mode: standardize(NetworkModeEnum, options.networkMode ?? 'public'),
-      has_agent: options.agent !== false,
-    };
-  });
+      if (options.json) {
+        console.log(JSON.stringify(result));
+      } else {
+        printCreateSummary(projectName!, result.agentName, options.language, options.framework);
+        if (options.skipInstall) {
+          console.log(
+            "\nDependency installation was skipped. Run 'npm install' in agentcore/cdk/ and 'uv sync' in your agent directory manually."
+          );
+        }
+      }
+
+      return knownAttrs;
+    },
+    knownAttrs
+  );
 }
 
 export const registerCreate = (program: Command) => {
diff --git a/src/cli/telemetry/README.md b/src/cli/telemetry/README.md
@@ -62,11 +62,12 @@ async function withCommandRunTelemetry<C extends Command, R extends OperationRes
 
 **Behavior:**
 
-- On success (`{ success: true }`): records success telemetry with `attrs`, returns the result.
-- On failure (`{ success: false, error }`): records failure telemetry, returns the result to the caller.
-- On throw: records failure telemetry, returns `{ success: false, error }` so callers don't leak unhandled rejections.
+- On success: records `attrs` with `exit_reason: 'success'`, returns the result.
+- On failure/throw: records `attrs` with `exit_reason: 'failure'`, returns `{ success: false, error }`.
 - If telemetry is unavailable: runs `fn()` untracked.
 
+Since `attrs` are passed upfront, they are always recorded — even on failure.
+
 **Example with attributes:**
 
 ```ts
@@ -95,6 +96,22 @@ await runCliCommand('add.widget', !!options.json, async () => {
 });
 ```
 
+To record attrs on failure, pass `knownAttrs` as the fourth argument:
+
+```ts
+const knownAttrs = { widget_type: standardize(WidgetType, options.type), count: options.items.length };
+await runCliCommand(
+  'add.widget',
+  !!options.json,
+  async () => {
+    const result = await widgetPrimitive.add(options);
+    if (!result.success) throw new Error(result.error);
+    return knownAttrs;
+  },
+  knownAttrs
+);
+```
+
 ## Key Points
 
 - Telemetry never crashes the CLI — `standardize()` falls back gracefully, `resilientParse` defaults invalid fields to
diff --git a/src/cli/telemetry/__tests__/client.test.ts b/src/cli/telemetry/__tests__/client.test.ts
@@ -173,5 +173,54 @@ describe('TelemetryClient', () => {
         exit_reason: 'cancel',
       });
     });
+
+    it('records fallbackAttrs on failure when provided', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await expect(
+        client.withCommandRun(
+          'create',
+          async () => {
+            throw new Error('validation failed');
+          },
+          {
+            language: 'python',
+            framework: 'strands',
+            model_provider: 'bedrock',
+            memory: 'none',
+            protocol: 'http',
+            build: 'codezip',
+            agent_type: 'create',
+            network_mode: 'public',
+            has_agent: true,
+          }
+        )
+      ).rejects.toThrow('validation failed');
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs).toMatchObject({
+        exit_reason: 'failure',
+        error_name: 'UnknownError',
+        language: 'python',
+        framework: 'strands',
+        model_provider: 'bedrock',
+        has_agent: 'true',
+      });
+    });
+
+    it('records empty attrs on failure when fallbackAttrs not provided', async () => {
+      const sink = new InMemorySink();
+      const client = new TelemetryClient(sink);
+
+      await expect(
+        client.withCommandRun('deploy', async () => {
+          throw new Error('boom');
+        })
+      ).rejects.toThrow('boom');
+
+      expect(sink.metrics).toHaveLength(1);
+      expect(sink.metrics[0]!.attrs.language).toBeUndefined();
+    });
   });
 });
diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts
@@ -31,11 +31,15 @@ export async function withCommandRunTelemetry<C extends Command, R extends Opera
 
   let result: R | undefined;
   try {
-    await client.withCommandRun(command, async () => {
-      result = await fn();
-      if (!result.success) throw result.error;
-      return attrs;
-    });
+    await client.withCommandRun(
+      command,
+      async () => {
+        result = await fn();
+        if (!result.success) throw result.error;
+        return attrs;
+      },
+      attrs
+    );
   } catch (e) {
     // withCommandRun re-throws after recording failure telemetry.
     // If result was set, fn() returned a failure result — return it directly.
@@ -52,19 +56,21 @@ export async function withCommandRunTelemetry<C extends Command, R extends Opera
  * Record telemetry, print errors, and exit the process.
  * Use in CLI command handlers where the command is the final action.
  * The callback returns attrs on success and throws on failure.
+ * Pass knownAttrs to record command-specific attributes even on failure.
  */
 export async function runCliCommand<C extends Command>(
   command: C,
   json: boolean,
-  fn: () => Promise<CommandAttrs<C>>
+  fn: () => Promise<CommandAttrs<C>>,
+  knownAttrs?: Partial<CommandAttrs<C>>
 ): Promise<never> {
   try {
     const client = await getTelemetryClient();
     if (!client) {
       await fn();
       process.exit(0);
     }
-    await client.withCommandRun(command, fn);
+    await client.withCommandRun(command, fn, knownAttrs);
     process.exit(0);
   } catch (error) {
     if (json) {
diff --git a/src/cli/telemetry/client.ts b/src/cli/telemetry/client.ts
@@ -26,7 +26,8 @@ export class TelemetryClient {
    */
   async withCommandRun<C extends Command>(
     command: C,
-    fn: () => CommandAttrs<C> | typeof CANCELLED | Promise<CommandAttrs<C> | typeof CANCELLED>
+    fn: () => CommandAttrs<C> | typeof CANCELLED | Promise<CommandAttrs<C> | typeof CANCELLED>,
+    fallbackAttrs?: Partial<CommandAttrs<C>>
   ): Promise<void> {
     const start = performance.now();
     try {
@@ -43,7 +44,7 @@ export class TelemetryClient {
         error_name: classifyError(err),
         is_user_error: isUserError(err),
       };
-      this.recordCommandRun(command, failureResult, {}, Math.round(performance.now() - start));
+      this.recordCommandRun(command, failureResult, fallbackAttrs ?? {}, Math.round(performance.now() - start));
       throw err;
     } finally {
       try {
@@ -75,9 +76,8 @@ export class TelemetryClient {
 
       // Validate command attrs resiliently: invalid fields default to 'unknown'
       // instead of dropping the entire metric.
-      // On failure/cancel the callback attrs are empty so validation is skipped.
       const validatedAttrs =
-        result.exit_reason !== 'failure' && result.exit_reason !== 'cancel'
+        Object.keys(attrs as Record<string, unknown>).length > 0
           ? resilientParse(COMMAND_SCHEMAS[command], attrs as Record<string, unknown>)
           : attrs;
 
