chore(integration-tests): improve error logging for better debugging (#1109)

## Summary
- Integration tests can sometimes be a little flaky. It isn't clear what
the errors are, but seems like some sort of request throttling.
Rerunning the tests succeeds. Adding some better logging for this.
- No changes to actual extension code.

## Changes
- **invoke.ts**: Log `TooManyRequestsException` reason and
`retryAfterSeconds`
- **lambda.ts**: Add detailed error logging to `publishVersion()`,
`setTimestampEnvVar()`, `waitForSnapStartReady()`
- **datadog.ts**: Log HTTP status, rate limit headers, and query context
for API errors
- **misc**: Moved lambda invoke code from invoke.ts to lambda.ts.

## Example Output
```
Lambda invocation failed for 'my-function': TooManyRequestsException - Rate exceeded - (Reason: ConcurrentSnapshotCreateLimitExceeded) - (retryAfterSeconds: 5)
Error searching traces: Error (query: 'service:my-service @request_id:abc123'): HTTP 429 Too Many Requests (retry-after: 30s)
```

## Test plan
- [x] TypeScript build passes
- [x] Manual verification during next integration test run

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: jchrostek-dd

integration-tests/tests/utils/datadog.ts

@@ -1,4 +1,4 @@
-import axios, { AxiosInstance } from 'axios';
+import axios, { AxiosInstance, AxiosError } from 'axios';
 
 const DD_API_KEY = process.env.DD_API_KEY;
 const DD_APP_KEY = process.env.DD_APP_KEY;
@@ -17,6 +17,43 @@ const datadogClient: AxiosInstance = axios.create({
   timeout: 30000,
 });
 
+function formatDatadogError(error: unknown, query: string): string {
+  if (error instanceof AxiosError && error.response) {
+    const status = error.response.status;
+    const statusText = error.response.statusText || '';
+    let message = `HTTP ${status} ${statusText}`.trim();
+
+    // Include rate limit info for 429 errors
+    if (status === 429) {
+      const retryAfter = error.response.headers['x-ratelimit-reset'] || error.response.headers['retry-after'];
+      if (retryAfter) {
+        message += ` (retry-after: ${retryAfter}s)`;
+      }
+      const remaining = error.response.headers['x-ratelimit-remaining'];
+      if (remaining !== undefined) {
+        message += ` (remaining: ${remaining})`;
+      }
+    }
+
+    // Include response body if available
+    const responseData = error.response.data;
+    if (responseData) {
+      const bodyStr = typeof responseData === 'string' ? responseData : JSON.stringify(responseData);
+      if (bodyStr && bodyStr !== '{}') {
+        message += ` - ${bodyStr}`;
+      }
+    }
+
+    return `Error (query: '${query}'): ${message}`;
+  }
+
+  if (error instanceof Error) {
+    return `Error (query: '${query}'): ${error.message}`;
+  }
+
+  return `Error (query: '${query}'): ${String(error)}`;
+}
+
 export interface DatadogTelemetry {
   requestId: string;
   statusCode?: number;
@@ -72,12 +109,11 @@ export async function getTraces(
   const now = Date.now();
   const fromTime = now - (1 * 60 * 60 * 1000); // 1 hour ago
   const toTime = now;
-  try {
-    // Convert service name to lowercase as Datadog stores it that way
-    const serviceNameLower = serviceName.toLowerCase();
-
-    const query = `service:${serviceNameLower} @request_id:${requestId}`;
+  // Convert service name to lowercase as Datadog stores it that way
+  const serviceNameLower = serviceName.toLowerCase();
+  const query = `service:${serviceNameLower} @request_id:${requestId}`;
 
+  try {
     console.log(`Searching for traces: ${query}`);
 
     // First, find spans matching the request_id to get trace IDs
@@ -164,8 +200,8 @@ export async function getTraces(
     }
 
     return traces;
-  } catch (error: any) {
-    console.error('Error searching traces:', error.response?.data || error.message);
+  } catch (error: unknown) {
+    console.error(`Error searching traces: ${formatDatadogError(error, query)}`);
     throw error;
   }
 }
@@ -182,9 +218,9 @@ export async function getLogs(
   const now = Date.now();
   const fromTime = now - (2 * 60 * 60 * 1000); // 2 hours ago
   const toTime = now;
-  try {
-    const query = `service:${serviceName} @lambda.request_id:${requestId}`;
+  const query = `service:${serviceName} @lambda.request_id:${requestId}`;
 
+  try {
     console.log(`Searching for logs: ${query}`);
 
     const response = await datadogClient.post('/api/v2/logs/events/search', {
@@ -214,8 +250,8 @@ export async function getLogs(
     });
 
     return logs;
-  } catch (error: any) {
-    console.error('Error searching logs:', error.response?.data || error.message);
+  } catch (error: unknown) {
+    console.error(`Error searching logs: ${formatDatadogError(error, query)}`);
     throw error;
   }
 }

integration-tests/tests/utils/default.ts

@@ -1,4 +1,4 @@
-import { invokeLambda, InvocationResult } from './invoke';
+import { invokeLambda, InvocationResult } from './lambda';
 import { getDatadogTelemetryByRequestId, DatadogTelemetry } from './datadog';
 import { DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../../config';
 

integration-tests/tests/utils/invoke.ts

@@ -1,7 +1,70 @@
-import { LambdaClient, UpdateFunctionConfigurationCommand, GetFunctionConfigurationCommand, PublishVersionCommand } from '@aws-sdk/client-lambda';
+import {
+  LambdaClient,
+  InvokeCommand,
+  UpdateFunctionConfigurationCommand,
+  GetFunctionConfigurationCommand,
+  PublishVersionCommand,
+  TooManyRequestsException,
+  ServiceException,
+} from '@aws-sdk/client-lambda';
 
 const lambdaClient = new LambdaClient({ region: 'us-east-1' });
 
+function formatLambdaError(error: unknown): string {
+  if (error instanceof TooManyRequestsException) {
+    const parts = [error.name, error.message];
+    if (error.Reason) {
+      parts.push(`(Reason: ${error.Reason})`);
+    }
+    if (error.retryAfterSeconds !== undefined) {
+      parts.push(`(retryAfterSeconds: ${error.retryAfterSeconds})`);
+    }
+    return parts.join(' - ');
+  }
+
+  if (error instanceof ServiceException) {
+    return `${error.name} - ${error.message}`;
+  }
+
+  if (error instanceof Error) {
+    return error.message;
+  }
+
+  return String(error);
+}
+
+export interface InvocationResult {
+  functionName: string;
+  requestId: string;
+  statusCode?: number;
+}
+
+export async function invokeLambda(
+  functionName: string,
+  payload: any = {},
+): Promise<InvocationResult> {
+  const command = new InvokeCommand({
+    FunctionName: functionName,
+    Payload: JSON.stringify(payload),
+  });
+
+  let response;
+  try {
+    response = await lambdaClient.send(command);
+  } catch (error: unknown) {
+    console.error(`Lambda invocation failed for '${functionName}': ${formatLambdaError(error)}`);
+    throw error;
+  }
+
+  const requestId: string = response.$metadata.requestId || '';
+
+  return {
+    functionName,
+    requestId,
+    statusCode: response.StatusCode || 200,
+  };
+}
+
 export async function forceColdStart(functionName: string): Promise<void> {
   await setTimestampEnvVar(functionName);
   await new Promise(resolve => setTimeout(resolve, 10000));
@@ -19,8 +82,8 @@ export async function publishVersion(functionName: string): Promise<string> {
     const version = response.Version || '$LATEST';
     console.debug(`Published version: ${version} for ${functionName}`);
     return version;
-  } catch (error: any) {
-    console.error('Failed to publish Lambda version:', error.message);
+  } catch (error: unknown) {
+    console.error(`Failed to publish Lambda version for '${functionName}': ${formatLambdaError(error)}`);
     throw error;
   }
 }
@@ -45,8 +108,8 @@ export async function setTimestampEnvVar(functionName: string): Promise<void> {
       Environment: updatedEnvironment,
     });
     await lambdaClient.send(updateConfigCommand);
-  } catch (error: any) {
-    console.error('Failed to set timestamp environment variable:', error.message);
+  } catch (error: unknown) {
+    console.error(`Failed to set timestamp environment variable for '${functionName}': ${formatLambdaError(error)}`);
     throw error;
   }
 }
@@ -71,8 +134,8 @@ export async function waitForSnapStartReady(functionName: string, version: strin
       }
 
       await new Promise(resolve => setTimeout(resolve, 10000));
-    } catch (error: any) {
-      console.error(`Error checking SnapStart status: ${error.message}`);
+    } catch (error: unknown) {
+      console.error(`Error checking SnapStart status for '${functionName}:${version}': ${formatLambdaError(error)}`);
       await new Promise(resolve => setTimeout(resolve, 10_000));
     }
   }