fix(e2e): prevent API key credential provider quota exhaustion

E2E teardown deleted credential providers after CFN stack teardown,
which often fails with "No resources defined". Leaked providers
accumulated across CI runs until the account hit its quota limit,
breaking all non-Bedrock e2e suites.

Three changes:
1. Move credential provider deletion before CFN teardown in afterAll
2. Add cleanupStaleCredentialProviders() in beforeAll to purge leaks
3. Add CI workflow step to delete stale E2e* providers before tests

Constraint: Teardown CFN step frequently fails when deploy never succeeded
Rejected: Only fix teardown ordering | does not handle pre-existing leaks from past runs
Confidence: high
Scope-risk: narrow
Not-tested: Race condition between parallel matrix jobs deleting same provider

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jesseturner21

.github/workflows/e2e-tests.yml

@@ -98,6 +98,18 @@ jobs:
       - run: npm run build
       - name: Install CLI globally
         run: npm install -g "$(npm pack | tail -1)"
+      - name: Clean up stale credential providers
+        run: |
+          # Delete leaked E2e* API key credential providers from previous CI runs
+          # to prevent account quota exhaustion
+          for name in $(aws bedrock-agentcore-control list-api-key-credential-providers \
+            --region ${{ inputs.aws_region || 'us-east-1' }} \
+            --query 'credentialProviders[?starts_with(name, `E2e`)].name' \
+            --output text); do
+              echo "Deleting stale provider: $name"
+              aws bedrock-agentcore-control delete-api-key-credential-provider \
+                --name "$name" --region ${{ inputs.aws_region || 'us-east-1' }} || true
+          done
       - name: Run E2E tests (${{ matrix.cdk-source }})
         env:
           AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}

e2e-tests/e2e-helper.ts

@@ -9,6 +9,7 @@ import {
 import {
   BedrockAgentCoreControlClient,
   DeleteApiKeyCredentialProviderCommand,
+  ListApiKeyCredentialProvidersCommand,
 } from '@aws-sdk/client-bedrock-agentcore-control';
 import { execSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
@@ -39,6 +40,9 @@ export function createE2ESuite(cfg: E2EConfig) {
     beforeAll(async () => {
       if (!canRun) return;
 
+      // Purge leaked credential providers from prior CI runs to avoid quota limits
+      await cleanupStaleCredentialProviders();
+
       testDir = join(tmpdir(), `agentcore-e2e-${randomUUID()}`);
       await mkdir(testDir, { recursive: true });
 
@@ -292,20 +296,53 @@ export function installCdkTarball(projectPath: string): void {
 }
 
 export async function teardownE2EProject(projectPath: string, agentName: string, modelProvider: string): Promise<void> {
-  await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath);
-  const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath);
-  if (result.exitCode !== 0) {
-    console.log('Teardown stdout:', result.stdout);
-    console.log('Teardown stderr:', result.stderr);
-  }
+  // Delete the API key credential provider FIRST — CFN teardown may fail,
+  // and leaked providers accumulate until the account hits its quota limit.
   if (modelProvider !== 'Bedrock' && agentName) {
     const providerName = `${agentName}${modelProvider}`;
     const region = process.env.AWS_REGION ?? 'us-east-1';
     try {
       const client = new BedrockAgentCoreControlClient({ region });
       await client.send(new DeleteApiKeyCredentialProviderCommand({ name: providerName }));
-    } catch {
-      // Best-effort cleanup
+      console.log(`Deleted credential provider: ${providerName}`);
+    } catch (err) {
+      console.warn(`Failed to delete credential provider ${providerName}:`, err);
+    }
+  }
+
+  await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath);
+  const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath);
+  if (result.exitCode !== 0) {
+    console.log('Teardown stdout:', result.stdout);
+    console.log('Teardown stderr:', result.stderr);
+  }
+}
+
+/**
+ * Delete stale E2e* API key credential providers left behind by previous
+ * CI runs whose teardown failed. Call this before tests to prevent quota
+ * exhaustion in the shared test account.
+ */
+export async function cleanupStaleCredentialProviders(): Promise<void> {
+  const region = process.env.AWS_REGION ?? 'us-east-1';
+  try {
+    const client = new BedrockAgentCoreControlClient({ region });
+    const response = await client.send(new ListApiKeyCredentialProvidersCommand({}));
+    const providers = response.credentialProviders ?? [];
+    const stale = providers.filter(p => p.name?.startsWith('E2e'));
+
+    if (stale.length === 0) return;
+
+    console.log(`Cleaning up ${stale.length} stale E2e* credential provider(s)...`);
+    for (const provider of stale) {
+      try {
+        await client.send(new DeleteApiKeyCredentialProviderCommand({ name: provider.name! }));
+        console.log(`  Deleted: ${provider.name}`);
+      } catch (err) {
+        console.warn(`  Failed to delete ${provider.name}:`, err);
+      }
     }
+  } catch (err) {
+    console.warn('Failed to list credential providers for cleanup:', err);
   }
 }