chore: split e2e workflow into PR-focused and weekly full suite (#367)

* chore: split e2e workflow into PR-focused and weekly full suite

Run only 4 focused tests (strands-bedrock, strands-openai, langgraph-bedrock,
googleadk-gemini) on PRs for fast feedback. Add a weekly workflow that runs the
full e2e suite every Monday at 9 AM EST.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: clean up API key credential providers in e2e test teardown

The e2e afterAll was only tearing down CDK stacks but not deleting the
API key credential providers created as a pre-deploy step. This caused
providers to accumulate and hit the account limit, failing all non-Bedrock
tests.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: jesseturner21

.github/workflows/e2e-tests-weekly.yml

@@ -0,0 +1,56 @@
+name: E2E Tests (Weekly Full Suite)
+on:
+  workflow_dispatch:
+    inputs:
+      aws_region:
+        description: 'AWS region for deployment'
+        default: 'us-east-1'
+  schedule:
+    - cron: '0 14 * * 1' # Every Monday at 9 AM EST (14:00 UTC)
+
+permissions:
+  id-token: write # OIDC — lets GitHub assume an AWS IAM role via short-lived token (no stored keys)
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    environment: e2e-testing
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: main
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20.x'
+          cache: 'npm'
+      - name: Configure git
+        run: |
+          git config --global user.email "ci@amazon.com"
+          git config --global user.name "CI"
+      - uses: astral-sh/setup-uv@v5
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.E2E_AWS_ROLE_ARN }}
+          aws-region: ${{ inputs.aws_region || 'us-east-1' }}
+      - name: Get AWS Account ID
+        id: aws
+        run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT"
+      - name: Get API keys from Secrets Manager
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            E2E,${{ secrets.E2E_SECRET_ARN }}
+          parse-json-secrets: true
+      - run: npm ci
+      - run: npm run build
+      - name: Run E2E tests
+        env:
+          AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}
+          AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
+          ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
+          GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
+        run: npm run test:e2e

.github/workflows/e2e-tests.yml

@@ -53,4 +53,4 @@ jobs:
           ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
-        run: npm run test:e2e
+        run: npx vitest run --project e2e strands-bedrock strands-openai langgraph-bedrock googleadk-gemini

e2e-tests/e2e-helper.ts

@@ -1,4 +1,8 @@
 import { hasAwsCredentials, parseJsonOutput, prereqs, runCLI } from '../src/test-utils/index.js';
+import {
+  BedrockAgentCoreControlClient,
+  DeleteApiKeyCredentialProviderCommand,
+} from '@aws-sdk/client-bedrock-agentcore-control';
 import { execSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { mkdir, rm, writeFile } from 'node:fs/promises';
@@ -99,6 +103,20 @@ export function createE2ESuite(cfg: E2EConfig) {
           console.log('Teardown stdout:', result.stdout);
           console.log('Teardown stderr:', result.stderr);
         }
+
+        // Delete the API key credential provider from the account.
+        // These are created as a pre-deploy step outside CDK and are not
+        // cleaned up by stack teardown, so we must delete them explicitly.
+        if (cfg.modelProvider !== 'Bedrock' && agentName) {
+          const providerName = `${agentName}${cfg.modelProvider}`;
+          const region = process.env.AWS_REGION ?? 'us-east-1';
+          try {
+            const client = new BedrockAgentCoreControlClient({ region });
+            await client.send(new DeleteApiKeyCredentialProviderCommand({ name: providerName }));
+          } catch {
+            // Best-effort cleanup — don't fail the test if deletion fails
+          }
+        }
       }
       if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
     }, 600000);