Merge remote-tracking branch 'origin/main' into fix-langgraph-observability

Author: jesseturner21

.github/workflows/build-and-test.yml

@@ -47,6 +47,24 @@ jobs:
         with:
           name: coverage-report
           path: coverage/
+      - name: Pack tarball
+        if: matrix.node-version == '20.x'
+        run: npm pack
+      - name: Install from tarball
+        if: matrix.node-version == '20.x'
+        run: npm install -g aws-agentcore-*.tgz
+      - name: Verify packaged CLI runs
+        if: matrix.node-version == '20.x'
+        run: |
+          agentcore --version
+          agentcore create --name sanitytest --language Python --framework Strands --model-provider Bedrock --memory none --json
+          test -f sanitytest/agentcore/agentcore.json
+      - name: Upload tarball artifact
+        if: matrix.node-version == '20.x'
+        uses: actions/upload-artifact@v7
+        with:
+          name: tarball
+          path: '*.tgz'
 
   coverage:
     needs: build

.github/workflows/cleanup-pr-tarballs.yml

@@ -0,0 +1,30 @@
+name: Cleanup PR Tarballs
+
+on:
+  schedule:
+    # Run daily at midnight UTC
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Delete PR tarball releases older than 7 days
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          CUTOFF=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-7d +%Y-%m-%dT%H:%M:%SZ)
+
+          gh release list --limit 100 --json tagName,createdAt,isPrerelease \
+            --jq '.[] | select(.isPrerelease and (.tagName | startswith("pr-")) and (.tagName | endswith("-tarball")))' \
+          | jq -r --arg cutoff "$CUTOFF" 'select(.createdAt < $cutoff) | .tagName' \
+          | while read -r TAG; do
+              echo "Deleting old tarball release: $TAG"
+              gh release delete "$TAG" --yes --cleanup-tag 2>/dev/null || true
+            done

.github/workflows/e2e-tests-weekly.yml .github/workflows/e2e-tests-full.yml.github/workflows/e2e-tests-weekly.yml renamed to .github/workflows/e2e-tests-full.yml

@@ -1,4 +1,4 @@
-name: E2E Tests (Weekly Full Suite)
+name: E2E Tests (Full Suite)
 on:
   workflow_dispatch:
     inputs:
@@ -7,6 +7,8 @@ on:
         default: 'us-east-1'
   schedule:
     - cron: '0 14 * * 1' # Every Monday at 9 AM EST (14:00 UTC)
+  push:
+    branches: [main]
 
 concurrency:
   group: e2e-${{ github.event.pull_request.number || github.ref }}
@@ -21,6 +23,10 @@ jobs:
     runs-on: ubuntu-latest
     environment: e2e-testing
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        cdk-source: [npm, main]
     steps:
       - uses: actions/checkout@v6
         with:
@@ -50,11 +56,26 @@ jobs:
           parse-json-secrets: true
       - run: npm ci
       - run: npm run build
-      - name: Run E2E tests
+      - name: Build CDK package from main
+        if: matrix.cdk-source == 'main'
+        run: |
+          git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo
+          cd /tmp/cdk-repo
+          npm ci
+          npm run build
+          TARBALL=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -1)
+          echo "CDK_TARBALL=$RUNNER_TEMP/$TARBALL" >> "$GITHUB_ENV"
+        env:
+          CDK_REPO_TOKEN: ${{ secrets.CDK_REPO_TOKEN }}
+          CDK_REPO: ${{ secrets.CDK_REPO_NAME }}
+      - name: Install CLI globally
+        run: npm install -g "$(npm pack | tail -1)"
+      - name: Run E2E tests (${{ matrix.cdk-source }})
         env:
           AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}
           AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
           ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
+          CDK_TARBALL: ${{ env.CDK_TARBALL }}
         run: npm run test:e2e

.github/workflows/e2e-tests.yml

@@ -47,6 +47,10 @@ jobs:
     runs-on: ubuntu-latest
     environment: e2e-testing
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        cdk-source: [npm, main]
     steps:
       - uses: actions/checkout@v6
         with:
@@ -74,13 +78,32 @@ jobs:
           secret-ids: |
             E2E,${{ secrets.E2E_SECRET_ARN }}
           parse-json-secrets: true
+
+      # Build @aws/agentcore-cdk from source for cross-package testing.
+      # Requires secrets: CDK_REPO_NAME (org/repo), CDK_REPO_TOKEN (fine-grained PAT)
+      - name: Build CDK package from main
+        if: matrix.cdk-source == 'main'
+        run: |
+          git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo
+          cd /tmp/cdk-repo
+          npm ci
+          npm run build
+          TARBALL=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -1)
+          echo "CDK_TARBALL=$RUNNER_TEMP/$TARBALL" >> "$GITHUB_ENV"
+        env:
+          CDK_REPO_TOKEN: ${{ secrets.CDK_REPO_TOKEN }}
+          CDK_REPO: ${{ secrets.CDK_REPO_NAME }}
+
       - run: npm ci
       - run: npm run build
-      - name: Run E2E tests
+      - name: Install CLI globally
+        run: npm install -g "$(npm pack | tail -1)"
+      - name: Run E2E tests (${{ matrix.cdk-source }})
         env:
           AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}
           AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
           ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
+          CDK_TARBALL: ${{ env.CDK_TARBALL }}
         run: npx vitest run --project e2e strands-bedrock strands-openai langgraph-bedrock googleadk-gemini

.github/workflows/pr-tarball.yml

@@ -0,0 +1,71 @@
+name: PR Tarball
+on:
+  pull_request_target:
+    branches: [main]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  pr-tarball:
+    runs-on: ubuntu-latest
+    if: >-
+      contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association)
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20.x'
+          cache: 'npm'
+      - name: Configure git
+        run: |
+          git config --global user.email "bedrock-agentcore-npm+ci@amazon.com"
+          git config --global user.name "CI"
+      - uses: astral-sh/setup-uv@v7
+      - run: npm ci
+      - run: npm run build --if-present
+      - run: npm pack
+      - name: Get tarball info
+        id: tarball
+        run: |
+          TARBALL_NAME=$(ls *.tgz | head -1 | xargs basename)
+          echo "name=$TARBALL_NAME" >> $GITHUB_OUTPUT
+      - name: Create or update PR release
+        id: release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          TARBALL_NAME: ${{ steps.tarball.outputs.name }}
+        run: |
+          TAG="pr-${PR_NUMBER}-tarball"
+
+          # Delete existing release if it exists (to update the tarball)
+          gh release delete "$TAG" --yes --cleanup-tag 2>/dev/null || true
+
+          # Create a new pre-release with the tarball
+          gh release create "$TAG" \
+            "${TARBALL_NAME}" \
+            --title "PR #${PR_NUMBER} Tarball" \
+            --notes "Auto-generated tarball for PR #${PR_NUMBER}." \
+            --prerelease \
+            --target "${{ github.event.pull_request.head.sha }}"
+
+          DOWNLOAD_URL="https://github.com/${{ github.repository }}/releases/download/${TAG}/${TARBALL_NAME}"
+          echo "url=$DOWNLOAD_URL" >> $GITHUB_OUTPUT
+      - name: Comment on PR
+        uses: marocchino/sticky-pull-request-comment@v3
+        with:
+          header: tarball
+          message: |
+            ## Package Tarball
+
+            **[${{ steps.tarball.outputs.name }}](${{ steps.release.outputs.url }})**
+
+            ### How to install
+
+            ```bash
+            npm install ${{ steps.release.outputs.url }}
+            ```

.gitignore

@@ -60,3 +60,4 @@ workdir-tmp/
 bun.lock
 
 .agentreview
+ProtocolTesting/

.mcp.json

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "tui-harness": {
+      "type": "http",
+      "url": "http://127.0.0.1:24100/mcp"
+    }
+  }
+}

AGENTS.md

@@ -24,12 +24,20 @@ Note: CDK L3 constructs are in a separate package `@aws/agentcore-cdk`.
 ## CLI Commands
 
 - `create` - Create new AgentCore project
-- `add` - Add resources (agent, memory, identity, target)
-- `remove` - Remove resources (agent, memory, identity, target, all)
+- `add` - Add resources (agent, memory, identity, evaluator, online-eval, target)
+- `remove` - Remove resources (agent, memory, identity, evaluator, online-eval, target, all)
 - `deploy` - Deploy infrastructure to AWS
 - `status` - Check deployment status
 - `dev` - Local development server (CodeZip: uvicorn with hot-reload; Container: Docker build + run with volume mount)
 - `invoke` - Invoke agents (local or deployed)
+- `run eval` - Run on-demand evaluation against agent sessions
+- `evals history` - View past eval run results
+- `pause online-eval` - Pause (disable) a deployed online eval config
+- `resume online-eval` - Resume (enable) a paused online eval config
+- `logs` - Stream or search agent runtime logs
+- `logs evals` - Stream or search online eval logs
+- `traces list` - List recent traces for a deployed agent
+- `traces get` - Download a trace to a JSON file
 - `package` - Package agent artifacts without deploying (zip for CodeZip, container image build for Container)
 - `validate` - Validate configuration files
 - `update` - Check for CLI updates
@@ -40,21 +48,20 @@ Note: CDK L3 constructs are in a separate package `@aws/agentcore-cdk`.
 - **Template agents**: Created from framework templates (Strands, LangChain_LangGraph, CrewAI, GoogleADK, OpenAIAgents,
   AutoGen)
 - **BYO agents**: Bring your own code with `agentcore add agent --type byo`
+- **Imported agents**: Import from Bedrock Agents with `agentcore add agent --type import`
 
 ### Build Types
 
 - **CodeZip**: Python source is packaged into a zip artifact and deployed to AgentCore Runtime (default)
 - **Container**: Agent is built as a Docker container image, deployed via ECR and CodeBuild. Requires a `Dockerfile` in
   the agent's code directory. Supported container runtimes: Docker, Podman, Finch.
 
-### Coming Soon
-
-- MCP gateway and tool support (`add gateway`, `add mcp-tool`) - currently hidden
-
 ## Primitives Architecture
 
-All resource types (agent, memory, identity, gateway, mcp-tool) are modeled as **primitives** — self-contained classes
-in `src/cli/primitives/` that own the full add/remove lifecycle for their resource type.
+All resource types (agent, memory, identity, evaluator, online-eval, gateway, mcp-tool) are modeled as **primitives** --
+self-contained classes in `src/cli/primitives/` that own the full add/remove lifecycle for their resource type.
+Resources support config-driven tagging via `agentcore.json` and `mcp.json`, with tags flowing through to deployed
+CloudFormation resources.
 
 Each primitive extends `BasePrimitive` and implements: `add()`, `remove()`, `previewRemove()`, `getRemovable()`,
 `registerCommands()`, and `addScreen()`.
@@ -64,8 +71,10 @@ Current primitives:
 - `AgentPrimitive` — agent creation (template + BYO), removal, credential resolution
 - `MemoryPrimitive` — memory creation with strategies, removal
 - `CredentialPrimitive` — credential/identity creation, .env management, removal
-- `GatewayPrimitive` — MCP gateway creation/removal (hidden, coming soon)
-- `GatewayTargetPrimitive` — MCP tool creation/removal with code generation (hidden, coming soon)
+- `EvaluatorPrimitive` — custom evaluator creation/removal with cross-reference validation
+- `OnlineEvalConfigPrimitive` — online eval config creation/removal
+- `GatewayPrimitive` — MCP gateway creation/removal
+- `GatewayTargetPrimitive` — MCP tool creation/removal with code generation
 
 Singletons are created in `registry.ts` and wired into CLI commands via `cli.ts`. See `src/cli/AGENTS.md` for details on
 adding new primitives.
@@ -121,3 +130,8 @@ See `docs/TESTING.md` for details.
 
 - Always look for existing types before creating a new type inline.
 - Re-usable constants must be defined in a constants file in the closest sensible subdirectory.
+
+## TUI Harness
+
+See `docs/tui-harness.md` for the full TUI harness usage guide (MCP tools, screen markers, examples, and error
+recovery).