Improve MCP developer experience

.github/workflows/regenerate-on-overrides.yml

@@ -0,0 +1,78 @@
+name: Regenerate Docs on Override Changes
+
+on:
+  push:
+    paths:
+      - 'docs-data/overrides.json'
+    branches-ignore:
+      - main
+
+jobs:
+  regenerate-docs:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: read
+
+    env:
+      NODE_VERSION: '24'
+      DOCS_OVERRIDES: docs-data/overrides.json
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.RP_AWS_CRED_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }}
+      - uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            ,sdlc/prod/github/actions_bot_token
+          parse-json-secrets: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ env.ACTIONS_BOT_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Install tools
+        run: |
+          npx --no-install doc-tools install-test-dependencies
+          rpk connect install
+
+      - name: Regenerate docs with updated overrides
+        run: |
+          npx --no-install doc-tools generate rpcn-connector-docs \
+            --fetch-connectors \
+            --overrides $DOCS_OVERRIDES \
+            --draft-missing \
+            --update-whats-new
+
+      - name: Commit and push if changed
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          if git diff --quiet && git diff --staged --quiet; then
+            echo "No changes after regeneration"
+          else
+            git add -A
+            git commit -m "docs: Regenerate with updated overrides"
+
+            # Use --force-with-lease to avoid overwriting concurrent changes
+            if ! git push --force-with-lease; then
+              echo "Push failed, fetching and rebasing..."
+              git fetch origin ${{ github.ref_name }}
+              git rebase origin/${{ github.ref_name }}
+              git push --force-with-lease
+            fi
+          fi

.github/workflows/test-cookbooks.yaml

@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        category: [jira, enrichments, filtering]
+        category: [jira]
       fail-fast: false
 
     steps:

.github/workflows/test-mcp-examples.yaml

@@ -5,16 +5,16 @@ on:
     branches: [main]
     paths:
       - 'modules/ai-agents/examples/**/*.yaml'
-      - 'modules/ai-agents/examples/test-mcp-examples.sh'
+      - 'modules/ai-agents/examples/test-all.sh'
   pull_request:
     branches: [main]
     paths:
       - 'modules/ai-agents/examples/**/*.yaml'
-      - 'modules/ai-agents/examples/test-mcp-examples.sh'
+      - 'modules/ai-agents/examples/test-all.sh'
 
 jobs:
-  lint-and-test:
-    name: Lint and Test All Examples
+  test-all:
+    name: Test All Examples
     runs-on: ubuntu-latest
 
     steps:
@@ -35,21 +35,16 @@ jobs:
           rpk connect install
 
       - name: Make test script executable
-        run: chmod +x modules/ai-agents/examples/test-mcp-examples.sh
+        run: chmod +x modules/ai-agents/examples/test-all.sh
 
-      # Test all examples
-      - name: Run automated test script
+      - name: Run comprehensive test suite
         run: |
           cd modules/ai-agents/examples
-          ./test-mcp-examples.sh
+          ./test-all.sh
 
-  test-matrix:
-    name: Test Examples by Component Type
+  test-mcp-tools:
+    name: Test MCP Tools
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        component: [processors, inputs, outputs, caches, o11y]
-      fail-fast: false
 
     steps:
       - name: Checkout code
@@ -58,20 +53,63 @@ jobs:
       - name: Install dependencies
         run: npm install
 
-      - name: Install yq
+      - name: Install tools
         run: |
-          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
-          sudo chmod +x /usr/local/bin/yq
+          npx doc-tools install-test-dependencies
+          rpk connect install
+
+      - name: Make test script executable
+        run: chmod +x modules/ai-agents/examples/test-all.sh
+
+      - name: Test MCP tool definitions
+        run: |
+          cd modules/ai-agents/examples
+          ./test-all.sh --mcp-only
+
+  test-pipelines:
+    name: Test Pipelines
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Install tools
+        run: |
+          npx doc-tools install-test-dependencies
+          rpk connect install
+
+      - name: Make test script executable
+        run: chmod +x modules/ai-agents/examples/test-all.sh
+
+      - name: Test pipeline examples
+        run: |
+          cd modules/ai-agents/examples
+          ./test-all.sh --pipelines
+
+  test-snippets:
+    name: Test Snippets
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: npm install
 
       - name: Install tools
         run: |
           npx doc-tools install-test-dependencies
           rpk connect install
 
       - name: Make test script executable
-        run: chmod +x modules/ai-agents/examples/test-mcp-examples.sh
+        run: chmod +x modules/ai-agents/examples/test-all.sh
 
-      - name: Test ${{ matrix.component }} examples
+      - name: Test config snippets
         run: |
           cd modules/ai-agents/examples
-          ./test-mcp-examples.sh ${{ matrix.component }}
+          ./test-all.sh --snippets

.github/workflows/update-docs.yml

@@ -13,13 +13,15 @@ jobs:
       id-token: write
       contents: read
 
+    # Allow concurrent runs
     concurrency:
       group: update-rpcn-connector-docs
-      cancel-in-progress: true
+      cancel-in-progress: false
 
     env:
-      NODE_VERSION: '22'
+      NODE_VERSION: '24'
       DOCS_OVERRIDES: docs-data/overrides.json
+      AUTO_BRANCH: auto-docs/update-rpcn-connector-docs
 
     steps:
       - uses: aws-actions/configure-aws-credentials@v4
@@ -36,6 +38,58 @@ jobs:
         uses: actions/checkout@v4
         with:
           token: ${{ env.ACTIONS_BOT_TOKEN }}
+          fetch-depth: 0  # Full history for rebasing
+
+      - name: Check for existing PR branch and set up workspace
+        id: setup
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Check if the auto-docs branch exists on remote
+          if git ls-remote --heads origin $AUTO_BRANCH | grep -q "$AUTO_BRANCH"; then
+            echo "branch_exists=true" >> "$GITHUB_OUTPUT"
+            echo "Found existing branch $AUTO_BRANCH"
+
+            # Fetch and checkout the existing branch
+            git fetch origin $AUTO_BRANCH
+            git checkout $AUTO_BRANCH
+
+            # Rebase on main to get latest changes
+            if git rebase origin/main; then
+              echo "rebase_success=true" >> "$GITHUB_OUTPUT"
+              echo "Successfully rebased on main"
+            else
+              echo "rebase_success=false" >> "$GITHUB_OUTPUT"
+              echo "Rebase failed - will handle conflicts"
+              git rebase --abort
+              # Stay on the branch but note we couldn't rebase
+            fi
+
+            # Check for manual commits (commits not from the bot)
+            MANUAL_COMMITS=$(git log origin/main..$AUTO_BRANCH --author="^(?!github-actions).*" --perl-regexp --oneline | head -20)
+            if [ -n "$MANUAL_COMMITS" ]; then
+              echo "has_manual_commits=true" >> "$GITHUB_OUTPUT"
+              echo "Found manual commits:"
+              echo "$MANUAL_COMMITS"
+              # Save the list for the PR body
+              {
+                echo "manual_commits<<EOF"
+                echo "$MANUAL_COMMITS"
+                echo "EOF"
+              } >> "$GITHUB_OUTPUT"
+            else
+              echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "branch_exists=false" >> "$GITHUB_OUTPUT"
+            echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
+            echo "rebase_success=true" >> "$GITHUB_OUTPUT"
+            echo "Branch $AUTO_BRANCH does not exist, will create new"
+
+            # Create new branch from main
+            git checkout -b $AUTO_BRANCH
+          fi
 
       - name: Set up Node.js (with npm cache)
         uses: actions/setup-node@v3
@@ -77,25 +131,92 @@ jobs:
 
           rm -f full_output.txt delta_report.md
 
-      - name: Check if branch already exists
-        id: check_branch
+      - name: Check for changes
+        id: changes
         run: |
-          if git ls-remote --heads origin auto-docs/update-rpcn-connector-docs | grep -q auto-docs; then
-            echo "exists=true" >> "$GITHUB_OUTPUT"
-            echo "Branch auto-docs/update-rpcn-connector-docs already exists. Skipping PR creation to preserve manual changes."
+          if git diff --quiet && git diff --staged --quiet; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+            echo "No changes detected"
           else
-            echo "exists=false" >> "$GITHUB_OUTPUT"
-            echo "Branch does not exist. Will create new PR."
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "Changes detected:"
+            git status --short
           fi
 
-      - name: Create Pull Request
-        if: steps.check_branch.outputs.exists == 'false'
+      - name: Commit and push changes
+        if: steps.changes.outputs.has_changes == 'true'
+        id: commit
+        run: |
+          git add -A
+
+          # Create commit message with timestamp for tracking
+          TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC")
+          git commit -m "docs: Update RPCN connector docs
+
+          Auto-generated at $TIMESTAMP"
+
+          # Push to the branch
+          if git push origin $AUTO_BRANCH --force-with-lease; then
+            echo "push_success=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "push_success=false" >> "$GITHUB_OUTPUT"
+            echo "Push failed - likely a concurrent update. Will retry."
+
+            # Fetch latest and try to rebase our commit
+            git fetch origin $AUTO_BRANCH
+            if git rebase origin/$AUTO_BRANCH; then
+              git push origin $AUTO_BRANCH --force-with-lease
+              echo "push_success=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "push_success=false" >> "$GITHUB_OUTPUT"
+              git rebase --abort
+            fi
+          fi
+
+      - name: Create or Update Pull Request
+        if: steps.changes.outputs.has_changes == 'true' && steps.commit.outputs.push_success == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ env.ACTIONS_BOT_TOKEN }}
           base: main
-          branch: auto-docs/update-rpcn-connector-docs
+          branch: ${{ env.AUTO_BRANCH }}
           title: 'auto-docs: Update RPCN connector docs'
           commit-message: 'docs: Update RPCN connector docs'
-          body: ${{ steps.generate.outputs.delta_report }}
+          body: |
+            ${{ steps.generate.outputs.delta_report }}
+
+            ${{ steps.setup.outputs.has_manual_commits == 'true' && format('
+            ---
+
+            ⚠️ **This PR contains manual commits that have been preserved:**
+
+            ```
+            {0}
+            ```
+
+            The automation has added new changes on top of these manual edits.
+            ', steps.setup.outputs.manual_commits) || '' }}
           labels: auto-docs
+          # Don't delete branch on merge - let the normal PR process handle it
+          delete-branch: false
+
+      - name: Handle no changes
+        if: steps.changes.outputs.has_changes == 'false'
+        run: |
+          echo "No documentation changes detected. This could mean:"
+          echo "- The connector docs are already up to date"
+          echo "- The generation produced identical output"
+          echo ""
+          if [ "${{ steps.setup.outputs.branch_exists }}" == "true" ]; then
+            echo "Existing PR branch preserved with any manual changes intact."
+          fi
+
+      - name: Handle push failure
+        if: steps.changes.outputs.has_changes == 'true' && steps.commit.outputs.push_success == 'false'
+        run: |
+          echo "::error::Failed to push changes after retry. This usually means:"
+          echo "- Another automation run is in progress"
+          echo "- There are unresolvable conflicts"
+          echo ""
+          echo "Manual intervention may be required."
+          exit 1