Skip to content

Commit f5f9699

Browse files
JakeSCahillcoderabbitai[bot]micheleRP
authored
Improve MCP developer experience (#348)
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Michele Cyran <michele@redpanda.com>
1 parent f565ac6 commit f5f9699

179 files changed

Lines changed: 27440 additions & 3083 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/regenerate-on-overrides.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: Regenerate Docs on Override Changes
2+
3+
on:
4+
push:
5+
paths:
6+
- 'docs-data/overrides.json'
7+
branches-ignore:
8+
- main
9+
10+
jobs:
11+
regenerate-docs:
12+
runs-on: ubuntu-latest
13+
14+
permissions:
15+
id-token: write
16+
contents: read
17+
18+
env:
19+
NODE_VERSION: '24'
20+
DOCS_OVERRIDES: docs-data/overrides.json
21+
22+
steps:
23+
- uses: aws-actions/configure-aws-credentials@v4
24+
with:
25+
aws-region: ${{ vars.RP_AWS_CRED_REGION }}
26+
role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }}
27+
- uses: aws-actions/aws-secretsmanager-get-secrets@v2
28+
with:
29+
secret-ids: |
30+
,sdlc/prod/github/actions_bot_token
31+
parse-json-secrets: true
32+
33+
- name: Checkout repository
34+
uses: actions/checkout@v4
35+
with:
36+
token: ${{ env.ACTIONS_BOT_TOKEN }}
37+
38+
- name: Set up Node.js
39+
uses: actions/setup-node@v3
40+
with:
41+
node-version: ${{ env.NODE_VERSION }}
42+
cache: 'npm'
43+
44+
- name: Install dependencies
45+
run: npm install
46+
47+
- name: Install tools
48+
run: |
49+
npx --no-install doc-tools install-test-dependencies
50+
rpk connect install
51+
52+
- name: Regenerate docs with updated overrides
53+
run: |
54+
npx --no-install doc-tools generate rpcn-connector-docs \
55+
--fetch-connectors \
56+
--overrides $DOCS_OVERRIDES \
57+
--draft-missing \
58+
--update-whats-new
59+
60+
- name: Commit and push if changed
61+
run: |
62+
git config user.name "github-actions[bot]"
63+
git config user.email "github-actions[bot]@users.noreply.github.com"
64+
65+
if git diff --quiet && git diff --staged --quiet; then
66+
echo "No changes after regeneration"
67+
else
68+
git add -A
69+
git commit -m "docs: Regenerate with updated overrides"
70+
71+
# Use --force-with-lease to avoid overwriting concurrent changes
72+
if ! git push --force-with-lease; then
73+
echo "Push failed, fetching and rebasing..."
74+
git fetch origin ${{ github.ref_name }}
75+
git rebase origin/${{ github.ref_name }}
76+
git push --force-with-lease
77+
fi
78+
fi

‎.github/workflows/test-cookbooks.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
runs-on: ubuntu-latest
4040
strategy:
4141
matrix:
42-
category: [jira, enrichments, filtering]
42+
category: [jira]
4343
fail-fast: false
4444

4545
steps:

‎.github/workflows/test-mcp-examples.yaml‎

Lines changed: 58 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,16 @@ on:
55
branches: [main]
66
paths:
77
- 'modules/ai-agents/examples/**/*.yaml'
8-
- 'modules/ai-agents/examples/test-mcp-examples.sh'
8+
- 'modules/ai-agents/examples/test-all.sh'
99
pull_request:
1010
branches: [main]
1111
paths:
1212
- 'modules/ai-agents/examples/**/*.yaml'
13-
- 'modules/ai-agents/examples/test-mcp-examples.sh'
13+
- 'modules/ai-agents/examples/test-all.sh'
1414

1515
jobs:
16-
lint-and-test:
17-
name: Lint and Test All Examples
16+
test-all:
17+
name: Test All Examples
1818
runs-on: ubuntu-latest
1919

2020
steps:
@@ -35,21 +35,16 @@ jobs:
3535
rpk connect install
3636
3737
- name: Make test script executable
38-
run: chmod +x modules/ai-agents/examples/test-mcp-examples.sh
38+
run: chmod +x modules/ai-agents/examples/test-all.sh
3939

40-
# Test all examples
41-
- name: Run automated test script
40+
- name: Run comprehensive test suite
4241
run: |
4342
cd modules/ai-agents/examples
44-
./test-mcp-examples.sh
43+
./test-all.sh
4544
46-
test-matrix:
47-
name: Test Examples by Component Type
45+
test-mcp-tools:
46+
name: Test MCP Tools
4847
runs-on: ubuntu-latest
49-
strategy:
50-
matrix:
51-
component: [processors, inputs, outputs, caches, o11y]
52-
fail-fast: false
5348

5449
steps:
5550
- name: Checkout code
@@ -58,20 +53,63 @@ jobs:
5853
- name: Install dependencies
5954
run: npm install
6055

61-
- name: Install yq
56+
- name: Install tools
6257
run: |
63-
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
64-
sudo chmod +x /usr/local/bin/yq
58+
npx doc-tools install-test-dependencies
59+
rpk connect install
60+
61+
- name: Make test script executable
62+
run: chmod +x modules/ai-agents/examples/test-all.sh
63+
64+
- name: Test MCP tool definitions
65+
run: |
66+
cd modules/ai-agents/examples
67+
./test-all.sh --mcp-only
68+
69+
test-pipelines:
70+
name: Test Pipelines
71+
runs-on: ubuntu-latest
72+
73+
steps:
74+
- name: Checkout code
75+
uses: actions/checkout@v4
76+
77+
- name: Install dependencies
78+
run: npm install
79+
80+
- name: Install tools
81+
run: |
82+
npx doc-tools install-test-dependencies
83+
rpk connect install
84+
85+
- name: Make test script executable
86+
run: chmod +x modules/ai-agents/examples/test-all.sh
87+
88+
- name: Test pipeline examples
89+
run: |
90+
cd modules/ai-agents/examples
91+
./test-all.sh --pipelines
92+
93+
test-snippets:
94+
name: Test Snippets
95+
runs-on: ubuntu-latest
96+
97+
steps:
98+
- name: Checkout code
99+
uses: actions/checkout@v4
100+
101+
- name: Install dependencies
102+
run: npm install
65103

66104
- name: Install tools
67105
run: |
68106
npx doc-tools install-test-dependencies
69107
rpk connect install
70108
71109
- name: Make test script executable
72-
run: chmod +x modules/ai-agents/examples/test-mcp-examples.sh
110+
run: chmod +x modules/ai-agents/examples/test-all.sh
73111

74-
- name: Test ${{ matrix.component }} examples
112+
- name: Test config snippets
75113
run: |
76114
cd modules/ai-agents/examples
77-
./test-mcp-examples.sh ${{ matrix.component }}
115+
./test-all.sh --snippets

‎.github/workflows/update-docs.yml‎

Lines changed: 134 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ jobs:
1313
id-token: write
1414
contents: read
1515

16+
# Allow concurrent runs
1617
concurrency:
1718
group: update-rpcn-connector-docs
18-
cancel-in-progress: true
19+
cancel-in-progress: false
1920

2021
env:
21-
NODE_VERSION: '22'
22+
NODE_VERSION: '24'
2223
DOCS_OVERRIDES: docs-data/overrides.json
24+
AUTO_BRANCH: auto-docs/update-rpcn-connector-docs
2325

2426
steps:
2527
- uses: aws-actions/configure-aws-credentials@v4
@@ -36,6 +38,58 @@ jobs:
3638
uses: actions/checkout@v4
3739
with:
3840
token: ${{ env.ACTIONS_BOT_TOKEN }}
41+
fetch-depth: 0 # Full history for rebasing
42+
43+
- name: Check for existing PR branch and set up workspace
44+
id: setup
45+
run: |
46+
git config user.name "github-actions[bot]"
47+
git config user.email "github-actions[bot]@users.noreply.github.com"
48+
49+
# Check if the auto-docs branch exists on remote
50+
if git ls-remote --heads origin $AUTO_BRANCH | grep -q "$AUTO_BRANCH"; then
51+
echo "branch_exists=true" >> "$GITHUB_OUTPUT"
52+
echo "Found existing branch $AUTO_BRANCH"
53+
54+
# Fetch and checkout the existing branch
55+
git fetch origin $AUTO_BRANCH
56+
git checkout $AUTO_BRANCH
57+
58+
# Rebase on main to get latest changes
59+
if git rebase origin/main; then
60+
echo "rebase_success=true" >> "$GITHUB_OUTPUT"
61+
echo "Successfully rebased on main"
62+
else
63+
echo "rebase_success=false" >> "$GITHUB_OUTPUT"
64+
echo "Rebase failed - will handle conflicts"
65+
git rebase --abort
66+
# Stay on the branch but note we couldn't rebase
67+
fi
68+
69+
# Check for manual commits (commits not from the bot)
70+
MANUAL_COMMITS=$(git log origin/main..$AUTO_BRANCH --author="^(?!github-actions).*" --perl-regexp --oneline | head -20)
71+
if [ -n "$MANUAL_COMMITS" ]; then
72+
echo "has_manual_commits=true" >> "$GITHUB_OUTPUT"
73+
echo "Found manual commits:"
74+
echo "$MANUAL_COMMITS"
75+
# Save the list for the PR body
76+
{
77+
echo "manual_commits<<EOF"
78+
echo "$MANUAL_COMMITS"
79+
echo "EOF"
80+
} >> "$GITHUB_OUTPUT"
81+
else
82+
echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
83+
fi
84+
else
85+
echo "branch_exists=false" >> "$GITHUB_OUTPUT"
86+
echo "has_manual_commits=false" >> "$GITHUB_OUTPUT"
87+
echo "rebase_success=true" >> "$GITHUB_OUTPUT"
88+
echo "Branch $AUTO_BRANCH does not exist, will create new"
89+
90+
# Create new branch from main
91+
git checkout -b $AUTO_BRANCH
92+
fi
3993
4094
- name: Set up Node.js (with npm cache)
4195
uses: actions/setup-node@v3
@@ -95,25 +149,92 @@ jobs:
95149
96150
rm -f full_output.txt pr_summary.txt
97151
98-
- name: Check if branch already exists
99-
id: check_branch
152+
- name: Check for changes
153+
id: changes
100154
run: |
101-
if git ls-remote --heads origin auto-docs/update-rpcn-connector-docs | grep -q auto-docs; then
102-
echo "exists=true" >> "$GITHUB_OUTPUT"
103-
echo "Branch auto-docs/update-rpcn-connector-docs already exists. Skipping PR creation to preserve manual changes."
155+
if git diff --quiet && git diff --staged --quiet; then
156+
echo "has_changes=false" >> "$GITHUB_OUTPUT"
157+
echo "No changes detected"
104158
else
105-
echo "exists=false" >> "$GITHUB_OUTPUT"
106-
echo "Branch does not exist. Will create new PR."
159+
echo "has_changes=true" >> "$GITHUB_OUTPUT"
160+
echo "Changes detected:"
161+
git status --short
107162
fi
108163
109-
- name: Create Pull Request
110-
if: steps.check_branch.outputs.exists == 'false'
164+
- name: Commit and push changes
165+
if: steps.changes.outputs.has_changes == 'true'
166+
id: commit
167+
run: |
168+
git add -A
169+
170+
# Create commit message with timestamp for tracking
171+
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC")
172+
git commit -m "docs: Update RPCN connector docs
173+
174+
Auto-generated at $TIMESTAMP"
175+
176+
# Push to the branch
177+
if git push origin $AUTO_BRANCH --force-with-lease; then
178+
echo "push_success=true" >> "$GITHUB_OUTPUT"
179+
else
180+
echo "push_success=false" >> "$GITHUB_OUTPUT"
181+
echo "Push failed - likely a concurrent update. Will retry."
182+
183+
# Fetch latest and try to rebase our commit
184+
git fetch origin $AUTO_BRANCH
185+
if git rebase origin/$AUTO_BRANCH; then
186+
git push origin $AUTO_BRANCH --force-with-lease
187+
echo "push_success=true" >> "$GITHUB_OUTPUT"
188+
else
189+
echo "push_success=false" >> "$GITHUB_OUTPUT"
190+
git rebase --abort
191+
fi
192+
fi
193+
194+
- name: Create or Update Pull Request
195+
if: steps.changes.outputs.has_changes == 'true' && steps.commit.outputs.push_success == 'true'
111196
uses: peter-evans/create-pull-request@v6
112197
with:
113198
token: ${{ env.ACTIONS_BOT_TOKEN }}
114199
base: main
115-
branch: auto-docs/update-rpcn-connector-docs
200+
branch: ${{ env.AUTO_BRANCH }}
116201
title: 'auto-docs: Update RPCN connector docs'
117202
commit-message: 'docs: Update RPCN connector docs'
118-
body: ${{ steps.generate.outputs.delta_report }}
203+
body: |
204+
${{ steps.generate.outputs.delta_report }}
205+
206+
${{ steps.setup.outputs.has_manual_commits == 'true' && format('
207+
---
208+
209+
⚠️ **This PR contains manual commits that have been preserved:**
210+
211+
```
212+
{0}
213+
```
214+
215+
The automation has added new changes on top of these manual edits.
216+
', steps.setup.outputs.manual_commits) || '' }}
119217
labels: auto-docs
218+
# Don't delete branch on merge - let the normal PR process handle it
219+
delete-branch: false
220+
221+
- name: Handle no changes
222+
if: steps.changes.outputs.has_changes == 'false'
223+
run: |
224+
echo "No documentation changes detected. This could mean:"
225+
echo "- The connector docs are already up to date"
226+
echo "- The generation produced identical output"
227+
echo ""
228+
if [ "${{ steps.setup.outputs.branch_exists }}" == "true" ]; then
229+
echo "Existing PR branch preserved with any manual changes intact."
230+
fi
231+
232+
- name: Handle push failure
233+
if: steps.changes.outputs.has_changes == 'true' && steps.commit.outputs.push_success == 'false'
234+
run: |
235+
echo "::error::Failed to push changes after retry. This usually means:"
236+
echo "- Another automation run is in progress"
237+
echo "- There are unresolvable conflicts"
238+
echo ""
239+
echo "Manual intervention may be required."
240+
exit 1

0 commit comments

Comments
 (0)