Skip to content

Commit 50654c1

Browse files
sundargthbSundar Raghavanjesseturner21
authored
ci: add backward compatibility integration test workflow (#357)
* ci: add workflow to run integration tests against previous release version * ci: add backward compatibility integration test workflow --------- Co-authored-by: Sundar Raghavan <sdraghav@amazon.com> Co-authored-by: Jesse Turner <57651174+jesseturner21@users.noreply.github.com>
1 parent 38425ec commit 50654c1

1 file changed

Lines changed: 263 additions & 0 deletions

File tree

‎.github/workflows/integration-testing-regression.yml‎

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
# .github/workflows/integration-testing-compat.yml
2+
#
3+
# Backward-compatibility regression check.
4+
#
5+
# ┌─────────────────────────────────────────────────────────────────┐
6+
# │ Library under test │ PR branch source (built from source) │
7+
# │ Test suite │ Previous release tag (e.g. v1.4.5) │
8+
# └─────────────────────────────────────────────────────────────────┘
9+
#
10+
# Question: "Does our new code silently break contracts that the
11+
# previous release's tests defined?"
12+
#
13+
# If a test that passed in v1.4.5 now fails against the PR branch,
14+
# that is a real regression — the new code broke an established contract.
15+
#
16+
# SECURITY MODEL (simplified):
17+
# Only runs on PRs from within the same repository (not forks).
18+
# This avoids the full collaborator-check + environment-approval gate
19+
# from integration-testing.yml while keeping AWS OIDC credentials safe —
20+
# fork PRs never have access to repo secrets or OIDC tokens on
21+
# pull_request triggers, so the fork check is the only guard needed.
22+
#
23+
# For fork contributors, a maintainer can trigger this manually via
24+
# workflow_dispatch after reviewing the PR.
25+
#
26+
# RESULTS:
27+
# Jobs are continue-on-error: true — failures are informational, not
28+
# blocking. A failure means the new code broke a previously-passing
29+
# test and warrants investigation before merge.
30+
31+
name: Integration Tests (Backward Compat)
32+
33+
on:
34+
pull_request:
35+
branches: [main]
36+
workflow_dispatch:
37+
inputs:
38+
tag:
39+
description: 'Release tag to use as the test suite baseline (e.g. v1.4.5). Leave empty for latest tag.'
40+
required: false
41+
type: string
42+
43+
permissions:
44+
contents: read
45+
46+
jobs:
47+
# ── Guard: skip fork PRs ────────────────────────────────────────────────────
48+
# Fork PRs cannot access repo secrets or OIDC tokens on pull_request triggers,
49+
# so they would fail at the AWS credentials step anyway. Failing fast here
50+
# gives a clearer signal than a cryptic credentials error.
51+
preflight:
52+
name: Preflight Check
53+
runs-on: ubuntu-latest
54+
outputs:
55+
should-run: ${{ steps.fork-check.outputs.result }}
56+
steps:
57+
- name: Check for fork PR
58+
id: fork-check
59+
uses: actions/github-script@v8
60+
with:
61+
result-encoding: string
62+
script: |
63+
if (!context.payload.pull_request) {
64+
// workflow_dispatch — always run
65+
return 'true';
66+
}
67+
const isFork =
68+
context.payload.pull_request.head.repo.full_name !== context.repo.owner + '/' + context.repo.repo;
69+
if (isFork) {
70+
console.log(
71+
'Fork PR detected. Skipping compat tests. ' +
72+
'A maintainer can trigger these manually via workflow_dispatch.'
73+
);
74+
return 'false';
75+
}
76+
console.log('✓ Internal PR — proceeding.');
77+
return 'true';
78+
79+
# ── Resolve the baseline tag ────────────────────────────────────────────────
80+
# Finds the most recent semver release tag in the repository.
81+
# For a PR in progress, the latest tag IS the previous release —
82+
# the current changes have not been tagged yet.
83+
resolve-tag:
84+
name: Resolve Baseline Tag
85+
runs-on: ubuntu-latest
86+
needs: preflight
87+
if: needs.preflight.outputs.should-run == 'true'
88+
outputs:
89+
tag: ${{ steps.resolve.outputs.tag }}
90+
steps:
91+
- name: Checkout (for tag resolution)
92+
uses: actions/checkout@v6
93+
with:
94+
fetch-depth: 0 # full history needed to see all tags
95+
fetch-tags: true
96+
97+
- name: Resolve tag
98+
id: resolve
99+
env:
100+
INPUT_TAG: ${{ github.event.inputs.tag }}
101+
run: |
102+
if [ -n "$INPUT_TAG" ]; then
103+
echo "Using manually specified tag: $INPUT_TAG"
104+
# Verify the tag actually exists
105+
if ! git rev-parse "$INPUT_TAG" > /dev/null 2>&1; then
106+
echo "❌ Tag '$INPUT_TAG' does not exist in this repository"
107+
exit 1
108+
fi
109+
echo "tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
110+
else
111+
# Latest semver tag — this is the previous published release.
112+
# --sort=-version:refname sorts tags as semantic versions descending.
113+
LATEST_TAG=$(git tag --sort=-version:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
114+
if [ -z "$LATEST_TAG" ]; then
115+
echo "❌ No semver release tags found (expected format: vX.Y.Z)"
116+
exit 1
117+
fi
118+
echo "Resolved baseline tag: $LATEST_TAG"
119+
echo "tag=$LATEST_TAG" >> "$GITHUB_OUTPUT"
120+
fi
121+
122+
# ── Compatibility tests ─────────────────────────────────────────────────────
123+
integ-test-compat:
124+
name: Compat (${{ matrix.group }}) — new code vs ${{ needs.resolve-tag.outputs.tag }} tests
125+
runs-on: ubuntu-latest
126+
needs: [preflight, resolve-tag]
127+
if: needs.preflight.outputs.should-run == 'true'
128+
129+
# Informational — failures mean regressions worth investigating,
130+
# not hard blockers. Shows as a yellow warning icon in the Actions UI.
131+
continue-on-error: true
132+
133+
permissions:
134+
id-token: write # Required for AWS OIDC
135+
contents: read
136+
137+
strategy:
138+
fail-fast: false
139+
matrix:
140+
include:
141+
- group: runtime
142+
path: tests_integ/runtime
143+
timeout: 10
144+
extra-deps: ""
145+
- group: memory
146+
path: >-
147+
tests_integ/memory/test_controlplane.py
148+
tests_integ/memory/test_memory_client.py
149+
tests_integ/memory/integrations/test_session_manager.py
150+
timeout: 10
151+
extra-deps: ""
152+
- group: evaluation
153+
path: tests_integ/evaluation
154+
timeout: 15
155+
extra-deps: "strands-agents-evals"
156+
- group: services
157+
path: tests_integ/services
158+
timeout: 5
159+
extra-deps: ""
160+
161+
steps:
162+
- name: Configure AWS Credentials
163+
uses: aws-actions/configure-aws-credentials@v6
164+
with:
165+
role-to-assume: ${{ secrets.AGENTCORE_INTEG_TEST_ROLE }}
166+
aws-region: us-west-2
167+
mask-aws-account-id: true
168+
169+
# Check out the PR branch — we need the full source to build the library.
170+
- name: Checkout PR branch (library source)
171+
uses: actions/checkout@v6
172+
with:
173+
ref: ${{ github.event.pull_request.head.sha || github.sha }}
174+
fetch-depth: 0
175+
fetch-tags: true
176+
persist-credentials: false
177+
178+
- name: Set up Python
179+
uses: actions/setup-python@v6
180+
with:
181+
python-version: '3.10'
182+
183+
# Build and install the library from the PR branch source.
184+
# This is the "new code" under test.
185+
- name: Build and install library from PR branch
186+
run: |
187+
pip install build
188+
python -m build --wheel --outdir dist/
189+
WHEEL=$(ls dist/*.whl | head -1)
190+
echo "Installing wheel: $WHEEL"
191+
pip install "$WHEEL"
192+
193+
INSTALLED=$(pip show bedrock-agentcore | grep '^Version' | awk '{print $2}')
194+
echo "✓ Installed bedrock-agentcore==$INSTALLED from PR branch source"
195+
196+
# Extract the test suite from the previous release tag using git archive.
197+
# We get exactly what shipped in that release — no drift, no local edits.
198+
# Only tests_integ/ is extracted to keep the working directory clean.
199+
- name: Extract test suite from ${{ needs.resolve-tag.outputs.tag }}
200+
env:
201+
BASELINE_TAG: ${{ needs.resolve-tag.outputs.tag }}
202+
run: |
203+
mkdir -p /tmp/baseline-tests
204+
echo "Extracting tests_integ/ from $BASELINE_TAG..."
205+
git archive "$BASELINE_TAG" tests_integ/ | tar -x -C /tmp/baseline-tests/
206+
207+
if [ ! -d "/tmp/baseline-tests/tests_integ" ]; then
208+
echo "❌ tests_integ/ not found in tag $BASELINE_TAG"
209+
echo "Verify the tag contains this directory: git ls-tree $BASELINE_TAG tests_integ/"
210+
exit 1
211+
fi
212+
213+
echo "✓ Extracted test suite from $BASELINE_TAG"
214+
echo "Test files found:"
215+
find /tmp/baseline-tests/tests_integ -name "*.py" | head -20
216+
217+
# Install test dependencies only — NOT the library (already installed above).
218+
- name: Install test dependencies
219+
run: |
220+
pip install --no-cache-dir \
221+
pytest pytest-xdist pytest-order \
222+
requests strands-agents uvicorn httpx starlette websockets \
223+
${{ matrix.extra-deps }}
224+
225+
# Run the old test suite against the new library.
226+
# A failure here means a test that passed in the previous release
227+
# now fails against the PR branch code — a genuine regression.
228+
- name: Run ${{ needs.resolve-tag.outputs.tag }} tests against PR branch code
229+
env:
230+
AWS_REGION: us-west-2
231+
PYTHONUNBUFFERED: "1"
232+
MEMORY_KINESIS_ARN: ${{ secrets.MEMORY_KINESIS_ARN }}
233+
MEMORY_ROLE_ARN: ${{ secrets.MEMORY_ROLE_ARN }}
234+
MEMORY_PREPOPULATED_ID: ${{ secrets.MEMORY_PREPOPULATED_ID }}
235+
RESOURCE_POLICY_TEST_ARN: ${{ secrets.RESOURCE_POLICY_TEST_ARN }}
236+
RESOURCE_POLICY_TEST_PRINCIPAL: ${{ secrets.RESOURCE_POLICY_TEST_PRINCIPAL }}
237+
BASELINE_TAG: ${{ needs.resolve-tag.outputs.tag }}
238+
timeout-minutes: ${{ matrix.timeout }}
239+
working-directory: /tmp/baseline-tests
240+
run: |
241+
echo "================================================"
242+
echo "Library: bedrock-agentcore (PR branch source)"
243+
echo "Test suite: $BASELINE_TAG tests_integ/${{ matrix.group }}"
244+
echo "================================================"
245+
pytest ${{ matrix.path }} \
246+
-n auto --dist=loadscope \
247+
-s --log-cli-level=INFO
248+
249+
# ── Skip notice for fork PRs ────────────────────────────────────────────────
250+
fork-skip-notice:
251+
name: Compat tests skipped (fork PR)
252+
runs-on: ubuntu-latest
253+
needs: preflight
254+
if: needs.preflight.outputs.should-run == 'false'
255+
permissions: {}
256+
steps:
257+
- name: Explain skip
258+
run: |
259+
echo "ℹ️ Backward-compat integration tests were skipped for this fork PR."
260+
echo ""
261+
echo "AWS credentials are not available to fork PRs for security reasons."
262+
echo "A maintainer can run these tests manually via:"
263+
echo " Actions → Integration Tests (Backward Compat) → Run workflow"

0 commit comments

Comments
 (0)