Skip to content

Commit 3dd2741

Browse files
TheiLLeniumStudiosTheiLLeniumStudios
authored
Merge pull request stakater#1074 from TheiLLeniumStudios/test/loadtests
feat: Add load test framework with observability metrics
2 parents 1652c62 + 16ff7f6 commit 3dd2741

27 files changed

Lines changed: 6839 additions & 109 deletions

File tree

‎.github/actions/loadtest/action.yml‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
name: 'Reloader Load Test'
2+
description: 'Run Reloader load tests with A/B comparison support'
3+
4+
inputs:
5+
old-ref:
6+
description: 'Git ref for "old" version (optional, enables A/B comparison)'
7+
required: false
8+
default: ''
9+
new-ref:
10+
description: 'Git ref for "new" version (defaults to current checkout)'
11+
required: false
12+
default: ''
13+
old-image:
14+
description: 'Pre-built container image for "old" version (alternative to old-ref)'
15+
required: false
16+
default: ''
17+
new-image:
18+
description: 'Pre-built container image for "new" version (alternative to new-ref)'
19+
required: false
20+
default: ''
21+
scenarios:
22+
description: 'Scenarios to run: S1,S4,S6 or all'
23+
required: false
24+
default: 'S1,S4,S6'
25+
test-type:
26+
description: 'Test type label for summary: quick or full'
27+
required: false
28+
default: 'quick'
29+
duration:
30+
description: 'Test duration in seconds'
31+
required: false
32+
default: '60'
33+
kind-cluster:
34+
description: 'Name of existing Kind cluster (if empty, creates new one)'
35+
required: false
36+
default: ''
37+
post-comment:
38+
description: 'Post results as PR comment'
39+
required: false
40+
default: 'false'
41+
pr-number:
42+
description: 'PR number for commenting (required if post-comment is true)'
43+
required: false
44+
default: ''
45+
github-token:
46+
description: 'GitHub token for posting comments'
47+
required: false
48+
default: ${{ github.token }}
49+
comment-header:
50+
description: 'Optional header text for the comment'
51+
required: false
52+
default: ''
53+
54+
outputs:
55+
status:
56+
description: 'Overall test status: pass or fail'
57+
value: ${{ steps.run.outputs.status }}
58+
summary:
59+
description: 'Markdown summary of results'
60+
value: ${{ steps.summary.outputs.summary }}
61+
pass-count:
62+
description: 'Number of passed scenarios'
63+
value: ${{ steps.summary.outputs.pass_count }}
64+
fail-count:
65+
description: 'Number of failed scenarios'
66+
value: ${{ steps.summary.outputs.fail_count }}
67+
68+
runs:
69+
using: 'composite'
70+
steps:
71+
- name: Determine images to use
72+
id: images
73+
shell: bash
74+
run: |
75+
# Determine old image
76+
if [ -n "${{ inputs.old-image }}" ]; then
77+
echo "old=${{ inputs.old-image }}" >> $GITHUB_OUTPUT
78+
elif [ -n "${{ inputs.old-ref }}" ]; then
79+
echo "old=localhost/reloader:old" >> $GITHUB_OUTPUT
80+
echo "build_old=true" >> $GITHUB_OUTPUT
81+
else
82+
echo "old=" >> $GITHUB_OUTPUT
83+
fi
84+
85+
# Determine new image
86+
if [ -n "${{ inputs.new-image }}" ]; then
87+
echo "new=${{ inputs.new-image }}" >> $GITHUB_OUTPUT
88+
elif [ -n "${{ inputs.new-ref }}" ]; then
89+
echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT
90+
echo "build_new=true" >> $GITHUB_OUTPUT
91+
else
92+
# Default: build from current checkout
93+
echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT
94+
echo "build_new_current=true" >> $GITHUB_OUTPUT
95+
fi
96+
97+
- name: Build old image from ref
98+
if: steps.images.outputs.build_old == 'true'
99+
shell: bash
100+
run: |
101+
CURRENT_SHA=$(git rev-parse HEAD)
102+
git checkout ${{ inputs.old-ref }}
103+
docker build -t localhost/reloader:old .
104+
echo "Built old image from ref: ${{ inputs.old-ref }}"
105+
git checkout $CURRENT_SHA
106+
107+
- name: Build new image from ref
108+
if: steps.images.outputs.build_new == 'true'
109+
shell: bash
110+
run: |
111+
CURRENT_SHA=$(git rev-parse HEAD)
112+
git checkout ${{ inputs.new-ref }}
113+
docker build -t localhost/reloader:new .
114+
echo "Built new image from ref: ${{ inputs.new-ref }}"
115+
git checkout $CURRENT_SHA
116+
117+
- name: Build new image from current checkout
118+
if: steps.images.outputs.build_new_current == 'true'
119+
shell: bash
120+
run: |
121+
docker build -t localhost/reloader:new .
122+
echo "Built new image from current checkout"
123+
124+
- name: Build loadtest binary
125+
shell: bash
126+
run: |
127+
cd ${{ github.workspace }}/test/loadtest
128+
go build -o loadtest ./cmd/loadtest
129+
130+
- name: Determine cluster name
131+
id: cluster
132+
shell: bash
133+
run: |
134+
if [ -n "${{ inputs.kind-cluster }}" ]; then
135+
echo "name=${{ inputs.kind-cluster }}" >> $GITHUB_OUTPUT
136+
echo "skip=true" >> $GITHUB_OUTPUT
137+
else
138+
echo "name=reloader-loadtest" >> $GITHUB_OUTPUT
139+
echo "skip=false" >> $GITHUB_OUTPUT
140+
fi
141+
142+
- name: Load images into Kind
143+
shell: bash
144+
run: |
145+
CLUSTER="${{ steps.cluster.outputs.name }}"
146+
147+
if [ -n "${{ steps.images.outputs.old }}" ]; then
148+
echo "Loading old image: ${{ steps.images.outputs.old }}"
149+
kind load docker-image "${{ steps.images.outputs.old }}" --name "$CLUSTER" || true
150+
fi
151+
152+
echo "Loading new image: ${{ steps.images.outputs.new }}"
153+
kind load docker-image "${{ steps.images.outputs.new }}" --name "$CLUSTER" || true
154+
155+
- name: Run load tests
156+
id: run
157+
shell: bash
158+
run: |
159+
cd ${{ github.workspace }}/test/loadtest
160+
161+
ARGS="--new-image=${{ steps.images.outputs.new }}"
162+
ARGS="$ARGS --scenario=${{ inputs.scenarios }}"
163+
ARGS="$ARGS --duration=${{ inputs.duration }}"
164+
ARGS="$ARGS --cluster-name=${{ steps.cluster.outputs.name }}"
165+
ARGS="$ARGS --skip-image-load"
166+
167+
if [ -n "${{ steps.images.outputs.old }}" ]; then
168+
ARGS="$ARGS --old-image=${{ steps.images.outputs.old }}"
169+
fi
170+
171+
if [ "${{ steps.cluster.outputs.skip }}" = "true" ]; then
172+
ARGS="$ARGS --skip-cluster"
173+
fi
174+
175+
echo "Running: ./loadtest run $ARGS"
176+
if ./loadtest run $ARGS; then
177+
echo "status=pass" >> $GITHUB_OUTPUT
178+
else
179+
echo "status=fail" >> $GITHUB_OUTPUT
180+
fi
181+
182+
- name: Generate summary
183+
id: summary
184+
shell: bash
185+
run: |
186+
cd ${{ github.workspace }}/test/loadtest
187+
188+
# Generate markdown summary
189+
./loadtest summary \
190+
--results-dir=./results \
191+
--test-type=${{ inputs.test-type }} \
192+
--format=markdown > summary.md 2>/dev/null || true
193+
194+
# Output to GitHub Step Summary
195+
cat summary.md >> $GITHUB_STEP_SUMMARY
196+
197+
# Store summary for output (using heredoc for multiline)
198+
{
199+
echo 'summary<<EOF'
200+
cat summary.md
201+
echo 'EOF'
202+
} >> $GITHUB_OUTPUT
203+
204+
# Get pass/fail counts from JSON
205+
COUNTS=$(./loadtest summary --format=json 2>/dev/null | head -20 || echo '{}')
206+
echo "pass_count=$(echo "$COUNTS" | grep -o '"pass_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT
207+
echo "fail_count=$(echo "$COUNTS" | grep -o '"fail_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT
208+
209+
- name: Post PR comment
210+
if: inputs.post-comment == 'true' && inputs.pr-number != ''
211+
continue-on-error: true
212+
uses: actions/github-script@v7
213+
with:
214+
github-token: ${{ inputs.github-token }}
215+
script: |
216+
const fs = require('fs');
217+
const summaryPath = '${{ github.workspace }}/test/loadtest/summary.md';
218+
let summary = 'No results available';
219+
try {
220+
summary = fs.readFileSync(summaryPath, 'utf8');
221+
} catch (e) {
222+
console.log('Could not read summary file:', e.message);
223+
}
224+
225+
const header = '${{ inputs.comment-header }}';
226+
const status = '${{ steps.run.outputs.status }}';
227+
const statusEmoji = status === 'pass' ? ':white_check_mark:' : ':x:';
228+
229+
const body = [
230+
header ? header : `## ${statusEmoji} Load Test Results (${{ inputs.test-type }})`,
231+
'',
232+
summary,
233+
'',
234+
'---',
235+
`**Artifacts:** [Download](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`,
236+
].join('\n');
237+
238+
try {
239+
await github.rest.issues.createComment({
240+
owner: context.repo.owner,
241+
repo: context.repo.repo,
242+
issue_number: ${{ inputs.pr-number }},
243+
body: body
244+
});
245+
console.log('Comment posted successfully');
246+
} catch (error) {
247+
if (error.status === 403) {
248+
console.log('Could not post comment (fork PR with restricted permissions). Use /loadtest command to run with comment posting.');
249+
} else {
250+
throw error;
251+
}
252+
}
253+
254+
- name: Upload results
255+
uses: actions/upload-artifact@v4
256+
if: always()
257+
with:
258+
name: loadtest-${{ inputs.test-type }}-results
259+
path: |
260+
${{ github.workspace }}/test/loadtest/results/
261+
retention-days: 30
262+
263+
- name: Cleanup Kind cluster (only if we created it)
264+
if: always() && steps.cluster.outputs.skip == 'false'
265+
shell: bash
266+
run: |
267+
kind delete cluster --name ${{ steps.cluster.outputs.name }} || true

‎.github/workflows/loadtest.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Load Test (Full)
2+
3+
on:
4+
issue_comment:
5+
types: [created]
6+
7+
permissions:
8+
contents: read
9+
pull-requests: write
10+
issues: write
11+
12+
jobs:
13+
loadtest:
14+
# Only run on PR comments with /loadtest command
15+
if: |
16+
github.event.issue.pull_request &&
17+
contains(github.event.comment.body, '/loadtest')
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Add reaction to comment
22+
uses: actions/github-script@v7
23+
with:
24+
script: |
25+
await github.rest.reactions.createForIssueComment({
26+
owner: context.repo.owner,
27+
repo: context.repo.repo,
28+
comment_id: context.payload.comment.id,
29+
content: 'rocket'
30+
});
31+
32+
- name: Get PR details
33+
id: pr
34+
uses: actions/github-script@v7
35+
with:
36+
script: |
37+
const pr = await github.rest.pulls.get({
38+
owner: context.repo.owner,
39+
repo: context.repo.repo,
40+
pull_number: context.issue.number
41+
});
42+
core.setOutput('head_ref', pr.data.head.ref);
43+
core.setOutput('head_sha', pr.data.head.sha);
44+
core.setOutput('base_ref', pr.data.base.ref);
45+
core.setOutput('base_sha', pr.data.base.sha);
46+
console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`);
47+
48+
- name: Checkout PR branch
49+
uses: actions/checkout@v4
50+
with:
51+
ref: ${{ steps.pr.outputs.head_sha }}
52+
fetch-depth: 0 # Full history for building from base ref
53+
54+
- name: Set up Go
55+
uses: actions/setup-go@v5
56+
with:
57+
go-version: '1.25'
58+
cache: false
59+
60+
- name: Set up Docker Buildx
61+
uses: docker/setup-buildx-action@v3
62+
63+
- name: Install kind
64+
run: |
65+
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
66+
chmod +x ./kind
67+
sudo mv ./kind /usr/local/bin/kind
68+
69+
- name: Install kubectl
70+
run: |
71+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
72+
chmod +x kubectl
73+
sudo mv kubectl /usr/local/bin/kubectl
74+
75+
- name: Run full A/B comparison load test
76+
id: loadtest
77+
uses: ./.github/actions/loadtest
78+
with:
79+
old-ref: ${{ steps.pr.outputs.base_sha }}
80+
new-ref: ${{ steps.pr.outputs.head_sha }}
81+
scenarios: 'all'
82+
test-type: 'full'
83+
post-comment: 'true'
84+
pr-number: ${{ github.event.issue.number }}
85+
comment-header: |
86+
## Load Test Results (Full A/B Comparison)
87+
**Comparing:** `${{ steps.pr.outputs.base_ref }}` → `${{ steps.pr.outputs.head_ref }}`
88+
**Triggered by:** @${{ github.event.comment.user.login }}
89+
90+
- name: Add success reaction
91+
if: steps.loadtest.outputs.status == 'pass'
92+
uses: actions/github-script@v7
93+
with:
94+
script: |
95+
await github.rest.reactions.createForIssueComment({
96+
owner: context.repo.owner,
97+
repo: context.repo.repo,
98+
comment_id: context.payload.comment.id,
99+
content: '+1'
100+
});
101+
102+
- name: Add failure reaction
103+
if: steps.loadtest.outputs.status == 'fail'
104+
uses: actions/github-script@v7
105+
with:
106+
script: |
107+
await github.rest.reactions.createForIssueComment({
108+
owner: context.repo.owner,
109+
repo: context.repo.repo,
110+
comment_id: context.payload.comment.id,
111+
content: '-1'
112+
});

0 commit comments

Comments
 (0)