Skip to content

Commit 9a3edf1

Browse files
TheiLLeniumStudiosTheiLLeniumStudios
committed
feat: Load tests
1 parent a5d1012 commit 9a3edf1

17 files changed

Lines changed: 6196 additions & 87 deletions

File tree

‎.github/workflows/loadtest.yml‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
name: Load Test
2+
3+
on:
4+
issue_comment:
5+
types: [created]
6+
7+
jobs:
8+
loadtest:
9+
# Only run on PR comments with /loadtest command
10+
if: |
11+
github.event.issue.pull_request &&
12+
contains(github.event.comment.body, '/loadtest')
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: Add reaction to comment
17+
uses: actions/github-script@v7
18+
with:
19+
script: |
20+
await github.rest.reactions.createForIssueComment({
21+
owner: context.repo.owner,
22+
repo: context.repo.repo,
23+
comment_id: context.payload.comment.id,
24+
content: 'rocket'
25+
});
26+
27+
- name: Get PR details
28+
id: pr
29+
uses: actions/github-script@v7
30+
with:
31+
script: |
32+
const pr = await github.rest.pulls.get({
33+
owner: context.repo.owner,
34+
repo: context.repo.repo,
35+
pull_number: context.issue.number
36+
});
37+
core.setOutput('head_ref', pr.data.head.ref);
38+
core.setOutput('head_sha', pr.data.head.sha);
39+
core.setOutput('base_ref', pr.data.base.ref);
40+
core.setOutput('base_sha', pr.data.base.sha);
41+
console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`);
42+
43+
- name: Set up Go
44+
uses: actions/setup-go@v5
45+
with:
46+
go-version: '1.23'
47+
cache: false
48+
49+
- name: Set up Docker Buildx
50+
uses: docker/setup-buildx-action@v3
51+
52+
- name: Install kind
53+
run: |
54+
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
55+
chmod +x ./kind
56+
sudo mv ./kind /usr/local/bin/kind
57+
58+
# Build OLD image from base branch (e.g., main)
59+
- name: Checkout base branch (old)
60+
uses: actions/checkout@v4
61+
with:
62+
ref: ${{ steps.pr.outputs.base_ref }}
63+
path: old
64+
65+
- name: Build old image
66+
run: |
67+
cd old
68+
docker build -t localhost/reloader:old -f Dockerfile .
69+
echo "Built old image from ${{ steps.pr.outputs.base_ref }} (${{ steps.pr.outputs.base_sha }})"
70+
71+
# Build NEW image from PR branch
72+
- name: Checkout PR branch (new)
73+
uses: actions/checkout@v4
74+
with:
75+
ref: ${{ steps.pr.outputs.head_ref }}
76+
path: new
77+
78+
- name: Build new image
79+
run: |
80+
cd new
81+
docker build -t localhost/reloader:new -f Dockerfile .
82+
echo "Built new image from ${{ steps.pr.outputs.head_ref }} (${{ steps.pr.outputs.head_sha }})"
83+
84+
# Build and run loadtest from PR branch
85+
- name: Build loadtest tool
86+
run: |
87+
cd new/test/loadtest
88+
go build -o loadtest ./cmd/loadtest
89+
90+
- name: Run A/B comparison load test
91+
id: loadtest
92+
run: |
93+
cd new/test/loadtest
94+
./loadtest run \
95+
--old-image=localhost/reloader:old \
96+
--new-image=localhost/reloader:new \
97+
--scenario=all \
98+
--duration=60 2>&1 | tee loadtest-output.txt
99+
echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT
100+
101+
- name: Upload results
102+
uses: actions/upload-artifact@v4
103+
if: always()
104+
with:
105+
name: loadtest-results
106+
path: |
107+
new/test/loadtest/results/
108+
new/test/loadtest/loadtest-output.txt
109+
retention-days: 30
110+
111+
- name: Post results comment
112+
uses: actions/github-script@v7
113+
if: always()
114+
with:
115+
script: |
116+
const fs = require('fs');
117+
118+
let results = '';
119+
const resultsDir = 'new/test/loadtest/results';
120+
121+
// Collect summary of all scenarios
122+
let passCount = 0;
123+
let failCount = 0;
124+
const summaries = [];
125+
126+
if (fs.existsSync(resultsDir)) {
127+
const scenarios = fs.readdirSync(resultsDir).sort();
128+
for (const scenario of scenarios) {
129+
const reportPath = `${resultsDir}/${scenario}/report.txt`;
130+
if (fs.existsSync(reportPath)) {
131+
const report = fs.readFileSync(reportPath, 'utf8');
132+
133+
// Extract status from report
134+
const statusMatch = report.match(/Status:\s+(PASS|FAIL)/);
135+
const status = statusMatch ? statusMatch[1] : 'UNKNOWN';
136+
137+
if (status === 'PASS') passCount++;
138+
else failCount++;
139+
140+
// Extract key metrics for summary
141+
const actionMatch = report.match(/action_total\s+[\d.]+\s+[\d.]+\s+[\d.]+/);
142+
const errorsMatch = report.match(/errors_total\s+[\d.]+\s+[\d.]+/);
143+
144+
summaries.push(`| ${scenario} | ${status === 'PASS' ? '✅' : '❌'} ${status} |`);
145+
146+
results += `\n<details>\n<summary>${status === 'PASS' ? '✅' : '❌'} ${scenario}</summary>\n\n\`\`\`\n${report}\n\`\`\`\n</details>\n`;
147+
}
148+
}
149+
}
150+
151+
if (!results) {
152+
// Read raw output if no reports
153+
if (fs.existsSync('new/test/loadtest/loadtest-output.txt')) {
154+
const output = fs.readFileSync('new/test/loadtest/loadtest-output.txt', 'utf8');
155+
const maxLen = 60000;
156+
results = output.length > maxLen
157+
? output.substring(output.length - maxLen)
158+
: output;
159+
results = `\`\`\`\n${results}\n\`\`\``;
160+
} else {
161+
results = 'No results available';
162+
}
163+
}
164+
165+
const overallStatus = failCount === 0 ? '✅ ALL PASSED' : `❌ ${failCount} FAILED`;
166+
167+
const body = `## Load Test Results ${overallStatus}
168+
169+
**Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new)
170+
**Old commit:** ${{ steps.pr.outputs.base_sha }}
171+
**New commit:** ${{ steps.pr.outputs.head_sha }}
172+
**Triggered by:** @${{ github.event.comment.user.login }}
173+
174+
### Summary
175+
176+
| Scenario | Status |
177+
|----------|--------|
178+
${summaries.join('\n')}
179+
180+
**Total:** ${passCount} passed, ${failCount} failed
181+
182+
### Detailed Results
183+
184+
${results}
185+
186+
<details>
187+
<summary>📦 Download full results</summary>
188+
189+
Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).
190+
</details>
191+
`;
192+
193+
await github.rest.issues.createComment({
194+
owner: context.repo.owner,
195+
repo: context.repo.repo,
196+
issue_number: context.issue.number,
197+
body: body
198+
});
199+
200+
- name: Add success reaction
201+
if: success()
202+
uses: actions/github-script@v7
203+
with:
204+
script: |
205+
await github.rest.reactions.createForIssueComment({
206+
owner: context.repo.owner,
207+
repo: context.repo.repo,
208+
comment_id: context.payload.comment.id,
209+
content: '+1'
210+
});
211+
212+
- name: Add failure reaction
213+
if: failure()
214+
uses: actions/github-script@v7
215+
with:
216+
script: |
217+
await github.rest.reactions.createForIssueComment({
218+
owner: context.repo.owner,
219+
repo: context.repo.repo,
220+
comment_id: context.payload.comment.id,
221+
content: '-1'
222+
});

‎internal/pkg/controller/controller.go‎

Lines changed: 68 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ func NewController(
103103

104104
// Add function to add a new object to the queue in case of creating a resource
105105
func (c *Controller) Add(obj interface{}) {
106+
// Record event received
107+
c.collectors.RecordEventReceived("add", c.resource)
106108

107109
switch object := obj.(type) {
108110
case *v1.Namespace:
@@ -112,11 +114,14 @@ func (c *Controller) Add(obj interface{}) {
112114

113115
if options.ReloadOnCreate == "true" {
114116
if !c.resourceInIgnoredNamespace(obj) && c.resourceInSelectedNamespaces(obj) && secretControllerInitialized && configmapControllerInitialized {
115-
c.queue.Add(handler.ResourceCreatedHandler{
116-
Resource: obj,
117-
Collectors: c.collectors,
118-
Recorder: c.recorder,
117+
c.enqueue(handler.ResourceCreatedHandler{
118+
Resource: obj,
119+
Collectors: c.collectors,
120+
Recorder: c.recorder,
121+
EnqueueTime: time.Now(), // Track when item was enqueued
119122
})
123+
} else {
124+
c.collectors.RecordSkipped("ignored_or_not_selected")
120125
}
121126
}
122127
}
@@ -166,31 +171,42 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) {
166171

167172
// Update function to add an old object and a new object to the queue in case of updating a resource
168173
func (c *Controller) Update(old interface{}, new interface{}) {
174+
// Record event received
175+
c.collectors.RecordEventReceived("update", c.resource)
176+
169177
switch new.(type) {
170178
case *v1.Namespace:
171179
return
172180
}
173181

174182
if !c.resourceInIgnoredNamespace(new) && c.resourceInSelectedNamespaces(new) {
175-
c.queue.Add(handler.ResourceUpdatedHandler{
183+
c.enqueue(handler.ResourceUpdatedHandler{
176184
Resource: new,
177185
OldResource: old,
178186
Collectors: c.collectors,
179187
Recorder: c.recorder,
188+
EnqueueTime: time.Now(), // Track when item was enqueued
180189
})
190+
} else {
191+
c.collectors.RecordSkipped("ignored_or_not_selected")
181192
}
182193
}
183194

184195
// Delete function to add an object to the queue in case of deleting a resource
185196
func (c *Controller) Delete(old interface{}) {
197+
// Record event received
198+
c.collectors.RecordEventReceived("delete", c.resource)
186199

187200
if options.ReloadOnDelete == "true" {
188201
if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized {
189-
c.queue.Add(handler.ResourceDeleteHandler{
190-
Resource: old,
191-
Collectors: c.collectors,
192-
Recorder: c.recorder,
202+
c.enqueue(handler.ResourceDeleteHandler{
203+
Resource: old,
204+
Collectors: c.collectors,
205+
Recorder: c.recorder,
206+
EnqueueTime: time.Now(), // Track when item was enqueued
193207
})
208+
} else {
209+
c.collectors.RecordSkipped("ignored_or_not_selected")
194210
}
195211
}
196212

@@ -201,6 +217,13 @@ func (c *Controller) Delete(old interface{}) {
201217
}
202218
}
203219

220+
// enqueue adds an item to the queue and records metrics
221+
func (c *Controller) enqueue(item interface{}) {
222+
c.queue.Add(item)
223+
c.collectors.RecordQueueAdd()
224+
c.collectors.SetQueueDepth(c.queue.Len())
225+
}
226+
204227
// Run function for controller which handles the queue
205228
func (c *Controller) Run(threadiness int, stopCh chan struct{}) {
206229
defer runtime.HandleCrash()
@@ -242,13 +265,36 @@ func (c *Controller) processNextItem() bool {
242265
if quit {
243266
return false
244267
}
268+
269+
// Update queue depth after getting item
270+
c.collectors.SetQueueDepth(c.queue.Len())
271+
245272
// Tell the queue that we are done with processing this key. This unblocks the key for other workers
246273
// This allows safe parallel processing because two events with the same key are never processed in
247274
// parallel.
248275
defer c.queue.Done(resourceHandler)
249276

277+
// Record queue latency if the handler supports it
278+
if h, ok := resourceHandler.(handler.TimedHandler); ok {
279+
queueLatency := time.Since(h.GetEnqueueTime())
280+
c.collectors.RecordQueueLatency(queueLatency)
281+
}
282+
283+
// Track reconcile/handler duration
284+
startTime := time.Now()
285+
250286
// Invoke the method containing the business logic
251287
err := resourceHandler.(handler.ResourceHandler).Handle()
288+
289+
duration := time.Since(startTime)
290+
291+
// Record reconcile metrics
292+
if err != nil {
293+
c.collectors.RecordReconcile("error", duration)
294+
} else {
295+
c.collectors.RecordReconcile("success", duration)
296+
}
297+
252298
// Handle the error if something went wrong during the execution of the business logic
253299
c.handleErr(err, resourceHandler)
254300
return true
@@ -261,16 +307,26 @@ func (c *Controller) handleErr(err error, key interface{}) {
261307
// This ensures that future processing of updates for this key is not delayed because of
262308
// an outdated error history.
263309
c.queue.Forget(key)
310+
311+
// Record successful event processing
312+
c.collectors.RecordEventProcessed("unknown", c.resource, "success")
264313
return
265314
}
266315

316+
// Record error
317+
c.collectors.RecordError("handler_error")
318+
267319
// This controller retries 5 times if something goes wrong. After that, it stops trying.
268320
if c.queue.NumRequeues(key) < 5 {
269321
logrus.Errorf("Error syncing events: %v", err)
270322

323+
// Record retry
324+
c.collectors.RecordRetry()
325+
271326
// Re-enqueue the key rate limited. Based on the rate limiter on the
272327
// queue and the re-enqueue history, the key will be processed later again.
273328
c.queue.AddRateLimited(key)
329+
c.collectors.SetQueueDepth(c.queue.Len())
274330
return
275331
}
276332

@@ -279,4 +335,7 @@ func (c *Controller) handleErr(err error, key interface{}) {
279335
runtime.HandleError(err)
280336
logrus.Errorf("Dropping key out of the queue: %v", err)
281337
logrus.Debugf("Dropping the key %q out of the queue: %v", key, err)
338+
339+
// Record failed event processing
340+
c.collectors.RecordEventProcessed("unknown", c.resource, "dropped")
282341
}

0 commit comments

Comments
 (0)