From 9a3edf13d2a1b87dfd258d26fec532037cdc4301 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:03:26 +0100 Subject: [PATCH 01/21] feat: Load tests --- .github/workflows/loadtest.yml | 222 ++ internal/pkg/controller/controller.go | 77 +- internal/pkg/controller/controller_test.go | 90 +- internal/pkg/handler/create.go | 34 +- internal/pkg/handler/delete.go | 33 +- internal/pkg/handler/handler.go | 11 +- internal/pkg/handler/update.go | 32 +- internal/pkg/handler/upgrade.go | 43 +- internal/pkg/metrics/prometheus.go | 373 ++- test/loadtest/README.md | 544 +++++ test/loadtest/cmd/loadtest/main.go | 1582 +++++++++++++ test/loadtest/go.mod | 50 + test/loadtest/go.sum | 154 ++ test/loadtest/internal/cluster/kind.go | 313 +++ .../internal/prometheus/prometheus.go | 452 ++++ test/loadtest/internal/scenarios/scenarios.go | 2092 +++++++++++++++++ test/loadtest/manifests/prometheus.yaml | 181 ++ 17 files changed, 6196 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/loadtest.yml create mode 100644 test/loadtest/README.md create mode 100644 test/loadtest/cmd/loadtest/main.go create mode 100644 test/loadtest/go.mod create mode 100644 test/loadtest/go.sum create mode 100644 test/loadtest/internal/cluster/kind.go create mode 100644 test/loadtest/internal/prometheus/prometheus.go create mode 100644 test/loadtest/internal/scenarios/scenarios.go create mode 100644 test/loadtest/manifests/prometheus.yaml diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml new file mode 100644 index 000000000..a62e3e32e --- /dev/null +++ b/.github/workflows/loadtest.yml @@ -0,0 +1,222 @@ +name: Load Test + +on: + issue_comment: + types: [created] + +jobs: + loadtest: + # Only run on PR comments with /loadtest command + if: | + github.event.issue.pull_request && + contains(github.event.comment.body, '/loadtest') + runs-on: ubuntu-latest + + steps: + - name: Add reaction to comment + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + }); + + - name: Get PR details + id: pr + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput('head_ref', pr.data.head.ref); + core.setOutput('head_sha', pr.data.head.sha); + core.setOutput('base_ref', pr.data.base.ref); + core.setOutput('base_sha', pr.data.base.sha); + console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`); + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' + cache: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + # Build OLD image from base branch (e.g., main) + - name: Checkout base branch (old) + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.base_ref }} + path: old + + - name: Build old image + run: | + cd old + docker build -t localhost/reloader:old -f Dockerfile . + echo "Built old image from ${{ steps.pr.outputs.base_ref }} (${{ steps.pr.outputs.base_sha }})" + + # Build NEW image from PR branch + - name: Checkout PR branch (new) + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.head_ref }} + path: new + + - name: Build new image + run: | + cd new + docker build -t localhost/reloader:new -f Dockerfile . + echo "Built new image from ${{ steps.pr.outputs.head_ref }} (${{ steps.pr.outputs.head_sha }})" + + # Build and run loadtest from PR branch + - name: Build loadtest tool + run: | + cd new/test/loadtest + go build -o loadtest ./cmd/loadtest + + - name: Run A/B comparison load test + id: loadtest + run: | + cd new/test/loadtest + ./loadtest run \ + --old-image=localhost/reloader:old \ + --new-image=localhost/reloader:new \ + --scenario=all \ + --duration=60 2>&1 | tee loadtest-output.txt + echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT + + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: loadtest-results + path: | + new/test/loadtest/results/ + new/test/loadtest/loadtest-output.txt + retention-days: 30 + + - name: Post results comment + uses: actions/github-script@v7 + if: always() + with: + script: | + const fs = require('fs'); + + let results = ''; + const resultsDir = 'new/test/loadtest/results'; + + // Collect summary of all scenarios + let passCount = 0; + let failCount = 0; + const summaries = []; + + if (fs.existsSync(resultsDir)) { + const scenarios = fs.readdirSync(resultsDir).sort(); + for (const scenario of scenarios) { + const reportPath = `${resultsDir}/${scenario}/report.txt`; + if (fs.existsSync(reportPath)) { + const report = fs.readFileSync(reportPath, 'utf8'); + + // Extract status from report + const statusMatch = report.match(/Status:\s+(PASS|FAIL)/); + const status = statusMatch ? statusMatch[1] : 'UNKNOWN'; + + if (status === 'PASS') passCount++; + else failCount++; + + // Extract key metrics for summary + const actionMatch = report.match(/action_total\s+[\d.]+\s+[\d.]+\s+[\d.]+/); + const errorsMatch = report.match(/errors_total\s+[\d.]+\s+[\d.]+/); + + summaries.push(`| ${scenario} | ${status === 'PASS' ? '✅' : '❌'} ${status} |`); + + results += `\n
\n${status === 'PASS' ? '✅' : '❌'} ${scenario}\n\n\`\`\`\n${report}\n\`\`\`\n
\n`; + } + } + } + + if (!results) { + // Read raw output if no reports + if (fs.existsSync('new/test/loadtest/loadtest-output.txt')) { + const output = fs.readFileSync('new/test/loadtest/loadtest-output.txt', 'utf8'); + const maxLen = 60000; + results = output.length > maxLen + ? output.substring(output.length - maxLen) + : output; + results = `\`\`\`\n${results}\n\`\`\``; + } else { + results = 'No results available'; + } + } + + const overallStatus = failCount === 0 ? '✅ ALL PASSED' : `❌ ${failCount} FAILED`; + + const body = `## Load Test Results ${overallStatus} + + **Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new) + **Old commit:** ${{ steps.pr.outputs.base_sha }} + **New commit:** ${{ steps.pr.outputs.head_sha }} + **Triggered by:** @${{ github.event.comment.user.login }} + + ### Summary + + | Scenario | Status | + |----------|--------| + ${summaries.join('\n')} + + **Total:** ${passCount} passed, ${failCount} failed + + ### Detailed Results + + ${results} + +
+ 📦 Download full results + + Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}). +
+ `; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + + - name: Add success reaction + if: success() + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: '+1' + }); + + - name: Add failure reaction + if: failure() + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: '-1' + }); diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 15b2e0f17..7dd3f5eb2 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -103,6 +103,8 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { + // Record event received + c.collectors.RecordEventReceived("add", c.resource) switch object := obj.(type) { case *v1.Namespace: @@ -112,11 +114,14 @@ func (c *Controller) Add(obj interface{}) { if options.ReloadOnCreate == "true" { if !c.resourceInIgnoredNamespace(obj) && c.resourceInSelectedNamespaces(obj) && secretControllerInitialized && configmapControllerInitialized { - c.queue.Add(handler.ResourceCreatedHandler{ - Resource: obj, - Collectors: c.collectors, - Recorder: c.recorder, + c.enqueue(handler.ResourceCreatedHandler{ + Resource: obj, + Collectors: c.collectors, + Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } } @@ -166,31 +171,42 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { + // Record event received + c.collectors.RecordEventReceived("update", c.resource) + switch new.(type) { case *v1.Namespace: return } if !c.resourceInIgnoredNamespace(new) && c.resourceInSelectedNamespaces(new) { - c.queue.Add(handler.ResourceUpdatedHandler{ + c.enqueue(handler.ResourceUpdatedHandler{ Resource: new, OldResource: old, Collectors: c.collectors, Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { + // Record event received + c.collectors.RecordEventReceived("delete", c.resource) if options.ReloadOnDelete == "true" { if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized { - c.queue.Add(handler.ResourceDeleteHandler{ - Resource: old, - Collectors: c.collectors, - Recorder: c.recorder, + c.enqueue(handler.ResourceDeleteHandler{ + Resource: old, + Collectors: c.collectors, + Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } @@ -201,6 +217,13 @@ func (c *Controller) Delete(old interface{}) { } } +// enqueue adds an item to the queue and records metrics +func (c *Controller) enqueue(item interface{}) { + c.queue.Add(item) + c.collectors.RecordQueueAdd() + c.collectors.SetQueueDepth(c.queue.Len()) +} + // Run function for controller which handles the queue func (c *Controller) Run(threadiness int, stopCh chan struct{}) { defer runtime.HandleCrash() @@ -242,13 +265,36 @@ func (c *Controller) processNextItem() bool { if quit { return false } + + // Update queue depth after getting item + c.collectors.SetQueueDepth(c.queue.Len()) + // Tell the queue that we are done with processing this key. This unblocks the key for other workers // This allows safe parallel processing because two events with the same key are never processed in // parallel. defer c.queue.Done(resourceHandler) + // Record queue latency if the handler supports it + if h, ok := resourceHandler.(handler.TimedHandler); ok { + queueLatency := time.Since(h.GetEnqueueTime()) + c.collectors.RecordQueueLatency(queueLatency) + } + + // Track reconcile/handler duration + startTime := time.Now() + // Invoke the method containing the business logic err := resourceHandler.(handler.ResourceHandler).Handle() + + duration := time.Since(startTime) + + // Record reconcile metrics + if err != nil { + c.collectors.RecordReconcile("error", duration) + } else { + c.collectors.RecordReconcile("success", duration) + } + // Handle the error if something went wrong during the execution of the business logic c.handleErr(err, resourceHandler) return true @@ -261,16 +307,26 @@ func (c *Controller) handleErr(err error, key interface{}) { // This ensures that future processing of updates for this key is not delayed because of // an outdated error history. c.queue.Forget(key) + + // Record successful event processing + c.collectors.RecordEventProcessed("unknown", c.resource, "success") return } + // Record error + c.collectors.RecordError("handler_error") + // This controller retries 5 times if something goes wrong. After that, it stops trying. if c.queue.NumRequeues(key) < 5 { logrus.Errorf("Error syncing events: %v", err) + // Record retry + c.collectors.RecordRetry() + // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. c.queue.AddRateLimited(key) + c.collectors.SetQueueDepth(c.queue.Len()) return } @@ -279,4 +335,7 @@ func (c *Controller) handleErr(err error, key interface{}) { runtime.HandleError(err) logrus.Errorf("Dropping key out of the queue: %v", err) logrus.Debugf("Dropping the key %q out of the queue: %v", key, err) + + // Record failed event processing + c.collectors.RecordEventProcessed("unknown", c.resource, "dropped") } diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 63e6be3e2..7b4c728d8 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -2157,19 +2157,21 @@ func TestController_resourceInIgnoredNamespace(t *testing.T) { }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &Controller{ - client: tt.fields.client, - indexer: tt.fields.indexer, - queue: tt.fields.queue, - informer: tt.fields.informer, - namespace: tt.fields.namespace, - ignoredNamespaces: tt.fields.ignoredNamespaces, - } - if got := c.resourceInIgnoredNamespace(tt.args.raw); got != tt.want { - t.Errorf("Controller.resourceInIgnoredNamespace() = %v, want %v", got, tt.want) - } - }) + t.Run( + tt.name, func(t *testing.T) { + c := &Controller{ + client: tt.fields.client, + indexer: tt.fields.indexer, + queue: tt.fields.queue, + informer: tt.fields.informer, + namespace: tt.fields.namespace, + ignoredNamespaces: tt.fields.ignoredNamespaces, + } + if got := c.resourceInIgnoredNamespace(tt.args.raw); got != tt.want { + t.Errorf("Controller.resourceInIgnoredNamespace() = %v, want %v", got, tt.want) + } + }, + ) } } @@ -2331,35 +2333,37 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fakeClient := fake.NewSimpleClientset() - namespace, _ := fakeClient.CoreV1().Namespaces().Create(context.Background(), &tt.fields.namespace, metav1.CreateOptions{}) - logrus.Infof("created fakeClient namespace for testing = %s", namespace.Name) - - c := &Controller{ - client: fakeClient, - indexer: tt.fields.indexer, - queue: tt.fields.queue, - informer: tt.fields.informer, - namespace: tt.fields.namespace.Name, - namespaceSelector: tt.fields.namespaceSelector, - } - - listOptions := metav1.ListOptions{} - listOptions.LabelSelector = tt.fields.namespaceSelector - namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) - - for _, ns := range namespaces.Items { - c.addSelectedNamespaceToCache(ns) - } - - if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { - t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) - } - - for _, ns := range namespaces.Items { - c.removeSelectedNamespaceFromCache(ns) - } - }) + t.Run( + tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset() + namespace, _ := fakeClient.CoreV1().Namespaces().Create(context.Background(), &tt.fields.namespace, metav1.CreateOptions{}) + logrus.Infof("created fakeClient namespace for testing = %s", namespace.Name) + + c := &Controller{ + client: fakeClient, + indexer: tt.fields.indexer, + queue: tt.fields.queue, + informer: tt.fields.informer, + namespace: tt.fields.namespace.Name, + namespaceSelector: tt.fields.namespaceSelector, + } + + listOptions := metav1.ListOptions{} + listOptions.LabelSelector = tt.fields.namespaceSelector + namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) + + for _, ns := range namespaces.Items { + c.addSelectedNamespaceToCache(ns) + } + + if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { + t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) + } + + for _, ns := range namespaces.Items { + c.removeSelectedNamespaceFromCache(ns) + } + }, + ) } } diff --git a/internal/pkg/handler/create.go b/internal/pkg/handler/create.go index fab737888..5fd3014e0 100644 --- a/internal/pkg/handler/create.go +++ b/internal/pkg/handler/create.go @@ -1,6 +1,8 @@ package handler import ( + "time" + "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/options" @@ -11,23 +13,45 @@ import ( // ResourceCreatedHandler contains new objects type ResourceCreatedHandler struct { - Resource interface{} - Collectors metrics.Collectors - Recorder record.EventRecorder + Resource interface{} + Collectors metrics.Collectors + Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceCreatedHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes the newly created resource func (r ResourceCreatedHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil { logrus.Errorf("Resource creation handler received nil resource") + result = "error" } else { config, _ := r.GetConfig() // Send webhook if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err != nil { + result = "error" + } + return err } return nil } diff --git a/internal/pkg/handler/delete.go b/internal/pkg/handler/delete.go index 65c671e89..243602c57 100644 --- a/internal/pkg/handler/delete.go +++ b/internal/pkg/handler/delete.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "slices" + "time" "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/callbacks" @@ -20,23 +21,45 @@ import ( // ResourceDeleteHandler contains new objects type ResourceDeleteHandler struct { - Resource interface{} - Collectors metrics.Collectors - Recorder record.EventRecorder + Resource interface{} + Collectors metrics.Collectors + Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceDeleteHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes resources being deleted func (r ResourceDeleteHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil { logrus.Errorf("Resource delete handler received nil resource") + result = "error" } else { config, _ := r.GetConfig() // Send webhook if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + if err != nil { + result = "error" + } + return err } return nil } diff --git a/internal/pkg/handler/handler.go b/internal/pkg/handler/handler.go index 1f5858e58..9018f80ae 100644 --- a/internal/pkg/handler/handler.go +++ b/internal/pkg/handler/handler.go @@ -1,9 +1,18 @@ package handler -import "github.com/stakater/Reloader/pkg/common" +import ( + "time" + + "github.com/stakater/Reloader/pkg/common" +) // ResourceHandler handles the creation and update of resources type ResourceHandler interface { Handle() error GetConfig() (common.Config, string) } + +// TimedHandler is a handler that tracks when it was enqueued +type TimedHandler interface { + GetEnqueueTime() time.Time +} diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index ae0bb1e21..3ae108053 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -1,6 +1,8 @@ package handler import ( + "time" + "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/options" @@ -16,21 +18,47 @@ type ResourceUpdatedHandler struct { OldResource interface{} Collectors metrics.Collectors Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceUpdatedHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes the updated resource func (r ResourceUpdatedHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil || r.OldResource == nil { logrus.Errorf("Resource update handler received nil resource") + result = "error" } else { config, oldSHAData := r.GetConfig() if config.SHAValue != oldSHAData { // Send a webhook if update if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err != nil { + result = "error" + } + return err + } else { + // No data change - skip + result = "skipped" + r.Collectors.RecordSkipped("no_data_change") } } return nil diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 6f185f1e8..e355d5fd0 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "time" "github.com/parnurzeal/gorequest" "github.com/prometheus/client_golang/prometheus" @@ -236,23 +237,34 @@ func rollingUpgrade(clients kube.Clients, config common.Config, upgradeFuncs cal func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error { items := upgradeFuncs.ItemsFunc(clients, config.Namespace) + // Record workloads scanned + collectors.RecordWorkloadsScanned(upgradeFuncs.ResourceType, len(items)) + + matchedCount := 0 for _, item := range items { - err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) error { - return upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) + err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { + matched, err := upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) + if matched { + matchedCount++ + } + return matched, err }) if err != nil { return err } } + // Record workloads matched + collectors.RecordWorkloadsMatched(upgradeFuncs.ResourceType, matchedCount) + return nil } -func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error { +func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error { var lastError error fetchResource := false // do not fetch resource on first attempt, already done by ItemsFunc err := wait.ExponentialBackoff(backoff, func() (bool, error) { - err := fn(fetchResource) + _, err := fn(fetchResource) fetchResource = true switch { case err == nil: @@ -270,17 +282,19 @@ func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error { return err } -func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) error { +func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) (bool, error) { + actionStartTime := time.Now() + accessor, err := meta.Accessor(resource) if err != nil { - return err + return false, err } resourceName := accessor.GetName() if fetchResource { resource, err = upgradeFuncs.ItemFunc(clients, resourceName, config.Namespace) if err != nil { - return err + return false, err } } annotations := upgradeFuncs.AnnotationsFunc(resource) @@ -289,13 +303,14 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca if !result.ShouldReload { logrus.Debugf("No changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace) - return nil + return false, nil } strategyResult := strategy(upgradeFuncs, resource, config, result.AutoReload) if strategyResult.Result != constants.Updated { - return nil + collectors.RecordSkipped("strategy_not_updated") + return false, nil } // find correct annotation and update the resource @@ -309,7 +324,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca _, err = PauseDeployment(deployment, clients, config.Namespace, pauseInterval) if err != nil { logrus.Errorf("Failed to pause deployment '%s' in namespace '%s': %v", resourceName, config.Namespace, err) - return err + return true, err } } } @@ -320,16 +335,19 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca err = upgradeFuncs.UpdateFunc(clients, config.Namespace, resource) } + actionLatency := time.Since(actionStartTime) + if err != nil { message := fmt.Sprintf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) collectors.Reloaded.With(prometheus.Labels{"success": "false"}).Inc() collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "false", "namespace": config.Namespace}).Inc() + collectors.RecordAction(upgradeFuncs.ResourceType, "error", actionLatency) if recorder != nil { recorder.Event(resource, v1.EventTypeWarning, "ReloadFail", message) } - return err + return true, err } else { message := fmt.Sprintf("Changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace) message += fmt.Sprintf(", Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace) @@ -338,6 +356,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc() collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": config.Namespace}).Inc() + collectors.RecordAction(upgradeFuncs.ResourceType, "success", actionLatency) alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD") if recorder != nil { recorder.Event(resource, v1.EventTypeNormal, "Reloaded", message) @@ -350,7 +369,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca } } - return nil + return true, nil } func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string) string { diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index 94153eace..e6f2f356b 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -1,54 +1,407 @@ package metrics import ( + "context" "net/http" + "net/url" "os" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/tools/metrics" ) +// clientGoRequestMetrics implements metrics.LatencyMetric and metrics.ResultMetric +// to expose client-go's rest_client_requests_total metric +type clientGoRequestMetrics struct { + requestCounter *prometheus.CounterVec + requestLatency *prometheus.HistogramVec +} + +func (m *clientGoRequestMetrics) Increment(ctx context.Context, code string, method string, host string) { + m.requestCounter.WithLabelValues(code, method, host).Inc() +} + +func (m *clientGoRequestMetrics) Observe(ctx context.Context, verb string, u url.URL, latency time.Duration) { + m.requestLatency.WithLabelValues(verb, u.Host).Observe(latency.Seconds()) +} + +var clientGoMetrics = &clientGoRequestMetrics{ + requestCounter: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host"}, + ), + requestLatency: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_duration_seconds", + Help: "Request latency in seconds. Broken down by verb and host.", + Buckets: []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30}, + }, + []string{"verb", "host"}, + ), +} + +func init() { + // Register the metrics collectors + prometheus.MustRegister(clientGoMetrics.requestCounter) + prometheus.MustRegister(clientGoMetrics.requestLatency) + + // Register our metrics implementation with client-go + metrics.RequestResult = clientGoMetrics + metrics.RequestLatency = clientGoMetrics +} + +// Collectors holds all Prometheus metrics collectors for Reloader. type Collectors struct { + // Existing metrics (preserved for backward compatibility) Reloaded *prometheus.CounterVec ReloadedByNamespace *prometheus.CounterVec + countByNamespace bool + + // Reconcile/Handler metrics + ReconcileTotal *prometheus.CounterVec // Total reconcile calls by result + ReconcileDuration *prometheus.HistogramVec // Time spent in reconcile/handler + + // Action metrics + ActionTotal *prometheus.CounterVec // Total actions by workload kind and result + ActionLatency *prometheus.HistogramVec // Time from event to action applied + + // Skip metrics + SkippedTotal *prometheus.CounterVec // Skipped operations by reason + + // Queue metrics + QueueDepth prometheus.Gauge // Current queue depth + QueueAdds prometheus.Counter // Total items added to queue + QueueLatency *prometheus.HistogramVec // Time spent in queue + + // Error and retry metrics + ErrorsTotal *prometheus.CounterVec // Errors by type + RetriesTotal prometheus.Counter // Total retries + + // Event processing metrics + EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) + EventsProcessed *prometheus.CounterVec // Events processed by type and result + + // Resource discovery metrics + WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind + WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind +} + +// RecordReload records a reload event with the given success status and namespace. +// Preserved for backward compatibility. +func (c *Collectors) RecordReload(success bool, namespace string) { + if c == nil { + return + } + + successLabel := "false" + if success { + successLabel = "true" + } + + c.Reloaded.With(prometheus.Labels{"success": successLabel}).Inc() + + if c.countByNamespace { + c.ReloadedByNamespace.With(prometheus.Labels{ + "success": successLabel, + "namespace": namespace, + }).Inc() + } +} + +// RecordReconcile records a reconcile/handler invocation. +func (c *Collectors) RecordReconcile(result string, duration time.Duration) { + if c == nil { + return + } + c.ReconcileTotal.With(prometheus.Labels{"result": result}).Inc() + c.ReconcileDuration.With(prometheus.Labels{"result": result}).Observe(duration.Seconds()) +} + +// RecordAction records a reload action on a workload. +func (c *Collectors) RecordAction(workloadKind string, result string, latency time.Duration) { + if c == nil { + return + } + c.ActionTotal.With(prometheus.Labels{"workload_kind": workloadKind, "result": result}).Inc() + c.ActionLatency.With(prometheus.Labels{"workload_kind": workloadKind}).Observe(latency.Seconds()) +} + +// RecordSkipped records a skipped operation with reason. +func (c *Collectors) RecordSkipped(reason string) { + if c == nil { + return + } + c.SkippedTotal.With(prometheus.Labels{"reason": reason}).Inc() +} + +// RecordQueueAdd records an item being added to the queue. +func (c *Collectors) RecordQueueAdd() { + if c == nil { + return + } + c.QueueAdds.Inc() +} + +// SetQueueDepth sets the current queue depth. +func (c *Collectors) SetQueueDepth(depth int) { + if c == nil { + return + } + c.QueueDepth.Set(float64(depth)) +} + +// RecordQueueLatency records how long an item spent in the queue. +func (c *Collectors) RecordQueueLatency(latency time.Duration) { + if c == nil { + return + } + c.QueueLatency.With(prometheus.Labels{}).Observe(latency.Seconds()) +} + +// RecordError records an error by type. +func (c *Collectors) RecordError(errorType string) { + if c == nil { + return + } + c.ErrorsTotal.With(prometheus.Labels{"type": errorType}).Inc() +} + +// RecordRetry records a retry attempt. +func (c *Collectors) RecordRetry() { + if c == nil { + return + } + c.RetriesTotal.Inc() +} + +// RecordEventReceived records an event being received. +func (c *Collectors) RecordEventReceived(eventType string, resourceType string) { + if c == nil { + return + } + c.EventsReceived.With(prometheus.Labels{"event_type": eventType, "resource_type": resourceType}).Inc() +} + +// RecordEventProcessed records an event being processed. +func (c *Collectors) RecordEventProcessed(eventType string, resourceType string, result string) { + if c == nil { + return + } + c.EventsProcessed.With(prometheus.Labels{"event_type": eventType, "resource_type": resourceType, "result": result}).Inc() +} + +// RecordWorkloadsScanned records workloads scanned during a reconcile. +func (c *Collectors) RecordWorkloadsScanned(kind string, count int) { + if c == nil { + return + } + c.WorkloadsScanned.With(prometheus.Labels{"kind": kind}).Add(float64(count)) +} + +// RecordWorkloadsMatched records workloads matched for reload. +func (c *Collectors) RecordWorkloadsMatched(kind string, count int) { + if c == nil { + return + } + c.WorkloadsMatched.With(prometheus.Labels{"kind": kind}).Add(float64(count)) } func NewCollectors() Collectors { + // Existing metrics (preserved) reloaded := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", Name: "reload_executed_total", Help: "Counter of reloads executed by Reloader.", }, - []string{ - "success", - }, + []string{"success"}, ) - - //set 0 as default value reloaded.With(prometheus.Labels{"success": "true"}).Add(0) reloaded.With(prometheus.Labels{"success": "false"}).Add(0) - reloaded_by_namespace := prometheus.NewCounterVec( + reloadedByNamespace := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", Name: "reload_executed_total_by_namespace", Help: "Counter of reloads executed by Reloader by namespace.", }, - []string{ - "success", - "namespace", + []string{"success", "namespace"}, + ) + + // === NEW: Comprehensive metrics === + + reconcileTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "reconcile_total", + Help: "Total number of reconcile/handler invocations by result.", + }, + []string{"result"}, + ) + + reconcileDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "reconcile_duration_seconds", + Help: "Time spent in reconcile/handler in seconds.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, + }, + []string{"result"}, + ) + + actionTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "action_total", + Help: "Total number of reload actions by workload kind and result.", + }, + []string{"workload_kind", "result"}, + ) + + actionLatency := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "action_latency_seconds", + Help: "Time from event received to action applied in seconds.", + Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, + }, + []string{"workload_kind"}, + ) + + skippedTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "skipped_total", + Help: "Total number of skipped operations by reason.", + }, + []string{"reason"}, + ) + + queueDepth := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "reloader", + Name: "workqueue_depth", + Help: "Current depth of the work queue.", + }, + ) + + queueAdds := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workqueue_adds_total", + Help: "Total number of items added to the work queue.", + }, + ) + + queueLatency := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "workqueue_latency_seconds", + Help: "Time spent in the work queue in seconds.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}, + }, + []string{}, + ) + + errorsTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "errors_total", + Help: "Total number of errors by type.", + }, + []string{"type"}, + ) + + retriesTotal := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "retries_total", + Help: "Total number of retry attempts.", + }, + ) + + eventsReceived := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "events_received_total", + Help: "Total number of events received by type and resource.", + }, + []string{"event_type", "resource_type"}, + ) + + eventsProcessed := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "events_processed_total", + Help: "Total number of events processed by type, resource, and result.", + }, + []string{"event_type", "resource_type", "result"}, + ) + + workloadsScanned := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workloads_scanned_total", + Help: "Total number of workloads scanned by kind.", }, + []string{"kind"}, ) + + workloadsMatched := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workloads_matched_total", + Help: "Total number of workloads matched for reload by kind.", + }, + []string{"kind"}, + ) + return Collectors{ Reloaded: reloaded, - ReloadedByNamespace: reloaded_by_namespace, + ReloadedByNamespace: reloadedByNamespace, + countByNamespace: os.Getenv("METRICS_COUNT_BY_NAMESPACE") == "enabled", + + ReconcileTotal: reconcileTotal, + ReconcileDuration: reconcileDuration, + ActionTotal: actionTotal, + ActionLatency: actionLatency, + SkippedTotal: skippedTotal, + QueueDepth: queueDepth, + QueueAdds: queueAdds, + QueueLatency: queueLatency, + ErrorsTotal: errorsTotal, + RetriesTotal: retriesTotal, + EventsReceived: eventsReceived, + EventsProcessed: eventsProcessed, + WorkloadsScanned: workloadsScanned, + WorkloadsMatched: workloadsMatched, } } func SetupPrometheusEndpoint() Collectors { collectors := NewCollectors() + + // Register all metrics prometheus.MustRegister(collectors.Reloaded) + prometheus.MustRegister(collectors.ReconcileTotal) + prometheus.MustRegister(collectors.ReconcileDuration) + prometheus.MustRegister(collectors.ActionTotal) + prometheus.MustRegister(collectors.ActionLatency) + prometheus.MustRegister(collectors.SkippedTotal) + prometheus.MustRegister(collectors.QueueDepth) + prometheus.MustRegister(collectors.QueueAdds) + prometheus.MustRegister(collectors.QueueLatency) + prometheus.MustRegister(collectors.ErrorsTotal) + prometheus.MustRegister(collectors.RetriesTotal) + prometheus.MustRegister(collectors.EventsReceived) + prometheus.MustRegister(collectors.EventsProcessed) + prometheus.MustRegister(collectors.WorkloadsScanned) + prometheus.MustRegister(collectors.WorkloadsMatched) if os.Getenv("METRICS_COUNT_BY_NAMESPACE") == "enabled" { prometheus.MustRegister(collectors.ReloadedByNamespace) diff --git a/test/loadtest/README.md b/test/loadtest/README.md new file mode 100644 index 000000000..7182bb3a9 --- /dev/null +++ b/test/loadtest/README.md @@ -0,0 +1,544 @@ +# Reloader Load Test Framework + +This framework provides A/B comparison testing between two Reloader container images. + +## Overview + +The load test framework: +1. Creates a local kind cluster (1 control-plane + 6 worker nodes) +2. Deploys Prometheus for metrics collection +3. Loads the provided Reloader container images into the cluster +4. Runs standardized test scenarios (S1-S13) +5. Collects metrics via Prometheus scraping +6. Generates comparison reports with pass/fail criteria + +## Prerequisites + +- Docker or Podman +- kind (Kubernetes in Docker) +- kubectl +- Go 1.22+ + +## Building + +```bash +cd test/loadtest +go build -o loadtest ./cmd/loadtest +``` + +## Quick Start + +```bash +# Compare two published images (e.g., different versions) +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 + +# Run a specific scenario +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 \ + --scenario=S2 \ + --duration=120 + +# Test only a single image (no comparison) +./loadtest run --new-image=myregistry/reloader:dev + +# Use local images built with docker/podman +./loadtest run \ + --old-image=localhost/reloader:baseline \ + --new-image=localhost/reloader:feature-branch + +# Skip cluster creation (use existing kind cluster) +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 \ + --skip-cluster + +# Run all scenarios in parallel on 4 clusters (faster execution) +./loadtest run \ + --new-image=localhost/reloader:dev \ + --parallelism=4 + +# Run all 13 scenarios in parallel (one cluster per scenario) +./loadtest run \ + --new-image=localhost/reloader:dev \ + --parallelism=13 + +# Generate report from existing results +./loadtest report --scenario=S2 --results-dir=./results +``` + +## Command Line Options + +### Run Command + +| Option | Description | Default | +|--------|-------------|---------| +| `--old-image=IMAGE` | Container image for "old" version | - | +| `--new-image=IMAGE` | Container image for "new" version | - | +| `--scenario=ID` | Test scenario: S1-S13 or "all" | all | +| `--duration=SECONDS` | Test duration in seconds | 60 | +| `--parallelism=N` | Run N scenarios in parallel on N kind clusters | 1 | +| `--skip-cluster` | Skip kind cluster creation (use existing, only for parallelism=1) | false | +| `--results-dir=DIR` | Directory for results | ./results | + +**Note:** At least one of `--old-image` or `--new-image` is required. Provide both for A/B comparison. + +### Report Command + +| Option | Description | Default | +|--------|-------------|---------| +| `--scenario=ID` | Scenario to report on (required) | - | +| `--results-dir=DIR` | Directory containing results | ./results | +| `--output=FILE` | Output file (default: stdout) | - | + +## Test Scenarios + +| ID | Name | Description | +|-----|-----------------------|-------------------------------------------------| +| S1 | Burst Updates | Many ConfigMap/Secret updates in quick succession | +| S2 | Fan-Out | One ConfigMap used by many (50) workloads | +| S3 | High Cardinality | Many CMs/Secrets across many namespaces | +| S4 | No-Op Updates | Updates that don't change data (annotation only)| +| S5 | Workload Churn | Deployments created/deleted rapidly | +| S6 | Controller Restart | Restart controller pod under load | +| S7 | API Pressure | Many concurrent update requests | +| S8 | Large Objects | ConfigMaps > 100KB | +| S9 | Multi-Workload Types | Tests all workload types (Deploy, STS, DS) | +| S10 | Secrets + Mixed | Secrets and mixed ConfigMap+Secret workloads | +| S11 | Annotation Strategy | Tests `--reload-strategy=annotations` | +| S12 | Pause & Resume | Tests pause-period during rapid updates | +| S13 | Complex References | Init containers, valueFrom, projected volumes | + +## Metrics Reference + +This section explains each metric collected during load tests, what it measures, and what different values might indicate. + +### Counter Metrics (Totals) + +#### `reconcile_total` +**What it measures:** The total number of reconciliation loops executed by the controller. + +**What it indicates:** +- **Higher in new vs old:** The new controller-runtime implementation may batch events differently. This is often expected behavior, not a problem. +- **Lower in new vs old:** Better event batching/deduplication. Controller-runtime's work queue naturally deduplicates events. +- **Expected behavior:** The new implementation typically has *fewer* reconciles due to intelligent event batching. + +#### `action_total` +**What it measures:** The total number of reload actions triggered (rolling restarts of Deployments/StatefulSets/DaemonSets). + +**What it indicates:** +- **Should match expected value:** Both implementations should trigger the same number of reloads for the same workload. +- **Lower than expected:** Some updates were missed - potential bug or race condition. +- **Higher than expected:** Duplicate reloads triggered - inefficiency but not data loss. + +#### `reload_executed_total` +**What it measures:** Successful reload operations executed, labeled by `success=true/false`. + +**What it indicates:** +- **`success=true` count:** Number of workloads successfully restarted. +- **`success=false` count:** Failed restart attempts (API errors, permission issues). +- **Should match `action_total`:** If significantly lower, reloads are failing. + +#### `workloads_scanned_total` +**What it measures:** Number of workloads (Deployments, etc.) scanned when checking for ConfigMap/Secret references. + +**What it indicates:** +- **High count:** Controller is scanning many workloads per reconcile. +- **Expected behavior:** Should roughly match the number of workloads × number of reconciles. +- **Optimization signal:** If very high, namespace filtering or label selectors could help. + +#### `workloads_matched_total` +**What it measures:** Number of workloads that matched (reference the changed ConfigMap/Secret). + +**What it indicates:** +- **Should match `reload_executed_total`:** Every matched workload should be reloaded. +- **Higher than reloads:** Some matched workloads weren't reloaded (potential issue). + +#### `errors_total` +**What it measures:** Total errors encountered, labeled by error type. + +**What it indicates:** +- **Should be 0:** Any errors indicate problems. +- **Common causes:** API server timeouts, RBAC issues, resource conflicts. +- **Critical metric:** Non-zero errors in production should be investigated. + +### API Efficiency Metrics (REST Client) + +These metrics track Kubernetes API server calls made by Reloader. Lower values indicate more efficient operation with less API server load. + +#### `rest_client_requests_total` +**What it measures:** Total number of HTTP requests made to the Kubernetes API server. + +**What it indicates:** +- **Lower is better:** Fewer API calls means less load on the API server. +- **High count:** May indicate inefficient caching or excessive reconciles. +- **Comparison use:** Shows overall API efficiency between implementations. + +#### `rest_client_requests_get` +**What it measures:** Number of GET requests (fetching individual resources or listings). + +**What it indicates:** +- **Includes:** Fetching ConfigMaps, Secrets, Deployments, etc. +- **Higher count:** More frequent resource fetching, possibly due to cache misses. +- **Expected behavior:** Controller-runtime's caching should reduce GET requests compared to direct API calls. + +#### `rest_client_requests_patch` +**What it measures:** Number of PATCH requests (partial updates to resources). + +**What it indicates:** +- **Used for:** Rolling restart annotations on workloads. +- **Should correlate with:** `reload_executed_total` - each reload typically requires one PATCH. +- **Lower is better:** Fewer patches means more efficient batching or deduplication. + +#### `rest_client_requests_put` +**What it measures:** Number of PUT requests (full resource updates). + +**What it indicates:** +- **Used for:** Full object replacements (less common than PATCH). +- **Should be low:** Most updates use PATCH for efficiency. +- **High count:** May indicate suboptimal update strategy. + +#### `rest_client_requests_errors` +**What it measures:** Number of failed API requests (4xx/5xx responses). + +**What it indicates:** +- **Should be 0:** Errors indicate API server issues or permission problems. +- **Common causes:** Rate limiting, RBAC issues, resource conflicts, network issues. +- **Non-zero:** Investigate API server logs and Reloader permissions. + +### Latency Metrics (Percentiles) + +All latency metrics are reported in **seconds**. The report shows p50 (median), p95, and p99 percentiles. + +#### `reconcile_duration (s)` +**What it measures:** Time spent inside each reconcile loop, from start to finish. + +**What it indicates:** +- **p50 (median):** Typical reconcile time. Should be < 100ms for good performance. +- **p95:** 95th percentile - only 5% of reconciles take longer than this. +- **p99:** 99th percentile - indicates worst-case performance. + +**Interpreting differences:** +- **New higher than old:** Controller-runtime reconciles may do more work per loop but run fewer times. Check `reconcile_total` - if it's lower, this is expected. +- **Minor differences (< 0.5s absolute):** Not significant for sub-second values. + +#### `action_latency (s)` +**What it measures:** End-to-end time from ConfigMap/Secret change detection to workload restart triggered. + +**What it indicates:** +- **This is the user-facing latency:** How long users wait for their config changes to take effect. +- **p50 < 1s:** Excellent - most changes apply within a second. +- **p95 < 5s:** Good - even under load, changes apply quickly. +- **p99 > 10s:** May need investigation - some changes take too long. + +**What affects this:** +- API server responsiveness +- Number of workloads to scan +- Concurrent updates competing for resources + +### Understanding the Report + +#### Report Columns + +``` +Metric Old New Expected Old✓ New✓ Status +------ --- --- -------- ---- ---- ------ +action_total 100.00 100.00 100 ✓ ✓ pass +action_latency_p95 (s) 0.15 0.04 - - - pass +``` + +- **Old/New:** Measured values from each implementation +- **Expected:** Known expected value (for throughput metrics) +- **Old✓/New✓:** Whether the value is within 15% of expected (✓ = yes, ✗ = no, - = no expected value) +- **Status:** pass/fail based on comparison thresholds + +#### Pass/Fail Logic + +| Metric Type | Pass Condition | +|-------------|----------------| +| Throughput (action_total, reload_executed_total) | New value within 15% of expected | +| Latency (p50, p95, p99) | New not more than threshold% worse than old, OR absolute difference < minimum threshold | +| Errors | New ≤ Old (ideally both 0) | +| API Efficiency (rest_client_requests_*) | New ≤ Old (lower is better), or New not more than 50% higher | + +#### Latency Thresholds + +Latency comparisons use both percentage AND absolute thresholds to avoid false failures: + +| Metric | Max % Worse | Min Absolute Diff | +|--------|-------------|-------------------| +| p50 | 100% | 0.5s | +| p95 | 100% | 1.0s | +| p99 | 100% | 1.0s | + +**Example:** If old p50 = 0.01s and new p50 = 0.08s: +- Percentage difference: +700% (would fail % check) +- Absolute difference: 0.07s (< 0.5s threshold) +- **Result: PASS** (both values are fast enough that the difference doesn't matter) + +### Resource Consumption Metrics + +These metrics track CPU, memory, and Go runtime resource usage. Lower values generally indicate more efficient operation. + +#### Memory Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `memory_rss_mb_avg` | Average RSS (resident set size) memory | MB | +| `memory_rss_mb_max` | Peak RSS memory during test | MB | +| `memory_heap_mb_avg` | Average Go heap allocation | MB | +| `memory_heap_mb_max` | Peak Go heap allocation | MB | + +**What to watch for:** +- **High RSS:** May indicate memory leaks or inefficient caching +- **High heap:** Many objects being created (check GC metrics) +- **Growing over time:** Potential memory leak + +#### CPU Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `cpu_cores_avg` | Average CPU usage rate | cores | +| `cpu_cores_max` | Peak CPU usage rate | cores | + +**What to watch for:** +- **High CPU:** Inefficient algorithms or excessive reconciles +- **Spiky max:** May indicate burst handling issues + +#### Go Runtime Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `goroutines_avg` | Average goroutine count | count | +| `goroutines_max` | Peak goroutine count | count | +| `gc_pause_p99_ms` | 99th percentile GC pause time | ms | + +**What to watch for:** +- **High goroutines:** Potential goroutine leak or unbounded concurrency +- **High GC pause:** Large heap or allocation pressure + +### Scenario-Specific Expectations + +| Scenario | Key Metrics to Watch | Expected Behavior | +|----------|---------------------|-------------------| +| S1 (Burst) | action_latency_p99, cpu_cores_max, goroutines_max | Should handle bursts without queue backup | +| S2 (Fan-Out) | reconcile_total, workloads_matched, memory_rss_mb_max | One CM change → 50 workload reloads | +| S3 (High Cardinality) | reconcile_duration, memory_heap_mb_avg | Many namespaces shouldn't increase memory | +| S4 (No-Op) | action_total = 0, cpu_cores_avg should be low | Minimal resource usage for no-op | +| S5 (Churn) | errors_total, goroutines_avg | Graceful handling, no goroutine leak | +| S6 (Restart) | All metrics captured | Metrics survive controller restart | +| S7 (API Pressure) | errors_total, cpu_cores_max, goroutines_max | No errors under concurrent load | +| S8 (Large Objects) | memory_rss_mb_max, gc_pause_p99_ms | Large ConfigMaps don't cause OOM or GC issues | +| S9 (Multi-Workload) | reload_executed_total per type | All workload types (Deploy, STS, DS) reload | +| S10 (Secrets) | reload_executed_total, workloads_matched | Both Secrets and ConfigMaps trigger reloads | +| S11 (Annotation) | workload annotations present | Deployments get `last-reloaded-from` annotation | +| S12 (Pause) | reload_executed_total << updates | Pause-period reduces reload frequency | +| S13 (Complex) | reload_executed_total | All reference types trigger reloads | + +### Troubleshooting + +#### New implementation shows 0 for all metrics +- Check if Prometheus is scraping the new Reloader pod +- Verify pod annotations: `prometheus.io/scrape: "true"` +- Check Prometheus targets: `http://localhost:9091/targets` + +#### Metrics don't match expected values +- Verify test ran to completion (check logs) +- Ensure Prometheus scraped final metrics (18s wait after test) +- Check for pod restarts during test (metrics reset on restart - handled by `increase()`) + +#### High latency in new implementation +- Check Reloader pod resource limits +- Look for API server throttling in logs +- Compare `reconcile_total` - fewer reconciles with higher duration may be normal + +#### REST client errors are non-zero +- **Common causes:** + - Optional CRD schemes registered but CRDs not installed (e.g., Argo Rollouts, OpenShift DeploymentConfig) + - API server rate limiting under high load + - RBAC permissions missing for certain resource types +- **Argo Rollouts errors:** If you see ~4 errors per test, ensure `--enable-argo-rollouts=false` if not using Argo Rollouts +- **OpenShift errors:** Similarly, ensure DeploymentConfig support is disabled on non-OpenShift clusters + +#### REST client requests much higher in new implementation +- Check if caching is working correctly +- Look for excessive re-queuing in controller logs +- Compare `reconcile_total` - more reconciles naturally means more API calls + +## Report Format + +The report generator produces a comparison table with units and expected value indicators: + +``` +================================================================================ + RELOADER A/B COMPARISON REPORT +================================================================================ + +Scenario: S2 +Generated: 2026-01-03 14:30:00 +Status: PASS +Summary: All metrics within acceptable thresholds + +Test: S2: Fan-out test - 1 CM update triggers 50 deployment reloads + +-------------------------------------------------------------------------------- + METRIC COMPARISONS +-------------------------------------------------------------------------------- +(Old✓/New✓ = meets expected value within 15%) + +Metric Old New Expected Old✓ New✓ Status +------ --- --- -------- ---- ---- ------ +reconcile_total 50.00 25.00 - - - pass +reconcile_duration_p50 (s) 0.01 0.05 - - - pass +reconcile_duration_p95 (s) 0.02 0.15 - - - pass +action_total 50.00 50.00 50 ✓ ✓ pass +action_latency_p50 (s) 0.05 0.03 - - - pass +action_latency_p95 (s) 0.12 0.08 - - - pass +errors_total 0.00 0.00 - - - pass +reload_executed_total 50.00 50.00 50 ✓ ✓ pass +workloads_scanned_total 50.00 50.00 50 ✓ ✓ pass +workloads_matched_total 50.00 50.00 50 ✓ ✓ pass +rest_client_requests_total 850 720 - - - pass +rest_client_requests_get 500 420 - - - pass +rest_client_requests_patch 300 250 - - - pass +rest_client_requests_errors 0 0 - - - pass +``` + +Reports are saved to `results//report.txt` after each test. + +## Directory Structure + +``` +test/loadtest/ +├── cmd/ +│ └── loadtest/ # Unified CLI (run + report) +│ └── main.go +├── internal/ +│ ├── cluster/ # Kind cluster management +│ │ └── kind.go +│ ├── prometheus/ # Prometheus deployment & querying +│ │ └── prometheus.go +│ ├── reloader/ # Reloader deployment +│ │ └── deploy.go +│ └── scenarios/ # Test scenario implementations +│ └── scenarios.go +├── manifests/ +│ └── prometheus.yaml # Prometheus deployment manifest +├── results/ # Generated after tests +│ └── / +│ ├── old/ # Old version data +│ │ ├── *.json # Prometheus metric snapshots +│ │ └── reloader.log # Reloader pod logs +│ ├── new/ # New version data +│ │ ├── *.json # Prometheus metric snapshots +│ │ └── reloader.log # Reloader pod logs +│ ├── expected.json # Expected values from test +│ └── report.txt # Comparison report +├── go.mod +├── go.sum +└── README.md +``` + +## Building Local Images for Testing + +If you want to test local code changes: + +```bash +# Build the new Reloader image from current source +docker build -t localhost/reloader:dev -f Dockerfile . + +# Build from a different branch/commit +git checkout feature-branch +docker build -t localhost/reloader:feature -f Dockerfile . + +# Then run comparison +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=localhost/reloader:feature +``` + +## Interpreting Results + +### PASS +All metrics are within acceptable thresholds. The new implementation is comparable or better than the old one. + +### FAIL +One or more metrics exceeded thresholds. Review the specific metrics: +- **Latency degradation**: p95/p99 latencies are significantly higher +- **Missed reloads**: `reload_executed_total` differs significantly +- **Errors increased**: `errors_total` is higher in new version + +### Investigation + +If tests fail, check: +1. Pod logs: `kubectl logs -n reloader-new deployment/reloader` (or check `results//new/reloader.log`) +2. Resource usage: `kubectl top pods -n reloader-new` +3. Events: `kubectl get events -n reloader-test` + +## Parallel Execution + +The `--parallelism` option enables running scenarios on multiple kind clusters simultaneously, significantly reducing total test time. + +### How It Works + +1. **Multiple Clusters**: Creates N kind clusters named `reloader-loadtest-0`, `reloader-loadtest-1`, etc. +2. **Separate Prometheus**: Each cluster gets its own Prometheus instance with a unique port (9091, 9092, etc.) +3. **Worker Pool**: Scenarios are distributed to workers via a channel, with each worker running on its own cluster +4. **Independent Execution**: Each scenario runs in complete isolation with no resource contention + +### Usage + +```bash +# Run 4 scenarios at a time (creates 4 clusters) +./loadtest run --new-image=my-image:tag --parallelism=4 + +# Run all 13 scenarios in parallel (creates 13 clusters) +./loadtest run --new-image=my-image:tag --parallelism=13 --scenario=all +``` + +### Resource Requirements + +Parallel execution requires significant system resources: + +| Parallelism | Clusters | Est. Memory | Est. CPU | +|-------------|----------|-------------|----------| +| 1 (default) | 1 | ~4GB | 2-4 cores | +| 4 | 4 | ~16GB | 8-16 cores | +| 13 | 13 | ~52GB | 26-52 cores | + +### Notes + +- The `--skip-cluster` option is not supported with parallelism > 1 +- Each worker loads images independently, so initial setup takes longer +- All results are written to the same `--results-dir` with per-scenario subdirectories +- If a cluster setup fails, remaining workers continue with available clusters +- Parallelism automatically reduces to match scenario count if set higher + +## CI Integration + +### GitHub Actions + +Load tests can be triggered on pull requests by commenting `/loadtest`: + +``` +/loadtest +``` + +This will: +1. Build a container image from the PR branch +2. Run all load test scenarios against it +3. Post results as a PR comment +4. Upload detailed results as artifacts + +### Make Target + +Run load tests locally or in CI: + +```bash +# From repository root +make loadtest +``` + +This builds the container image and runs all scenarios with a 60-second duration. diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go new file mode 100644 index 000000000..19f7b1dba --- /dev/null +++ b/test/loadtest/cmd/loadtest/main.go @@ -0,0 +1,1582 @@ +// Package main implements the unified load test CLI for Reloader A/B comparison. +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/stakater/Reloader/test/loadtest/internal/cluster" + "github.com/stakater/Reloader/test/loadtest/internal/prometheus" + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + "github.com/stakater/Reloader/test/loadtest/internal/scenarios" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + clusterName = "reloader-loadtest" + testNamespace = "reloader-test" +) + +// workerContext holds all resources for a single worker (cluster + prometheus). +type workerContext struct { + id int + clusterMgr *cluster.Manager + promMgr *prometheus.Manager + kubeClient kubernetes.Interface + kubeContext string + runtime string +} + +// Config holds CLI configuration. +type Config struct { + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + ResultsDir string + ManifestsDir string + Parallelism int // Number of parallel clusters (1 = sequential) +} + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + cmd := os.Args[1] + switch cmd { + case "run": + runCommand(os.Args[2:]) + case "report": + reportCommand(os.Args[2:]) + case "help", "--help", "-h": + printUsage() + default: + fmt.Printf("Unknown command: %s\n", cmd) + printUsage() + os.Exit(1) + } +} + +func printUsage() { + fmt.Print(`Reloader Load Test CLI + +Usage: + loadtest run [options] Run A/B comparison tests + loadtest report [options] Generate comparison report + loadtest help Show this help + +Run Options: + --old-image=IMAGE Container image for "old" version (required for comparison) + --new-image=IMAGE Container image for "new" version (required for comparison) + --scenario=ID Test scenario: S1-S13 or "all" (default: all) + --duration=SECONDS Test duration in seconds (default: 60) + --parallelism=N Run N scenarios in parallel on N clusters (default: 1) + --skip-cluster Skip kind cluster creation (use existing, only for parallelism=1) + --results-dir=DIR Directory for results (default: ./results) + +Report Options: + --scenario=ID Scenario to report on (required) + --results-dir=DIR Directory containing results (default: ./results) + --output=FILE Output file (default: stdout) + +Examples: + # Compare two images + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 + + # Run specific scenario + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 + + # Test single image (no comparison) + loadtest run --new-image=localhost/reloader:test + + # Run all scenarios in parallel on 4 clusters + loadtest run --new-image=localhost/reloader:test --parallelism=4 + + # Run all 13 scenarios in parallel (one cluster per scenario) + loadtest run --new-image=localhost/reloader:test --parallelism=13 + + # Generate report + loadtest report --scenario=S2 --results-dir=./results +`) +} + +func parseArgs(args []string) Config { + cfg := Config{ + Scenario: "all", + Duration: 60, + ResultsDir: "./results", + Parallelism: 1, + } + + // Find manifests dir relative to executable or current dir + execPath, _ := os.Executable() + execDir := filepath.Dir(execPath) + cfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") + if _, err := os.Stat(cfg.ManifestsDir); os.IsNotExist(err) { + // Try relative to current dir + cfg.ManifestsDir = "./manifests" + } + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--old-image="): + cfg.OldImage = strings.TrimPrefix(arg, "--old-image=") + case strings.HasPrefix(arg, "--new-image="): + cfg.NewImage = strings.TrimPrefix(arg, "--new-image=") + case strings.HasPrefix(arg, "--scenario="): + cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") + case strings.HasPrefix(arg, "--duration="): + fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration) + case strings.HasPrefix(arg, "--parallelism="): + fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism) + case arg == "--skip-cluster": + cfg.SkipCluster = true + case strings.HasPrefix(arg, "--results-dir="): + cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--manifests-dir="): + cfg.ManifestsDir = strings.TrimPrefix(arg, "--manifests-dir=") + } + } + + // Validate parallelism + if cfg.Parallelism < 1 { + cfg.Parallelism = 1 + } + + return cfg +} + +func runCommand(args []string) { + cfg := parseArgs(args) + + // Validate required args + if cfg.OldImage == "" && cfg.NewImage == "" { + log.Fatal("At least one of --old-image or --new-image is required") + } + + // Determine mode + runOld := cfg.OldImage != "" + runNew := cfg.NewImage != "" + runBoth := runOld && runNew + + log.Printf("Configuration:") + log.Printf(" Scenario: %s", cfg.Scenario) + log.Printf(" Duration: %ds", cfg.Duration) + log.Printf(" Parallelism: %d", cfg.Parallelism) + if cfg.OldImage != "" { + log.Printf(" Old image: %s", cfg.OldImage) + } + if cfg.NewImage != "" { + log.Printf(" New image: %s", cfg.NewImage) + } + + // Detect container runtime + runtime, err := cluster.DetectContainerRuntime() + if err != nil { + log.Fatalf("Failed to detect container runtime: %v", err) + } + log.Printf(" Container runtime: %s", runtime) + + // Setup context with signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Println("Received shutdown signal...") + cancel() + }() + + // Determine scenarios to run + scenariosToRun := []string{cfg.Scenario} + if cfg.Scenario == "all" { + scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } + + // Skip-cluster only works for parallelism=1 + if cfg.SkipCluster && cfg.Parallelism > 1 { + log.Fatal("--skip-cluster is not supported with --parallelism > 1") + } + + // If parallelism > 1, use parallel execution + if cfg.Parallelism > 1 { + runParallel(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) + return + } + + // Sequential execution (parallelism == 1) + runSequential(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) +} + +// runSequential runs scenarios one by one on a single cluster. +func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + // Create cluster manager + clusterMgr := cluster.NewManager(cluster.Config{ + Name: clusterName, + ContainerRuntime: runtime, + }) + + // Create/verify cluster + if cfg.SkipCluster { + log.Println("Skipping cluster creation (using existing)") + if !clusterMgr.Exists() { + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", clusterName) + } + } else { + log.Println("Creating kind cluster...") + if err := clusterMgr.Create(ctx); err != nil { + log.Fatalf("Failed to create cluster: %v", err) + } + } + + // Deploy Prometheus + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManager(promManifest) + + log.Println("Installing Prometheus...") + if err := promMgr.Deploy(ctx); err != nil { + log.Fatalf("Failed to deploy Prometheus: %v", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + log.Fatalf("Failed to start Prometheus port-forward: %v", err) + } + defer promMgr.StopPortForward() + + // Load images into kind + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } + } + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } + } + + // Pre-pull test images + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) // Ignore errors + + // Get kubernetes client + kubeClient, err := getKubeClient("") + if err != nil { + log.Fatalf("Failed to create kubernetes client: %v", err) + } + + for _, scenarioID := range scenariosToRun { + log.Printf("========================================") + log.Printf("=== Starting scenario %s ===", scenarioID) + log.Printf("========================================") + + // Clean up from previous scenario + cleanupTestNamespaces(ctx, "") + cleanupReloader(ctx, "old", "") + cleanupReloader(ctx, "new", "") + + // Reset Prometheus + if err := promMgr.Reset(ctx); err != nil { + log.Printf("Warning: failed to reset Prometheus: %v", err) + } + + // Create test namespace + createTestNamespace(ctx, "") + + if runOld { + // Test old version + oldMgr := reloader.NewManager(reloader.Config{ + Version: "old", + Image: cfg.OldImage, + }) + + if err := oldMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy old Reloader: %v", err) + continue + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) + collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) + + if runBoth { + // Clean up for new version + cleanupTestNamespaces(ctx, "") + oldMgr.Cleanup(ctx) + promMgr.Reset(ctx) + createTestNamespace(ctx, "") + } + } + + if runNew { + // Test new version + newMgr := reloader.NewManager(reloader.Config{ + Version: "new", + Image: cfg.NewImage, + }) + + if err := newMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy new Reloader: %v", err) + continue + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) + collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) + } + + // Generate report + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + log.Printf("=== Scenario %s complete ===", scenarioID) + } + + log.Println("Load test complete!") + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +// runParallel runs scenarios in parallel on N separate kind clusters. +func runParallel(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + numWorkers := cfg.Parallelism + if numWorkers > len(scenariosToRun) { + numWorkers = len(scenariosToRun) + log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) + } + + log.Printf("Starting parallel execution with %d workers", numWorkers) + + // Create workers + workers := make([]*workerContext, numWorkers) + var setupWg sync.WaitGroup + setupErrors := make(chan error, numWorkers) + + log.Println("Setting up worker clusters...") + for i := 0; i < numWorkers; i++ { + setupWg.Add(1) + go func(workerID int) { + defer setupWg.Done() + worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) + if err != nil { + setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) + return + } + workers[workerID] = worker + }(i) + } + + setupWg.Wait() + close(setupErrors) + + // Check for setup errors + for err := range setupErrors { + log.Printf("Error: %v", err) + } + + // Verify all workers are ready + readyWorkers := 0 + for _, w := range workers { + if w != nil { + readyWorkers++ + } + } + if readyWorkers == 0 { + log.Fatal("No workers ready, aborting") + } + if readyWorkers < numWorkers { + log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) + } + + // Cleanup workers on exit + defer func() { + log.Println("Cleaning up worker clusters...") + for _, w := range workers { + if w != nil { + w.promMgr.StopPortForward() + } + } + }() + + // Create scenario channel + scenarioCh := make(chan string, len(scenariosToRun)) + for _, s := range scenariosToRun { + scenarioCh <- s + } + close(scenarioCh) + + // Results tracking + var resultsMu sync.Mutex + completedScenarios := make([]string, 0, len(scenariosToRun)) + + // Start workers + var wg sync.WaitGroup + for _, worker := range workers { + if worker == nil { + continue + } + wg.Add(1) + go func(w *workerContext) { + defer wg.Done() + for scenarioID := range scenarioCh { + select { + case <-ctx.Done(): + return + default: + } + + log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) + + // Clean up from previous scenario + cleanupTestNamespaces(ctx, w.kubeContext) + cleanupReloader(ctx, "old", w.kubeContext) + cleanupReloader(ctx, "new", w.kubeContext) + + // Reset Prometheus + if err := w.promMgr.Reset(ctx); err != nil { + log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) + } + + // Create test namespace + createTestNamespace(ctx, w.kubeContext) + + if runOld { + runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) + } + + if runNew { + runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) + } + + // Generate report + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + resultsMu.Lock() + completedScenarios = append(completedScenarios, scenarioID) + resultsMu.Unlock() + + log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) + } + }(worker) + } + + wg.Wait() + + log.Println("Load test complete!") + log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +// setupWorker creates a cluster and deploys prometheus for a single worker. +func setupWorker(ctx context.Context, cfg Config, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { + workerName := fmt.Sprintf("%s-%d", clusterName, workerID) + promPort := 9091 + workerID + + log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) + + clusterMgr := cluster.NewManager(cluster.Config{ + Name: workerName, + ContainerRuntime: runtime, + PortOffset: workerID, // Each cluster gets unique ports + }) + + if err := clusterMgr.Create(ctx); err != nil { + return nil, fmt.Errorf("creating cluster: %w", err) + } + + kubeContext := clusterMgr.Context() + + // Deploy Prometheus + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) + + log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) + if err := promMgr.Deploy(ctx); err != nil { + return nil, fmt.Errorf("deploying prometheus: %w", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + return nil, fmt.Errorf("starting prometheus port-forward: %w", err) + } + + // Load images + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } + } + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } + } + + // Pre-pull test images + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + // Get kubernetes client for this context + kubeClient, err := getKubeClient(kubeContext) + if err != nil { + return nil, fmt.Errorf("creating kubernetes client: %w", err) + } + + log.Printf("[Worker %d] Ready", workerID) + return &workerContext{ + id: workerID, + clusterMgr: clusterMgr, + promMgr: promMgr, + kubeClient: kubeClient, + kubeContext: kubeContext, + runtime: runtime, + }, nil +} + +// runVersionOnWorker runs a single version test on a worker. +func runVersionOnWorker(ctx context.Context, w *workerContext, cfg Config, scenarioID, version, image string, cleanupAfter bool) { + mgr := reloader.NewManager(reloader.Config{ + Version: version, + Image: image, + }) + mgr.SetKubeContext(w.kubeContext) + + if err := mgr.Deploy(ctx); err != nil { + log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) + return + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { + log.Printf("[Worker %d] Warning: %v", w.id, err) + log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) + } + + runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) + collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) + + if cleanupAfter { + cleanupTestNamespaces(ctx, w.kubeContext) + mgr.Cleanup(ctx) + w.promMgr.Reset(ctx) + createTestNamespace(ctx, w.kubeContext) + } +} + +func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { + runner, ok := scenarios.Registry[scenarioID] + if !ok { + log.Printf("Unknown scenario: %s", scenarioID) + return + } + + // For S6, set the reloader version + if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { + s6.ReloaderVersion = version + } + + // For S11, set the image to deploy its own Reloader + if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { + s11.Image = image + } + + log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) + + // Debug: check parent context state + if ctx.Err() != nil { + log.Printf("WARNING: Parent context already done: %v", ctx.Err()) + } + + // Add extra time for scenario setup (creating deployments, waiting for ready state) + // Some scenarios like S2 create 50 deployments which can take 2-3 minutes + timeout := time.Duration(duration)*time.Second + 5*time.Minute + log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) + + scenarioCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + expected, err := runner.Run(scenarioCtx, client, testNamespace, time.Duration(duration)*time.Second) + if err != nil { + log.Printf("Scenario %s failed: %v", scenarioID, err) + } + + scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) +} + +func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { + log.Printf("Waiting 5s for Reloader to finish processing events...") + time.Sleep(5 * time.Second) + + log.Printf("Waiting 8s for Prometheus to scrape final metrics...") + time.Sleep(8 * time.Second) + + log.Printf("Collecting metrics for %s...", version) + outputDir := filepath.Join(resultsDir, scenarioID, version) + if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { + log.Printf("Failed to collect metrics: %v", err) + } +} + +func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { + log.Printf("Collecting logs for %s...", version) + logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") + if err := mgr.CollectLogs(ctx, logPath); err != nil { + log.Printf("Failed to collect logs: %v", err) + } +} + +func generateReport(scenarioID, resultsDir string, isComparison bool) { + if isComparison { + log.Println("Generating comparison report...") + } else { + log.Println("Generating single-version report...") + } + + reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") + + // Use the report command + cmd := exec.Command(os.Args[0], "report", + fmt.Sprintf("--scenario=%s", scenarioID), + fmt.Sprintf("--results-dir=%s", resultsDir), + fmt.Sprintf("--output=%s", reportPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Run() + + // Also print to stdout + if data, err := os.ReadFile(reportPath); err == nil { + fmt.Println(string(data)) + } + + log.Printf("Report saved to: %s", reportPath) +} + +func getKubeClient(kubeContext string) (kubernetes.Interface, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, _ := os.UserHomeDir() + kubeconfig = filepath.Join(home, ".kube", "config") + } + + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} + configOverrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + configOverrides.CurrentContext = kubeContext + } + + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(config) +} + +func createTestNamespace(ctx context.Context, kubeContext string) { + args := []string{"create", "namespace", testNamespace, "--dry-run=client", "-o", "yaml"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, _ := cmd.Output() + + applyArgs := []string{"apply", "-f", "-"} + if kubeContext != "" { + applyArgs = append([]string{"--context", kubeContext}, applyArgs...) + } + applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) + applyCmd.Stdin = strings.NewReader(string(out)) + applyCmd.Run() +} + +func cleanupTestNamespaces(ctx context.Context, kubeContext string) { + log.Println("Cleaning up test resources...") + + // Main namespace + S3 extra namespaces + namespaces := []string{testNamespace} + for i := 0; i < 10; i++ { + namespaces = append(namespaces, fmt.Sprintf("%s-%d", testNamespace, i)) + } + + for _, ns := range namespaces { + args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } + + // Wait a bit for cleanup + time.Sleep(2 * time.Second) + + // Force delete remaining pods + for _, ns := range namespaces { + args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } +} + +func cleanupReloader(ctx context.Context, version string, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} + +// ============================================================================ +// REPORT COMMAND +// ============================================================================ + +func reportCommand(args []string) { + var scenarioID, resultsDir, outputFile string + resultsDir = "./results" + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--scenario="): + scenarioID = strings.TrimPrefix(arg, "--scenario=") + case strings.HasPrefix(arg, "--results-dir="): + resultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--output="): + outputFile = strings.TrimPrefix(arg, "--output=") + } + } + + if scenarioID == "" { + log.Fatal("--scenario is required for report command") + } + + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Fatalf("Failed to generate report: %v", err) + } + + output := renderScenarioReport(report) + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Report written to %s", outputFile) + } else { + fmt.Println(output) + } +} + +// PrometheusResponse represents a Prometheus API response for report parsing. +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// MetricComparison represents the comparison of a single metric. +type MetricComparison struct { + Name string + DisplayName string + Unit string + IsCounter bool + OldValue float64 + NewValue float64 + Expected float64 + Difference float64 + DiffPct float64 + Status string + Threshold float64 + OldMeetsExpected string + NewMeetsExpected string +} + +type metricInfo struct { + unit string + isCounter bool +} + +var metricInfoMap = map[string]metricInfo{ + "reconcile_total": {unit: "count", isCounter: true}, + "reconcile_duration_p50": {unit: "s", isCounter: false}, + "reconcile_duration_p95": {unit: "s", isCounter: false}, + "reconcile_duration_p99": {unit: "s", isCounter: false}, + "action_total": {unit: "count", isCounter: true}, + "action_latency_p50": {unit: "s", isCounter: false}, + "action_latency_p95": {unit: "s", isCounter: false}, + "action_latency_p99": {unit: "s", isCounter: false}, + "errors_total": {unit: "count", isCounter: true}, + "reload_executed_total": {unit: "count", isCounter: true}, + "workloads_scanned_total": {unit: "count", isCounter: true}, + "workloads_matched_total": {unit: "count", isCounter: true}, + "skipped_total_no_data_change": {unit: "count", isCounter: true}, + "rest_client_requests_total": {unit: "count", isCounter: true}, + "rest_client_requests_get": {unit: "count", isCounter: true}, + "rest_client_requests_patch": {unit: "count", isCounter: true}, + "rest_client_requests_put": {unit: "count", isCounter: true}, + "rest_client_requests_errors": {unit: "count", isCounter: true}, + + // Resource consumption metrics (gauges, not counters) + "memory_rss_mb_avg": {unit: "MB", isCounter: false}, + "memory_rss_mb_max": {unit: "MB", isCounter: false}, + "memory_heap_mb_avg": {unit: "MB", isCounter: false}, + "memory_heap_mb_max": {unit: "MB", isCounter: false}, + "cpu_cores_avg": {unit: "cores", isCounter: false}, + "cpu_cores_max": {unit: "cores", isCounter: false}, + "goroutines_avg": {unit: "count", isCounter: false}, + "goroutines_max": {unit: "count", isCounter: false}, + "gc_pause_p99_ms": {unit: "ms", isCounter: false}, +} + +// ReportExpectedMetrics matches the expected metrics from test scenarios. +type ReportExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// ScenarioReport represents the full report for a scenario. +type ScenarioReport struct { + Scenario string + Timestamp time.Time + Comparisons []MetricComparison + OverallStatus string + Summary string + PassCriteria []string + FailedCriteria []string + Expected ReportExpectedMetrics + TestDescription string +} + +// MetricType defines how to evaluate a metric. +type MetricType int + +const ( + LowerIsBetter MetricType = iota + ShouldMatch + HigherIsBetter + Informational // Reports values but doesn't affect pass/fail +) + +type ThresholdConfig struct { + maxDiff float64 + metricType MetricType + minAbsDiff float64 +} + +var thresholds = map[string]ThresholdConfig{ + "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, + "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, + "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, + // API metrics - use minAbsDiff to allow small differences + "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, // Pass if both < 100 + + // Resource consumption metrics + "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB + "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, // 50% or 30MB + "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, // 50% or 15MB + "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB + "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, // 100% or 0.1 cores + "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, // 100% or 0.2 cores + "goroutines_avg": {metricType: Informational}, // Info only - different architectures may use more goroutines + "goroutines_max": {metricType: Informational}, // Info only - different architectures may use more goroutines + "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, // 100% or 5ms +} + +func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { + oldDir := filepath.Join(resultsDir, scenario, "old") + newDir := filepath.Join(resultsDir, scenario, "new") + scenarioDir := filepath.Join(resultsDir, scenario) + + // Check which directories exist to determine mode + _, oldErr := os.Stat(oldDir) + _, newErr := os.Stat(newDir) + hasOld := oldErr == nil + hasNew := newErr == nil + isComparison := hasOld && hasNew + + // For single-version mode, determine which version we have + singleVersion := "" + singleDir := "" + if !isComparison { + if hasNew { + singleVersion = "new" + singleDir = newDir + } else if hasOld { + singleVersion = "old" + singleDir = oldDir + } else { + return nil, fmt.Errorf("no results found in %s", scenarioDir) + } + } + + report := &ScenarioReport{ + Scenario: scenario, + Timestamp: time.Now(), + } + + // Load expected metrics + expectedPath := filepath.Join(scenarioDir, "expected.json") + if data, err := os.ReadFile(expectedPath); err == nil { + if err := json.Unmarshal(data, &report.Expected); err != nil { + log.Printf("Warning: Could not parse expected metrics: %v", err) + } else { + report.TestDescription = report.Expected.Description + } + } + + // Handle single-version mode + if !isComparison { + return generateSingleVersionReport(report, singleDir, singleVersion, scenario) + } + + // Define metrics to compare + metricsToCompare := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + + // Resource consumption metrics + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + // Build expected values map + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + // First pass: collect all metric values + metricValues := make(map[string]struct{ old, new, expected float64 }) + + for _, m := range metricsToCompare { + oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load old metric %s: %v", m.name, err) + continue + } + + newData, err := loadMetricFile(filepath.Join(newDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load new metric %s: %v", m.name, err) + continue + } + + oldValue := m.selector(oldData) + newValue := m.selector(newData) + expected := expectedValues[m.name] + + metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} + } + + // Check context for smart pass/fail decisions + newMeetsActionExpected := false + newReconcileIsZero := false + isChurnScenario := scenario == "S5" // Workload churn has special pass/fail rules + if v, ok := metricValues["action_total"]; ok && v.expected > 0 { + tolerance := v.expected * 0.15 + newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance + } + if v, ok := metricValues["reconcile_total"]; ok { + newReconcileIsZero = v.new == 0 + } + + // Second pass: generate comparisons with context awareness + for _, m := range metricsToCompare { + v, ok := metricValues[m.name] + if !ok { + continue + } + + comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) + + // Context-aware adjustments for API metrics + if strings.HasPrefix(m.name, "rest_client_requests") { + // If new correctly processed all expected reloads but old didn't, + // higher API calls in new is expected (it's doing the work correctly) + if newMeetsActionExpected && comparison.Status != "pass" { + if oldMeets, ok := metricValues["action_total"]; ok { + oldTolerance := oldMeets.expected * 0.15 + oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance + if oldMissed { + comparison.Status = "pass" + } + } + } + // If new has 0 reconciles (no-op scenario), API differences are fine + if newReconcileIsZero && comparison.Status != "pass" { + comparison.Status = "pass" + } + } + + // S5 (Workload Churn) specific adjustments: + // - "Not found" errors are expected when deployments are deleted during processing + // - No expected values for throughput, so compare old vs new (should be similar) + if isChurnScenario { + if m.name == "errors_total" { + // In churn scenarios, "not found" errors are expected when workloads + // are deleted while Reloader is processing them. Allow up to 50 errors. + if v.new < 50 && v.old < 50 { + comparison.Status = "pass" + } else if v.new <= v.old*1.5 { + // Also pass if new has similar or fewer errors than old + comparison.Status = "pass" + } + } + if m.name == "action_total" || m.name == "reload_executed_total" { + // No expected value for churn - compare old vs new + // Both should be similar (within 20% of each other) + if v.old > 0 { + diff := math.Abs(v.new-v.old) / v.old * 100 + if diff <= 20 { + comparison.Status = "pass" + } + } else if v.new > 0 { + // Old is 0, new has value - that's fine + comparison.Status = "pass" + } + } + } + + report.Comparisons = append(report.Comparisons, comparison) + + if comparison.Status == "pass" { + report.PassCriteria = append(report.PassCriteria, m.name) + } else if comparison.Status == "fail" { + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + + // Determine overall status + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = "All metrics within acceptable thresholds" + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +// generateSingleVersionReport creates a report for a single version (no comparison). +func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { + // Define metrics to collect + metricsToCollect := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + // Build expected values map + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + for _, m := range metricsToCollect { + data, err := loadMetricFile(filepath.Join(dataDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load metric %s: %v", m.name, err) + continue + } + + value := m.selector(data) + expected := expectedValues[m.name] + + info := metricInfoMap[m.name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + + displayName := m.name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) + } + + // For single-version, put the value in NewValue column + status := "info" + meetsExp := "-" + + // Check against expected if available + if expected > 0 { + meetsExp = meetsExpected(value, expected) + threshold, ok := thresholds[m.name] + if ok && threshold.metricType == ShouldMatch { + if meetsExp == "✓" { + status = "pass" + report.PassCriteria = append(report.PassCriteria, m.name) + } else { + status = "fail" + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + } + + if info.isCounter { + value = math.Round(value) + } + + report.Comparisons = append(report.Comparisons, MetricComparison{ + Name: m.name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + OldValue: 0, // No old value in single-version mode + NewValue: value, + Expected: expected, + OldMeetsExpected: "-", + NewMeetsExpected: meetsExp, + Status: status, + }) + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func loadMetricFile(path string) (PrometheusResponse, error) { + var resp PrometheusResponse + data, err := os.ReadFile(path) + if err != nil { + return resp, err + } + err = json.Unmarshal(data, &resp) + return resp, err +} + +func sumAllValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + return sum +} + +func sumSuccessValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if result.Metric["success"] == "true" { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + } + return sum +} + +func getFirstValue(data PrometheusResponse) float64 { + if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { + if v, ok := data.Data.Result[0].Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + return f + } + } + return 0 +} + +// bytesToMB converts bytes to megabytes. +func bytesToMB(data PrometheusResponse) float64 { + bytes := getFirstValue(data) + return bytes / (1024 * 1024) +} + +// secondsToMs converts seconds to milliseconds. +func secondsToMs(data PrometheusResponse) float64 { + seconds := getFirstValue(data) + return seconds * 1000 +} + +func meetsExpected(value, expected float64) string { + if expected == 0 { + return "-" + } + tolerance := expected * 0.15 + if math.Abs(value-expected) <= tolerance { + return "✓" + } + return "✗" +} + +func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { + diff := newValue - oldValue + absDiff := math.Abs(diff) + var diffPct float64 + if oldValue != 0 { + diffPct = (diff / oldValue) * 100 + } else if newValue != 0 { + diffPct = 100 + } + + threshold, ok := thresholds[name] + if !ok { + threshold = ThresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} + } + + info := metricInfoMap[name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + displayName := name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", name, info.unit) + } + + if info.isCounter { + oldValue = math.Round(oldValue) + newValue = math.Round(newValue) + } + + status := "pass" + oldMeetsExp := meetsExpected(oldValue, expected) + newMeetsExp := meetsExpected(newValue, expected) + + if expected > 0 && threshold.metricType == ShouldMatch { + if newMeetsExp == "✗" { + status = "fail" + } + } else { + switch threshold.metricType { + case LowerIsBetter: + if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { + status = "pass" + } else if diffPct > threshold.maxDiff { + status = "fail" + } + case HigherIsBetter: + if diffPct < -threshold.maxDiff { + status = "fail" + } + case ShouldMatch: + if math.Abs(diffPct) > threshold.maxDiff { + status = "fail" + } + case Informational: + status = "info" + } + } + + return MetricComparison{ + Name: name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + Expected: expected, + OldMeetsExpected: oldMeetsExp, + NewMeetsExpected: newMeetsExp, + OldValue: oldValue, + NewValue: newValue, + Difference: diff, + DiffPct: diffPct, + Status: status, + Threshold: threshold.maxDiff, + } +} + +func renderScenarioReport(report *ScenarioReport) string { + var sb strings.Builder + + // Detect single-version mode by checking if all OldValues are 0 + isSingleVersion := true + for _, c := range report.Comparisons { + if c.OldValue != 0 { + isSingleVersion = false + break + } + } + + sb.WriteString("\n") + sb.WriteString("================================================================================\n") + if isSingleVersion { + sb.WriteString(" RELOADER TEST REPORT\n") + } else { + sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") + } + sb.WriteString("================================================================================\n\n") + + fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) + fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) + fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) + + if report.TestDescription != "" { + fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) + } + + if report.Expected.ActionTotal > 0 { + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" EXPECTED VALUES\n") + sb.WriteString("--------------------------------------------------------------------------------\n") + fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) + fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) + if report.Expected.SkippedTotal > 0 { + fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + if isSingleVersion { + sb.WriteString(" METRICS\n") + } else { + sb.WriteString(" METRIC COMPARISONS\n") + } + sb.WriteString("--------------------------------------------------------------------------------\n") + + if isSingleVersion { + sb.WriteString("(✓ = meets expected value within 15%)\n\n") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "Metric", "Value", "Expected", "Met?", "Status") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "------", "-----", "--------", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", + c.DisplayName, c.NewValue, c.Expected, + c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } + } else { + sb.WriteString("(Old✓/New✓ = meets expected value within 15%)\n\n") + + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "Metric", "Old", "New", "Expected", "Old✓", "New✓", "Status") + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "------", "---", "---", "--------", "----", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, c.Expected, + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" PASS/FAIL CRITERIA\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) + for _, p := range report.PassCriteria { + fmt.Fprintf(&sb, " ✓ %s\n", p) + } + + if len(report.FailedCriteria) > 0 { + fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) + for _, f := range report.FailedCriteria { + fmt.Fprintf(&sb, " ✗ %s\n", f) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" THRESHOLDS USED\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "Metric", "Max Diff%", "Min Abs Diff", "Direction") + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "------", "---------", "------------", "---------") + + // Sort threshold names + var names []string + for name := range thresholds { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + t := thresholds[name] + var direction string + switch t.metricType { + case LowerIsBetter: + direction = "lower is better" + case HigherIsBetter: + direction = "higher is better" + case ShouldMatch: + direction = "should match" + case Informational: + direction = "info only" + } + minAbsDiff := "-" + if t.minAbsDiff > 0 { + minAbsDiff = fmt.Sprintf("%.1fs", t.minAbsDiff) + } + fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", + name, t.maxDiff, minAbsDiff, direction) + } + + sb.WriteString("\n================================================================================\n") + + return sb.String() +} diff --git a/test/loadtest/go.mod b/test/loadtest/go.mod new file mode 100644 index 000000000..ed5288210 --- /dev/null +++ b/test/loadtest/go.mod @@ -0,0 +1,50 @@ +module github.com/stakater/Reloader/test/loadtest + +go 1.22.0 + +require ( + k8s.io/api v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/test/loadtest/go.sum b/test/loadtest/go.sum new file mode 100644 index 000000000..a8edbda5a --- /dev/null +++ b/test/loadtest/go.sum @@ -0,0 +1,154 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go new file mode 100644 index 000000000..bcd5e19c6 --- /dev/null +++ b/test/loadtest/internal/cluster/kind.go @@ -0,0 +1,313 @@ +// Package cluster provides kind cluster management functionality. +package cluster + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// Config holds configuration for kind cluster operations. +type Config struct { + Name string + ContainerRuntime string // "docker" or "podman" + PortOffset int // Offset for host port mappings (for parallel clusters) +} + +// Manager handles kind cluster operations. +type Manager struct { + cfg Config +} + +// NewManager creates a new cluster manager. +func NewManager(cfg Config) *Manager { + return &Manager{cfg: cfg} +} + +// DetectContainerRuntime finds available container runtime. +func DetectContainerRuntime() (string, error) { + if _, err := exec.LookPath("podman"); err == nil { + return "podman", nil + } + if _, err := exec.LookPath("docker"); err == nil { + return "docker", nil + } + return "", fmt.Errorf("neither docker nor podman found in PATH") +} + +// Exists checks if the cluster already exists. +func (m *Manager) Exists() bool { + cmd := exec.Command("kind", "get", "clusters") + out, err := cmd.Output() + if err != nil { + return false + } + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == m.cfg.Name { + return true + } + } + return false +} + +// Delete deletes the kind cluster. +func (m *Manager) Delete(ctx context.Context) error { + cmd := exec.CommandContext(ctx, "kind", "delete", "cluster", "--name", m.cfg.Name) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// Create creates a new kind cluster with optimized settings. +func (m *Manager) Create(ctx context.Context) error { + if m.cfg.ContainerRuntime == "podman" { + os.Setenv("KIND_EXPERIMENTAL_PROVIDER", "podman") + } + + if m.Exists() { + fmt.Printf("Cluster %s already exists, deleting...\n", m.cfg.Name) + if err := m.Delete(ctx); err != nil { + return fmt.Errorf("deleting existing cluster: %w", err) + } + } + + // Calculate unique ports based on offset (for parallel clusters) + httpPort := 8080 + m.cfg.PortOffset + httpsPort := 8443 + m.cfg.PortOffset + + config := fmt.Sprintf(`kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + max-requests-inflight: "800" + max-mutating-requests-inflight: "400" + watch-cache-sizes: "configmaps#1000,secrets#1000,pods#1000" + controllerManager: + extraArgs: + kube-api-qps: "200" + kube-api-burst: "200" + scheduler: + extraArgs: + kube-api-qps: "200" + kube-api-burst: "200" + extraPortMappings: + - containerPort: 80 + hostPort: %d + protocol: TCP + - containerPort: 443 + hostPort: %d + protocol: TCP +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +`, httpPort, httpsPort) + cmd := exec.CommandContext(ctx, "kind", "create", "cluster", "--name", m.cfg.Name, "--config=-") + cmd.Stdin = strings.NewReader(config) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// GetKubeconfig returns the kubeconfig for the cluster. +func (m *Manager) GetKubeconfig() (string, error) { + cmd := exec.Command("kind", "get", "kubeconfig", "--name", m.cfg.Name) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("getting kubeconfig: %w", err) + } + return string(out), nil +} + +// Context returns the kubectl context name for this cluster. +func (m *Manager) Context() string { + return "kind-" + m.cfg.Name +} + +// Name returns the cluster name. +func (m *Manager) Name() string { + return m.cfg.Name +} + +// LoadImage loads a container image into the kind cluster. +func (m *Manager) LoadImage(ctx context.Context, image string) error { + // First check if image exists locally + if !m.imageExistsLocally(image) { + fmt.Printf(" Image not found locally, pulling: %s\n", image) + pullCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) + pullCmd.Stdout = os.Stdout + pullCmd.Stderr = os.Stderr + if err := pullCmd.Run(); err != nil { + return fmt.Errorf("pulling image %s: %w", image, err) + } + } else { + fmt.Printf(" Image found locally: %s\n", image) + } + + fmt.Printf(" Copying image to kind cluster...\n") + + if m.cfg.ContainerRuntime == "podman" { + // For podman, save to archive and load + tmpFile := fmt.Sprintf("/tmp/kind-image-%d.tar", time.Now().UnixNano()) + defer os.Remove(tmpFile) + + saveCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "save", image, "-o", tmpFile) + if err := saveCmd.Run(); err != nil { + return fmt.Errorf("saving image %s: %w", image, err) + } + + loadCmd := exec.CommandContext(ctx, "kind", "load", "image-archive", tmpFile, "--name", m.cfg.Name) + loadCmd.Stdout = os.Stdout + loadCmd.Stderr = os.Stderr + if err := loadCmd.Run(); err != nil { + return fmt.Errorf("loading image archive: %w", err) + } + } else { + loadCmd := exec.CommandContext(ctx, "kind", "load", "docker-image", image, "--name", m.cfg.Name) + loadCmd.Stdout = os.Stdout + loadCmd.Stderr = os.Stderr + if err := loadCmd.Run(); err != nil { + return fmt.Errorf("loading image %s: %w", image, err) + } + } + + return nil +} + +// imageExistsLocally checks if an image exists in the local container runtime. +func (m *Manager) imageExistsLocally(image string) bool { + // Try "image exists" command (works for podman) + cmd := exec.Command(m.cfg.ContainerRuntime, "image", "exists", image) + if err := cmd.Run(); err == nil { + return true + } + + // Try "image inspect" (works for both docker and podman) + cmd = exec.Command(m.cfg.ContainerRuntime, "image", "inspect", image) + if err := cmd.Run(); err == nil { + return true + } + + // Try listing images and grep + cmd = exec.Command(m.cfg.ContainerRuntime, "images", "--format", "{{.Repository}}:{{.Tag}}") + out, err := cmd.Output() + if err == nil { + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == image { + return true + } + } + } + + return false +} + +// PullImage pulls an image using the container runtime. +func (m *Manager) PullImage(ctx context.Context, image string) error { + cmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// ExecKubectl runs a kubectl command against the cluster. +func (m *Manager) ExecKubectl(ctx context.Context, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "kubectl", args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%w: %s", err, stderr.String()) + } + return stdout.Bytes(), nil +} diff --git a/test/loadtest/internal/prometheus/prometheus.go b/test/loadtest/internal/prometheus/prometheus.go new file mode 100644 index 000000000..f16df782b --- /dev/null +++ b/test/loadtest/internal/prometheus/prometheus.go @@ -0,0 +1,452 @@ +// Package prometheus provides Prometheus deployment and querying functionality. +package prometheus + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Manager handles Prometheus operations. +type Manager struct { + manifestPath string + portForward *exec.Cmd + localPort int + kubeContext string // Optional: use specific kubeconfig context +} + +// NewManager creates a new Prometheus manager. +func NewManager(manifestPath string) *Manager { + return &Manager{ + manifestPath: manifestPath, + localPort: 9091, // Use 9091 to avoid conflicts + } +} + +// NewManagerWithPort creates a Prometheus manager with a custom port. +func NewManagerWithPort(manifestPath string, port int, kubeContext string) *Manager { + return &Manager{ + manifestPath: manifestPath, + localPort: port, + kubeContext: kubeContext, + } +} + +// kubectl returns kubectl args with optional context +func (m *Manager) kubectl(args ...string) []string { + if m.kubeContext != "" { + return append([]string{"--context", m.kubeContext}, args...) + } + return args +} + +// Deploy deploys Prometheus to the cluster. +func (m *Manager) Deploy(ctx context.Context) error { + // Create namespace + cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("create", "namespace", "monitoring", "--dry-run=client", "-o", "yaml")...) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("generating namespace yaml: %w", err) + } + + applyCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", "-")...) + applyCmd.Stdin = strings.NewReader(string(out)) + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying namespace: %w", err) + } + + // Apply Prometheus manifest + applyCmd = exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", m.manifestPath)...) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying prometheus manifest: %w", err) + } + + // Wait for Prometheus to be ready + fmt.Println("Waiting for Prometheus to be ready...") + waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", + "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for prometheus: %w", err) + } + + return nil +} + +// StartPortForward starts port-forwarding to Prometheus. +func (m *Manager) StartPortForward(ctx context.Context) error { + m.StopPortForward() + + // Start port-forward + m.portForward = exec.CommandContext(ctx, "kubectl", m.kubectl("port-forward", + "-n", "monitoring", "svc/prometheus", fmt.Sprintf("%d:9090", m.localPort))...) + + if err := m.portForward.Start(); err != nil { + return fmt.Errorf("starting port-forward: %w", err) + } + + // Wait for port-forward to be ready + for i := 0; i < 30; i++ { + time.Sleep(time.Second) + if m.isAccessible() { + fmt.Printf("Prometheus accessible at http://localhost:%d\n", m.localPort) + return nil + } + } + + return fmt.Errorf("prometheus port-forward not ready after 30s") +} + +// StopPortForward stops the port-forward process. +func (m *Manager) StopPortForward() { + if m.portForward != nil && m.portForward.Process != nil { + m.portForward.Process.Kill() + m.portForward = nil + } + // Also kill any lingering port-forwards + exec.Command("pkill", "-f", fmt.Sprintf("kubectl port-forward.*prometheus.*%d", m.localPort)).Run() +} + +// Reset restarts Prometheus to clear all metrics. +func (m *Manager) Reset(ctx context.Context) error { + m.StopPortForward() + + // Delete Prometheus pod to reset metrics + cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("delete", "pod", "-n", "monitoring", + "-l", "app=prometheus", "--grace-period=0", "--force")...) + cmd.Run() // Ignore errors + + // Wait for new pod + fmt.Println("Waiting for Prometheus to restart...") + waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", + "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for prometheus restart: %w", err) + } + + // Restart port-forward + if err := m.StartPortForward(ctx); err != nil { + return err + } + + // Wait for scraping to initialize + fmt.Println("Waiting 5s for Prometheus to initialize scraping...") + time.Sleep(5 * time.Second) + + return nil +} + +func (m *Manager) isAccessible() bool { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", m.localPort), 2*time.Second) + if err != nil { + return false + } + conn.Close() + + // Also try HTTP + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/status/config", m.localPort)) + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == 200 +} + +// URL returns the local Prometheus URL. +func (m *Manager) URL() string { + return fmt.Sprintf("http://localhost:%d", m.localPort) +} + +// WaitForTarget waits for a specific job to be scraped by Prometheus. +func (m *Manager) WaitForTarget(ctx context.Context, job string, timeout time.Duration) error { + fmt.Printf("Waiting for Prometheus to discover and scrape job '%s'...\n", job) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if m.isTargetHealthy(job) { + fmt.Printf("Prometheus is scraping job '%s'\n", job) + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + } + + // Print debug info on timeout + m.printTargetStatus(job) + return fmt.Errorf("timeout waiting for Prometheus to scrape job '%s'", job) +} + +// isTargetHealthy checks if a job has at least one healthy target. +func (m *Manager) isTargetHealthy(job string) bool { + resp, err := http.Get(fmt.Sprintf("%s/api/v1/targets", m.URL())) + if err != nil { + return false + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + + var result struct { + Status string `json:"status"` + Data struct { + ActiveTargets []struct { + Labels map[string]string `json:"labels"` + Health string `json:"health"` + } `json:"activeTargets"` + } `json:"data"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return false + } + + for _, target := range result.Data.ActiveTargets { + if target.Labels["job"] == job && target.Health == "up" { + return true + } + } + return false +} + +// printTargetStatus prints debug info about targets. +func (m *Manager) printTargetStatus(job string) { + resp, err := http.Get(fmt.Sprintf("%s/api/v1/targets", m.URL())) + if err != nil { + fmt.Printf("Failed to get targets: %v\n", err) + return + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result struct { + Data struct { + ActiveTargets []struct { + Labels map[string]string `json:"labels"` + Health string `json:"health"` + LastError string `json:"lastError"` + ScrapeURL string `json:"scrapeUrl"` + } `json:"activeTargets"` + } `json:"data"` + } + + if err := json.Unmarshal(body, &result); err != nil { + fmt.Printf("Failed to parse targets: %v\n", err) + return + } + + fmt.Printf("Prometheus targets for job '%s':\n", job) + found := false + for _, target := range result.Data.ActiveTargets { + if target.Labels["job"] == job { + found = true + fmt.Printf(" - %s: health=%s, lastError=%s\n", + target.ScrapeURL, target.Health, target.LastError) + } + } + if !found { + fmt.Printf(" No targets found for job '%s'\n", job) + fmt.Printf(" Available jobs: ") + jobs := make(map[string]bool) + for _, target := range result.Data.ActiveTargets { + jobs[target.Labels["job"]] = true + } + for j := range jobs { + fmt.Printf("%s ", j) + } + fmt.Println() + } +} + +// HasMetrics checks if the specified job has any metrics available. +func (m *Manager) HasMetrics(ctx context.Context, job string) bool { + query := fmt.Sprintf(`up{job="%s"}`, job) + result, err := m.Query(ctx, query) + if err != nil { + return false + } + return len(result.Data.Result) > 0 && result.Data.Result[0].Value[1] == "1" +} + +// QueryResponse represents a Prometheus query response. +type QueryResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// Query executes a PromQL query and returns the response. +func (m *Manager) Query(ctx context.Context, query string) (*QueryResponse, error) { + u := fmt.Sprintf("%s/api/v1/query?query=%s", m.URL(), url.QueryEscape(query)) + + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) + if err != nil { + return nil, err + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("querying prometheus: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + + var result QueryResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + return &result, nil +} + +// CollectMetrics collects all metrics for a scenario and writes to output directory. +func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario string) error { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("creating output directory: %w", err) + } + + timeRange := "10m" + + // For S6 (restart scenario), use increase() to handle counter resets + useIncrease := scenario == "S6" + + // Counter metrics + counterMetrics := []string{ + "reloader_reconcile_total", + "reloader_action_total", + "reloader_skipped_total", + "reloader_errors_total", + "reloader_events_received_total", + "reloader_workloads_scanned_total", + "reloader_workloads_matched_total", + "reloader_reload_executed_total", + } + + for _, metric := range counterMetrics { + var query string + if useIncrease { + query = fmt.Sprintf(`sum(increase(%s{job="%s"}[%s])) by (success, reason)`, metric, job, timeRange) + } else { + query = fmt.Sprintf(`sum(%s{job="%s"}) by (success, reason)`, metric, job) + } + + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, metric+".json")); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", metric, err) + } + } + + // Histogram percentiles + histogramMetrics := []struct { + name string + prefix string + }{ + {"reloader_reconcile_duration_seconds", "reconcile"}, + {"reloader_action_latency_seconds", "action"}, + } + + for _, hm := range histogramMetrics { + for _, pct := range []int{50, 95, 99} { + quantile := float64(pct) / 100 + query := fmt.Sprintf(`histogram_quantile(%v, sum(rate(%s_bucket{job="%s"}[%s])) by (le))`, + quantile, hm.name, job, timeRange) + outFile := filepath.Join(outputDir, fmt.Sprintf("%s_p%d.json", hm.prefix, pct)) + if err := m.queryAndSave(ctx, query, outFile); err != nil { + fmt.Printf("Warning: failed to collect %s p%d: %v\n", hm.name, pct, err) + } + } + } + + // REST client metrics + restQueries := map[string]string{ + "rest_client_requests_total.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s"})`, job), + "rest_client_requests_get.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="GET"})`, job), + "rest_client_requests_patch.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="PATCH"})`, job), + "rest_client_requests_put.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="PUT"})`, job), + "rest_client_requests_errors.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",code=~"[45].."}) or vector(0)`, job), + } + + for filename, query := range restQueries { + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, filename)); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", filename, err) + } + } + + // Resource consumption metrics (memory, CPU, goroutines) + resourceQueries := map[string]string{ + // Memory metrics (in bytes) + "memory_rss_bytes_avg.json": fmt.Sprintf(`avg_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), + "memory_rss_bytes_max.json": fmt.Sprintf(`max_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), + "memory_rss_bytes_cur.json": fmt.Sprintf(`process_resident_memory_bytes{job="%s"}`, job), + + // Heap memory (Go runtime) + "memory_heap_bytes_avg.json": fmt.Sprintf(`avg_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), + "memory_heap_bytes_max.json": fmt.Sprintf(`max_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), + + // CPU metrics (rate of CPU seconds used) + "cpu_usage_cores_avg.json": fmt.Sprintf(`rate(process_cpu_seconds_total{job="%s"}[%s])`, job, timeRange), + "cpu_usage_cores_max.json": fmt.Sprintf(`max_over_time(rate(process_cpu_seconds_total{job="%s"}[1m])[%s:1m])`, job, timeRange), + + // Goroutines (concurrency indicator) + "goroutines_avg.json": fmt.Sprintf(`avg_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), + "goroutines_max.json": fmt.Sprintf(`max_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), + "goroutines_cur.json": fmt.Sprintf(`go_goroutines{job="%s"}`, job), + + // GC metrics + "gc_duration_seconds_p99.json": fmt.Sprintf(`histogram_quantile(0.99, sum(rate(go_gc_duration_seconds_bucket{job="%s"}[%s])) by (le))`, job, timeRange), + + // Threads + "threads_cur.json": fmt.Sprintf(`go_threads{job="%s"}`, job), + } + + for filename, query := range resourceQueries { + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, filename)); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", filename, err) + } + } + + return nil +} + +func (m *Manager) queryAndSave(ctx context.Context, query, outputPath string) error { + result, err := m.Query(ctx, query) + if err != nil { + // Write empty result on error + emptyResult := `{"status":"success","data":{"resultType":"vector","result":[]}}` + return os.WriteFile(outputPath, []byte(emptyResult), 0644) + } + + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + + return os.WriteFile(outputPath, data, 0644) +} diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go new file mode 100644 index 000000000..ed48a3bfa --- /dev/null +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -0,0 +1,2092 @@ +// Package scenarios contains all load test scenario implementations. +package scenarios + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sync" + "time" + + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" +) + +// ExpectedMetrics holds the expected values for metrics verification. +type ExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// Runner defines the interface for test scenarios. +type Runner interface { + Name() string + Description() string + Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) +} + +// Registry holds all available test scenarios. +var Registry = map[string]Runner{ + "S1": &BurstUpdateScenario{}, + "S2": &FanOutScenario{}, + "S3": &HighCardinalityScenario{}, + "S4": &NoOpUpdateScenario{}, + "S5": &WorkloadChurnScenario{}, + "S6": &ControllerRestartScenario{}, + "S7": &APIPressureScenario{}, + "S8": &LargeObjectScenario{}, + "S9": &MultiWorkloadTypeScenario{}, + "S10": &SecretsAndMixedScenario{}, + "S11": &AnnotationStrategyScenario{}, + "S12": &PauseResumeScenario{}, + "S13": &ComplexReferencesScenario{}, +} + +// WriteExpectedMetrics writes expected metrics to a JSON file. +func WriteExpectedMetrics(scenario, resultsDir string, expected ExpectedMetrics) error { + if resultsDir == "" { + return nil + } + + dir := filepath.Join(resultsDir, scenario) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating results directory: %w", err) + } + + data, err := json.MarshalIndent(expected, "", " ") + if err != nil { + return fmt.Errorf("marshaling expected metrics: %w", err) + } + + path := filepath.Join(dir, "expected.json") + if err := os.WriteFile(path, data, 0644); err != nil { + return fmt.Errorf("writing expected metrics: %w", err) + } + + log.Printf("Expected metrics written to %s", path) + return nil +} + +// BurstUpdateScenario - Many ConfigMap/Secret updates in quick succession. +type BurstUpdateScenario struct{} + +func (s *BurstUpdateScenario) Name() string { return "S1" } +func (s *BurstUpdateScenario) Description() string { return "Burst ConfigMap/Secret updates" } + +func (s *BurstUpdateScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S1: Creating base ConfigMaps and Deployments...") + + const numConfigMaps = 10 + const numDeployments = 10 + + setupCtx := context.Background() + + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("burst-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("burst-deploy-%d", i), namespace, fmt.Sprintf("burst-cm-%d", i)) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S1: Starting burst updates...") + + updateCount := 0 + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S1: Context cancelled, completed %d burst updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + WorkloadsMatchedTotal: updateCount, + Description: fmt.Sprintf("S1: %d burst updates, each triggers 1 deployment reload", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("burst-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("value-%d-%d", updateCount, time.Now().UnixNano()) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S1: Completed %d burst updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + WorkloadsMatchedTotal: updateCount, + Description: fmt.Sprintf("S1: %d burst updates, each triggers 1 deployment reload", updateCount), + }, nil +} + +// FanOutScenario - One ConfigMap used by many workloads. +type FanOutScenario struct{} + +func (s *FanOutScenario) Name() string { return "S2" } +func (s *FanOutScenario) Description() string { return "Fan-out (one CM -> many workloads)" } + +func (s *FanOutScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S2: Creating shared ConfigMap and multiple Deployments...") + + const numDeployments = 50 + setupCtx := context.Background() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-cm", + Namespace: namespace, + }, + Data: map[string]string{"config": "initial"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) + } + + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("fanout-deploy-%d", i), namespace, "shared-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment %d: %v", i, err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S2: Updating shared ConfigMap...") + + // Check context state before starting update loop + if ctx.Err() != nil { + log.Printf("S2: WARNING - Context already done before update loop: %v", ctx.Err()) + } + if deadline, ok := ctx.Deadline(); ok { + remaining := time.Until(deadline) + log.Printf("S2: Context deadline in %v", remaining) + if remaining < 10*time.Second { + log.Printf("S2: WARNING - Very little time remaining on context!") + } + } else { + log.Println("S2: Context has no deadline") + } + + updateCount := 0 + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + log.Printf("S2: Will run updates for %v (duration=%v)", duration-5*time.Second, duration) + + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + expectedActions := updateCount * numDeployments + log.Printf("S2: Context done (err=%v), completed %d fan-out updates", ctx.Err(), updateCount) + return ExpectedMetrics{ + ActionTotal: expectedActions, + ReloadExecutedTotal: expectedActions, + WorkloadsScannedTotal: expectedActions, + WorkloadsMatchedTotal: expectedActions, + Description: fmt.Sprintf("S2: %d updates × %d deployments = %d expected reloads", updateCount, numDeployments, expectedActions), + }, nil + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "shared-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update shared ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S2: Updated shared ConfigMap (should trigger %d reloads)", numDeployments) + } + } + } + + expectedActions := updateCount * numDeployments + log.Printf("S2: Completed %d fan-out updates, expected %d total actions", updateCount, expectedActions) + return ExpectedMetrics{ + ActionTotal: expectedActions, + ReloadExecutedTotal: expectedActions, + WorkloadsScannedTotal: expectedActions, + WorkloadsMatchedTotal: expectedActions, + Description: fmt.Sprintf("S2: %d updates × %d deployments = %d expected reloads", updateCount, numDeployments, expectedActions), + }, nil +} + +// HighCardinalityScenario - Many ConfigMaps/Secrets across many namespaces. +type HighCardinalityScenario struct{} + +func (s *HighCardinalityScenario) Name() string { return "S3" } +func (s *HighCardinalityScenario) Description() string { + return "High cardinality (many CMs, many namespaces)" +} + +func (s *HighCardinalityScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S3: Creating high cardinality resources...") + + setupCtx := context.Background() + + namespaces := []string{namespace} + for i := 0; i < 10; i++ { + ns := fmt.Sprintf("%s-%d", namespace, i) + if _, err := client.CoreV1().Namespaces().Create(setupCtx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create namespace %s: %v", ns, err) + } else { + namespaces = append(namespaces, ns) + } + } + + for _, ns := range namespaces { + for i := 0; i < 20; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("hc-cm-%d", i), + Namespace: ns, + }, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(ns).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("hc-deploy-%d", i), ns, fmt.Sprintf("hc-cm-%d", i)) + client.AppsV1().Deployments(ns).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + } + + if err := waitForAllNamespacesReady(setupCtx, client, namespaces, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S3: Starting random updates across namespaces...") + + updateDuration := duration - 5*time.Second + if updateDuration < 30*time.Second { + updateDuration = 30 * time.Second + } + + updateCount := 0 + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + updateCtx, updateCancel := context.WithTimeout(context.Background(), updateDuration) + defer updateCancel() + + endTime := time.Now().Add(updateDuration) + log.Printf("S3: Will run updates for %v (until %v)", updateDuration, endTime.Format("15:04:05")) + + for time.Now().Before(endTime) { + select { + case <-updateCtx.Done(): + log.Printf("S3: Completed %d high cardinality updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S3: %d updates across %d namespaces", updateCount, len(namespaces)), + }, nil + case <-ticker.C: + ns := namespaces[rand.Intn(len(namespaces))] + cmIndex := rand.Intn(20) + cm, err := client.CoreV1().ConfigMaps(ns).Get(setupCtx, fmt.Sprintf("hc-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(ns).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + + log.Printf("S3: Completed %d high cardinality updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S3: %d updates across %d namespaces", updateCount, len(namespaces)), + }, nil +} + +// NoOpUpdateScenario - Updates that don't actually change data. +type NoOpUpdateScenario struct{} + +func (s *NoOpUpdateScenario) Name() string { return "S4" } +func (s *NoOpUpdateScenario) Description() string { return "No-op updates (same data)" } + +func (s *NoOpUpdateScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S4: Creating ConfigMaps and Deployments for no-op test...") + + setupCtx := context.Background() + + for i := 0; i < 10; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("noop-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "static-value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("noop-deploy-%d", i), namespace, fmt.Sprintf("noop-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S4: Starting no-op updates (annotation changes only)...") + + updateCount := 0 + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S4: Completed %d no-op updates", updateCount) + return ExpectedMetrics{ + ActionTotal: 0, + ReloadExecutedTotal: 0, + SkippedTotal: updateCount, + Description: fmt.Sprintf("S4: %d no-op updates, all should be skipped", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(10) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("noop-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + if cm.Annotations == nil { + cm.Annotations = make(map[string]string) + } + cm.Annotations["noop-counter"] = fmt.Sprintf("%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + + log.Printf("S4: Completed %d no-op updates (should see 0 actions)", updateCount) + return ExpectedMetrics{ + ActionTotal: 0, + ReloadExecutedTotal: 0, + SkippedTotal: updateCount, + Description: fmt.Sprintf("S4: %d no-op updates, all should be skipped", updateCount), + }, nil +} + +// WorkloadChurnScenario - Deployments created and deleted rapidly. +type WorkloadChurnScenario struct{} + +func (s *WorkloadChurnScenario) Name() string { return "S5" } +func (s *WorkloadChurnScenario) Description() string { return "Workload churn (rapid create/delete)" } + +func (s *WorkloadChurnScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S5: Creating base ConfigMap...") + + setupCtx := context.Background() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "churn-cm", Namespace: namespace}, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + + log.Println("S5: Starting workload churn...") + + var wg sync.WaitGroup + var mu sync.Mutex + deployCounter := 0 + deleteCounter := 0 + cmUpdateCount := 0 + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + deployName := fmt.Sprintf("churn-deploy-%d", deployCounter) + deploy := createDeployment(deployName, namespace, "churn-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err == nil { + mu.Lock() + deployCounter++ + mu.Unlock() + } + if deployCounter > 10 { + oldName := fmt.Sprintf("churn-deploy-%d", deployCounter-10) + if err := client.AppsV1().Deployments(namespace).Delete(setupCtx, oldName, metav1.DeleteOptions{}); err == nil { + mu.Lock() + deleteCounter++ + mu.Unlock() + } + } + } + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "churn-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", cmUpdateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + mu.Lock() + cmUpdateCount++ + mu.Unlock() + } + } + } + }() + + wg.Wait() + log.Printf("S5: Created %d, deleted %d deployments, %d CM updates", deployCounter, deleteCounter, cmUpdateCount) + + // S5 does NOT set expected values for action_total/reload_executed_total because: + // - There are ~10 active deployments at any time (creates new, deletes old) + // - Each CM update triggers reloads on ALL active deployments + // - Exact counts depend on timing of creates/deletes vs CM updates + // - "Not found" errors are expected when a deployment is deleted during processing + // Instead, S5 pass/fail compares old vs new (both should be similar) + return ExpectedMetrics{ + // No expected values - churn makes exact counts unpredictable + Description: fmt.Sprintf("S5: Churn test - %d deploys created, %d deleted, %d CM updates, ~10 active deploys at any time", deployCounter, deleteCounter, cmUpdateCount), + }, nil +} + +// ControllerRestartScenario - Restart controller under load. +type ControllerRestartScenario struct { + ReloaderVersion string +} + +func (s *ControllerRestartScenario) Name() string { return "S6" } +func (s *ControllerRestartScenario) Description() string { + return "Controller restart under load" +} + +func (s *ControllerRestartScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S6: Creating resources and generating load...") + + setupCtx := context.Background() + + for i := 0; i < 20; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("restart-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("restart-deploy-%d", i), namespace, fmt.Sprintf("restart-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + var wg sync.WaitGroup + var mu sync.Mutex + updateCount := 0 + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cmIndex := rand.Intn(20) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("restart-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + mu.Lock() + updateCount++ + mu.Unlock() + } + } + } + }() + + reloaderNS := fmt.Sprintf("reloader-%s", s.ReloaderVersion) + if s.ReloaderVersion == "" { + reloaderNS = "reloader-new" + } + + log.Println("S6: Waiting 20 seconds before restarting controller...") + time.Sleep(20 * time.Second) + + log.Println("S6: Restarting Reloader pod...") + pods, err := client.CoreV1().Pods(reloaderNS).List(setupCtx, metav1.ListOptions{ + LabelSelector: "app=reloader", + }) + if err == nil && len(pods.Items) > 0 { + client.CoreV1().Pods(reloaderNS).Delete(setupCtx, pods.Items[0].Name, metav1.DeleteOptions{}) + } + + wg.Wait() + log.Printf("S6: Controller restart scenario completed with %d updates", updateCount) + return ExpectedMetrics{ + Description: fmt.Sprintf("S6: Restart test - %d updates during restart", updateCount), + }, nil +} + +// APIPressureScenario - Simulate API server pressure with many concurrent requests. +type APIPressureScenario struct{} + +func (s *APIPressureScenario) Name() string { return "S7" } +func (s *APIPressureScenario) Description() string { return "API pressure (many concurrent requests)" } + +func (s *APIPressureScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S7: Creating resources for API pressure test...") + + const numConfigMaps = 50 + setupCtx := context.Background() + + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("api-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("api-deploy-%d", i), namespace, fmt.Sprintf("api-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S7: Starting concurrent updates from multiple goroutines...") + + updateDuration := duration - 5*time.Second + if updateDuration < 30*time.Second { + updateDuration = 30 * time.Second + } + + updateCtx, updateCancel := context.WithTimeout(context.Background(), updateDuration) + defer updateCancel() + + endTime := time.Now().Add(updateDuration) + log.Printf("S7: Will run updates for %v (until %v)", updateDuration, endTime.Format("15:04:05")) + + var wg sync.WaitGroup + var mu sync.Mutex + totalUpdates := 0 + + for g := 0; g < 10; g++ { + wg.Add(1) + go func(goroutineID int) { + defer wg.Done() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + updateCount := 0 + for time.Now().Before(endTime) { + select { + case <-updateCtx.Done(): + return + case <-ticker.C: + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("api-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("g%d-update-%d", goroutineID, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + mu.Lock() + totalUpdates += updateCount + mu.Unlock() + log.Printf("S7: Goroutine %d completed %d updates", goroutineID, updateCount) + }(g) + } + + wg.Wait() + log.Printf("S7: API pressure scenario completed with %d total updates", totalUpdates) + return ExpectedMetrics{ + ActionTotal: totalUpdates, + ReloadExecutedTotal: totalUpdates, + Description: fmt.Sprintf("S7: %d concurrent updates from 10 goroutines", totalUpdates), + }, nil +} + +// LargeObjectScenario - Large ConfigMaps/Secrets. +type LargeObjectScenario struct{} + +func (s *LargeObjectScenario) Name() string { return "S8" } +func (s *LargeObjectScenario) Description() string { return "Large ConfigMaps/Secrets (>100KB)" } + +func (s *LargeObjectScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S8: Creating large ConfigMaps...") + + setupCtx := context.Background() + + largeData := make([]byte, 100*1024) + for i := range largeData { + largeData[i] = byte('a' + (i % 26)) + } + largeValue := string(largeData) + + for i := 0; i < 10; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("large-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "large-key-1": largeValue, + "large-key-2": largeValue, + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create large ConfigMap %d: %v", i, err) + } + deploy := createDeployment(fmt.Sprintf("large-deploy-%d", i), namespace, fmt.Sprintf("large-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S8: Starting large object updates...") + + updateCount := 0 + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S8: Completed %d large object updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S8: %d large object (100KB) updates", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(10) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("large-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["large-key-1"] = largeValue[:len(largeValue)-10] + fmt.Sprintf("-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update large ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S8: Completed %d large object updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S8: %d large object (100KB) updates", updateCount), + }, nil +} + +// Helper functions + +func waitForDeploymentsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all deployments in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + deployments, err := client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list deployments: %w", err) + } + + allReady := true + notReady := 0 + for _, d := range deployments.Items { + if d.Status.ReadyReplicas < *d.Spec.Replicas { + allReady = false + notReady++ + } + } + + if allReady && len(deployments.Items) > 0 { + log.Printf("All %d deployments in %s are ready", len(deployments.Items), namespace) + return nil + } + + log.Printf("Waiting for deployments: %d/%d not ready yet...", notReady, len(deployments.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for deployments to be ready") +} + +func waitForAllNamespacesReady(ctx context.Context, client kubernetes.Interface, namespaces []string, timeout time.Duration) error { + log.Printf("Waiting for deployments in %d namespaces to be ready...", len(namespaces)) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + allReady := true + totalDeploys := 0 + notReady := 0 + + for _, ns := range namespaces { + deployments, err := client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + continue + } + for _, d := range deployments.Items { + totalDeploys++ + if d.Status.ReadyReplicas < *d.Spec.Replicas { + allReady = false + notReady++ + } + } + } + + if allReady && totalDeploys > 0 { + log.Printf("All %d deployments across %d namespaces are ready", totalDeploys, len(namespaces)) + return nil + } + + log.Printf("Waiting: %d/%d deployments not ready yet...", notReady, totalDeploys) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for deployments to be ready") +} + +func createDeployment(name, namespace, configMapName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDeploymentWithSecret(name, namespace, secretName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDeploymentWithBoth(name, namespace, configMapName, secretName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +// SecretsAndMixedScenario - Tests Secrets and mixed ConfigMap+Secret workloads. +type SecretsAndMixedScenario struct{} + +func (s *SecretsAndMixedScenario) Name() string { return "S10" } +func (s *SecretsAndMixedScenario) Description() string { + return "Secrets and mixed ConfigMap+Secret workloads" +} + +func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S10: Creating Secrets, ConfigMaps, and mixed workloads...") + + const numSecrets = 5 + const numConfigMaps = 5 + const numSecretOnlyDeploys = 5 + const numConfigMapOnlyDeploys = 3 + const numMixedDeploys = 2 + + setupCtx := context.Background() + + // Create Secrets + for i := 0; i < numSecrets; i++ { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("mixed-secret-%d", i), + Namespace: namespace, + }, + StringData: map[string]string{ + "password": fmt.Sprintf("initial-secret-%d", i), + }, + } + if _, err := client.CoreV1().Secrets(namespace).Create(setupCtx, secret, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Secret %s: %v", secret.Name, err) + } + } + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("mixed-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "config": fmt.Sprintf("initial-config-%d", i), + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Secret-only deployments + for i := 0; i < numSecretOnlyDeploys; i++ { + deploy := createDeploymentWithSecret( + fmt.Sprintf("secret-only-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-secret-%d", i%numSecrets), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Secret-only Deployment: %v", err) + } + } + + // Create ConfigMap-only deployments + for i := 0; i < numConfigMapOnlyDeploys; i++ { + deploy := createDeployment( + fmt.Sprintf("cm-only-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-cm-%d", i%numConfigMaps), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap-only Deployment: %v", err) + } + } + + // Create mixed deployments (using both Secret and ConfigMap) + for i := 0; i < numMixedDeploys; i++ { + deploy := createDeploymentWithBoth( + fmt.Sprintf("mixed-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-cm-%d", i%numConfigMaps), + fmt.Sprintf("mixed-secret-%d", i%numSecrets), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create mixed Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S10: Starting alternating Secret and ConfigMap updates...") + + secretUpdateCount := 0 + cmUpdateCount := 0 + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + updateSecret := true // Alternate between Secret and ConfigMap updates + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + case <-ticker.C: + if updateSecret { + // Update a random Secret + secretIndex := rand.Intn(numSecrets) + secret, err := client.CoreV1().Secrets(namespace).Get(setupCtx, fmt.Sprintf("mixed-secret-%d", secretIndex), metav1.GetOptions{}) + if err != nil { + continue + } + secret.StringData = map[string]string{ + "password": fmt.Sprintf("updated-secret-%d-%d", secretIndex, secretUpdateCount), + } + if _, err := client.CoreV1().Secrets(namespace).Update(setupCtx, secret, metav1.UpdateOptions{}); err == nil { + secretUpdateCount++ + } + } else { + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("mixed-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("updated-config-%d-%d", cmIndex, cmUpdateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + cmUpdateCount++ + } + } + updateSecret = !updateSecret + } + } + + log.Printf("S10: Completed %d Secret updates and %d ConfigMap updates", secretUpdateCount, cmUpdateCount) + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil +} + +func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { + // Secret updates trigger: secret-only deploys + mixed deploys + secretTriggeredReloads := secretUpdates * (secretOnlyDeploys + mixedDeploys) + // ConfigMap updates trigger: cm-only deploys + mixed deploys + cmTriggeredReloads := cmUpdates * (cmOnlyDeploys + mixedDeploys) + totalExpectedReloads := secretTriggeredReloads + cmTriggeredReloads + + return ExpectedMetrics{ + ActionTotal: totalExpectedReloads, + ReloadExecutedTotal: totalExpectedReloads, + Description: fmt.Sprintf("S10: %d Secret updates (→%d reloads) + %d CM updates (→%d reloads) = %d total", + secretUpdates, secretTriggeredReloads, cmUpdates, cmTriggeredReloads, totalExpectedReloads), + } +} + +// MultiWorkloadTypeScenario - Tests all supported workload types with a shared ConfigMap. +type MultiWorkloadTypeScenario struct{} + +func (s *MultiWorkloadTypeScenario) Name() string { return "S9" } +func (s *MultiWorkloadTypeScenario) Description() string { + return "Multi-workload types (Deploy, StatefulSet, DaemonSet, Job, CronJob)" +} + +func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S9: Creating shared ConfigMap and multiple workload types...") + + const numDeployments = 5 + const numStatefulSets = 3 + const numDaemonSets = 2 + + setupCtx := context.Background() + + // Create shared ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multi-type-cm", + Namespace: namespace, + }, + Data: map[string]string{"config": "initial"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) + } + + // Create Deployments + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("multi-deploy-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment %d: %v", i, err) + } + } + + // Create StatefulSets + for i := 0; i < numStatefulSets; i++ { + sts := createStatefulSet(fmt.Sprintf("multi-sts-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().StatefulSets(namespace).Create(setupCtx, sts, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create StatefulSet %d: %v", i, err) + } + } + + // Create DaemonSets + for i := 0; i < numDaemonSets; i++ { + ds := createDaemonSet(fmt.Sprintf("multi-ds-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().DaemonSets(namespace).Create(setupCtx, ds, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create DaemonSet %d: %v", i, err) + } + } + + // Wait for workloads to be ready + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + if err := waitForStatefulSetsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + if err := waitForDaemonSetsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S9: Starting ConfigMap updates to trigger reloads on all workload types...") + + updateCount := 0 + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets), nil + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "multi-type-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update shared ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S9: Updated shared ConfigMap (update #%d)", updateCount) + } + } + } + + log.Printf("S9: Completed %d ConfigMap updates", updateCount) + return s.calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets), nil +} + +func (s *MultiWorkloadTypeScenario) calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets int) ExpectedMetrics { + // Each CM update triggers reload on all workloads + totalWorkloads := numDeployments + numStatefulSets + numDaemonSets + expectedReloads := updateCount * totalWorkloads + + return ExpectedMetrics{ + ActionTotal: expectedReloads, + ReloadExecutedTotal: expectedReloads, + WorkloadsMatchedTotal: expectedReloads, + Description: fmt.Sprintf("S9: %d CM updates × %d workloads (%d Deploys + %d STS + %d DS) = %d reloads", + updateCount, totalWorkloads, numDeployments, numStatefulSets, numDaemonSets, expectedReloads), + } +} + +func createStatefulSet(name, namespace, configMapName string) *appsv1.StatefulSet { + replicas := int32(1) + terminationGracePeriod := int64(0) + + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: name, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDaemonSet(name, namespace, configMapName string) *appsv1.DaemonSet { + terminationGracePeriod := int64(0) + + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + // Use tolerations to run on all nodes including control-plane + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func waitForStatefulSetsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all StatefulSets in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + stsList, err := client.AppsV1().StatefulSets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list StatefulSets: %w", err) + } + + if len(stsList.Items) == 0 { + log.Printf("No StatefulSets found in %s", namespace) + return nil + } + + allReady := true + notReady := 0 + for _, sts := range stsList.Items { + if sts.Status.ReadyReplicas < *sts.Spec.Replicas { + allReady = false + notReady++ + } + } + + if allReady { + log.Printf("All %d StatefulSets in %s are ready", len(stsList.Items), namespace) + return nil + } + + log.Printf("Waiting for StatefulSets: %d/%d not ready yet...", notReady, len(stsList.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for StatefulSets to be ready") +} + +func waitForDaemonSetsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all DaemonSets in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + dsList, err := client.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list DaemonSets: %w", err) + } + + if len(dsList.Items) == 0 { + log.Printf("No DaemonSets found in %s", namespace) + return nil + } + + allReady := true + notReady := 0 + for _, ds := range dsList.Items { + if ds.Status.NumberReady < ds.Status.DesiredNumberScheduled { + allReady = false + notReady++ + } + } + + if allReady { + log.Printf("All %d DaemonSets in %s are ready", len(dsList.Items), namespace) + return nil + } + + log.Printf("Waiting for DaemonSets: %d/%d not ready yet...", notReady, len(dsList.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for DaemonSets to be ready") +} + +// ComplexReferencesScenario - Tests init containers, valueFrom, and projected volumes. +type ComplexReferencesScenario struct{} + +func (s *ComplexReferencesScenario) Name() string { return "S13" } +func (s *ComplexReferencesScenario) Description() string { + return "Complex references (init containers, valueFrom, projected volumes)" +} + +func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S13: Creating ConfigMaps and complex deployments with various reference types...") + + const numConfigMaps = 5 + const numDeployments = 5 + + setupCtx := context.Background() + + // Create ConfigMaps with multiple keys + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("complex-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "key1": fmt.Sprintf("value1-%d", i), + "key2": fmt.Sprintf("value2-%d", i), + "config": fmt.Sprintf("config-%d", i), + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create complex deployments with various reference types + for i := 0; i < numDeployments; i++ { + // Each deployment references multiple ConfigMaps in different ways + primaryCM := fmt.Sprintf("complex-cm-%d", i) + secondaryCM := fmt.Sprintf("complex-cm-%d", (i+1)%numConfigMaps) + + deploy := createComplexDeployment( + fmt.Sprintf("complex-deploy-%d", i), + namespace, + primaryCM, + secondaryCM, + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create complex Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S13: Starting ConfigMap updates to test all reference types...") + + updateCount := 0 + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("complex-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key1"] = fmt.Sprintf("updated-value1-%d-%d", cmIndex, updateCount) + cm.Data["config"] = fmt.Sprintf("updated-config-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S13: Updated complex-cm-%d (update #%d)", cmIndex, updateCount) + } + } + } + + log.Printf("S13: Completed %d ConfigMap updates", updateCount) + return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil +} + +func (s *ComplexReferencesScenario) calculateExpected(updateCount, numConfigMaps, numDeployments int) ExpectedMetrics { + // Each ConfigMap is referenced by: + // - 1 deployment as primary (envFrom in init + valueFrom in main + volume mount) + // - 1 deployment as secondary (projected volume) + // So each CM update triggers 2 deployments (on average with random updates) + // But since we're randomly updating, each update affects those 2 deployments + expectedReloadsPerUpdate := 2 + expectedReloads := updateCount * expectedReloadsPerUpdate + + return ExpectedMetrics{ + ActionTotal: expectedReloads, + ReloadExecutedTotal: expectedReloads, + Description: fmt.Sprintf("S13: %d CM updates × ~%d affected deploys = ~%d reloads (init containers, valueFrom, volumes, projected)", + updateCount, expectedReloadsPerUpdate, expectedReloads), + } +} + +// PauseResumeScenario - Tests pause-period functionality under rapid updates. +type PauseResumeScenario struct{} + +func (s *PauseResumeScenario) Name() string { return "S12" } +func (s *PauseResumeScenario) Description() string { + return "Pause & Resume (rapid updates with pause-period)" +} + +func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S12: Creating ConfigMaps and Deployments with pause-period annotation...") + + const numConfigMaps = 10 + const numDeployments = 10 + const pausePeriod = 15 * time.Second + const updateInterval = 2 * time.Second + + setupCtx := context.Background() + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("pause-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Deployments with pause-period annotation + for i := 0; i < numDeployments; i++ { + deploy := createDeploymentWithPause( + fmt.Sprintf("pause-deploy-%d", i), + namespace, + fmt.Sprintf("pause-cm-%d", i), + pausePeriod, + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Printf("S12: Starting rapid ConfigMap updates (every %v) with %v pause-period...", updateInterval, pausePeriod) + + updateCount := 0 + ticker := time.NewTicker(updateInterval) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("pause-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S12: Completed %d rapid updates (pause-period should reduce actual reloads)", updateCount) + return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil +} + +func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updateInterval, pausePeriod time.Duration) ExpectedMetrics { + // With pause-period, we expect fewer reloads than updates + // Each deployment gets updates at random, and pause-period prevents rapid consecutive reloads + // The exact count depends on the distribution of updates across ConfigMaps + // Rough estimate: each CM gets updated ~(updateCount/10) times + // With 15s pause and 2s interval, we get roughly 1 reload per pause period per CM + // So expected reloads ≈ duration / pausePeriod per deployment = (duration/pausePeriod) * numDeployments + + // This is an approximation - the actual value depends on random distribution + expectedCycles := int(duration / pausePeriod) + if expectedCycles < 1 { + expectedCycles = 1 + } + + return ExpectedMetrics{ + // Don't set exact expected values since pause-period makes counts unpredictable + // The scenario validates that reloads << updates due to pause behavior + Description: fmt.Sprintf("S12: %d updates with %v pause-period (expect ~%d reload cycles, actual reloads << updates)", + updateCount, pausePeriod, expectedCycles), + } +} + +// AnnotationStrategyScenario - Tests annotation-based reload strategy. +// This scenario deploys its own Reloader instance with --reload-strategy=annotations. +type AnnotationStrategyScenario struct { + // Image is the Reloader image to use. Must be set before running. + Image string +} + +func (s *AnnotationStrategyScenario) Name() string { return "S11" } +func (s *AnnotationStrategyScenario) Description() string { + return "Annotation reload strategy (--reload-strategy=annotations)" +} + +func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + if s.Image == "" { + return ExpectedMetrics{}, fmt.Errorf("S11 requires Image to be set (use the same image as --new-image)") + } + + log.Println("S11: Deploying Reloader with --reload-strategy=annotations...") + + // Deploy S11's own Reloader instance + reloaderNS := "reloader-s11" + mgr := reloader.NewManager(reloader.Config{ + Version: "s11", + Image: s.Image, + Namespace: reloaderNS, + ReloadStrategy: "annotations", + }) + + if err := mgr.Deploy(ctx); err != nil { + return ExpectedMetrics{}, fmt.Errorf("deploying S11 reloader: %w", err) + } + + // Ensure cleanup on exit + defer func() { + log.Println("S11: Cleaning up S11-specific Reloader...") + cleanupCtx := context.Background() + if err := mgr.Cleanup(cleanupCtx); err != nil { + log.Printf("Warning: failed to cleanup S11 reloader: %v", err) + } + }() + + log.Println("S11: Creating ConfigMaps and Deployments...") + + const numConfigMaps = 10 + const numDeployments = 10 + + setupCtx := context.Background() + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("annot-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Deployments + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("annot-deploy-%d", i), namespace, fmt.Sprintf("annot-cm-%d", i)) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S11: Starting ConfigMap updates with annotation strategy...") + + updateCount := 0 + annotationUpdatesSeen := 0 + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 10*time.Second) // Extra time for cleanup + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, annotationUpdatesSeen), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("annot-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + + // Periodically check for annotation updates on deployments + if updateCount%10 == 0 { + deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", cmIndex), metav1.GetOptions{}) + if err == nil { + if _, hasAnnotation := deploy.Spec.Template.Annotations["reloader.stakater.com/last-reloaded-from"]; hasAnnotation { + annotationUpdatesSeen++ + } + } + } + } + } + + // Final check: verify annotation strategy is working + log.Println("S11: Verifying annotation-based reload...") + time.Sleep(5 * time.Second) // Allow time for final updates to propagate + + deploysWithAnnotation := 0 + for i := 0; i < numDeployments; i++ { + deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", i), metav1.GetOptions{}) + if err != nil { + continue + } + if deploy.Spec.Template.Annotations != nil { + if _, ok := deploy.Spec.Template.Annotations["reloader.stakater.com/last-reloaded-from"]; ok { + deploysWithAnnotation++ + } + } + } + + log.Printf("S11: Completed %d updates, %d deployments have reload annotation", updateCount, deploysWithAnnotation) + return s.calculateExpected(updateCount, deploysWithAnnotation), nil +} + +func (s *AnnotationStrategyScenario) calculateExpected(updateCount, deploysWithAnnotation int) ExpectedMetrics { + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S11: %d updates with annotation strategy, %d deployments received annotation", + updateCount, deploysWithAnnotation), + } +} + +func createDeploymentWithPause(name, namespace, configMapName string, pausePeriod time.Duration) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + // Deployment-specific pause-period annotation + "deployment.reloader.stakater.com/pause-period": fmt.Sprintf("%ds", int(pausePeriod.Seconds())), + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +// createComplexDeployment creates a deployment with multiple ConfigMap reference types. +// - Init container using envFrom +// - Main container using env.valueFrom.configMapKeyRef +// - Sidecar container using volume mount +// - Projected volume combining multiple ConfigMaps +func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + // Init container using envFrom + InitContainers: []corev1.Container{ + { + Name: "init", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "echo Init done"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + // Main container using valueFrom (individual keys) + { + Name: "main", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + Env: []corev1.EnvVar{ + { + Name: "CONFIG_KEY1", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Key: "key1", + }, + }, + }, + { + Name: "CONFIG_KEY2", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Key: "key2", + }, + }, + }, + }, + }, + // Sidecar using volume mount + { + Name: "sidecar", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/config", + }, + { + Name: "projected-volume", + MountPath: "/etc/projected", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + // Regular ConfigMap volume + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + }, + }, + }, + // Projected volume combining multiple ConfigMaps + { + Name: "projected-volume", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Items: []corev1.KeyToPath{ + { + Key: "key1", + Path: "primary-key1", + }, + }, + }, + }, + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secondaryCM, + }, + Items: []corev1.KeyToPath{ + { + Key: "key1", + Path: "secondary-key1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/test/loadtest/manifests/prometheus.yaml b/test/loadtest/manifests/prometheus.yaml new file mode 100644 index 000000000..f826f52f4 --- /dev/null +++ b/test/loadtest/manifests/prometheus.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring +data: + prometheus.yml: | + global: + scrape_interval: 2s + evaluation_interval: 2s + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'reloader-old' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - reloader-old + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'reloader-new' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - reloader-new + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: quay.io/prometheus/prometheus:v2.47.0 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.console.libraries=/usr/share/prometheus/console_libraries + - --web.console.templates=/usr/share/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 200m + memory: 512Mi + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: prometheus-config + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + selector: + app: prometheus + ports: + - port: 9090 + targetPort: 9090 + type: NodePort From 512278d740f0cadb9a4133b2d08a4d0da2bfd919 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:06:44 +0100 Subject: [PATCH 02/21] ci: Allow manual trigger --- .github/workflows/loadtest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a62e3e32e..a822fe7bd 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -3,6 +3,7 @@ name: Load Test on: issue_comment: types: [created] + workflow_dispatch: jobs: loadtest: From 5b63610f4f28fe2f5600260377e9d63f077eaf2f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:47:23 +0100 Subject: [PATCH 03/21] ci: Remove manual_dispatch from loadtests --- .github/workflows/loadtest.yml | 63 ++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a822fe7bd..db01e7ed2 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -3,7 +3,11 @@ name: Load Test on: issue_comment: types: [created] - workflow_dispatch: + +permissions: + contents: read + pull-requests: write + issues: write jobs: loadtest: @@ -56,6 +60,12 @@ jobs: chmod +x ./kind sudo mv ./kind /usr/local/bin/kind + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl + # Build OLD image from base branch (e.g., main) - name: Checkout base branch (old) uses: actions/checkout@v4 @@ -165,31 +175,32 @@ jobs: const overallStatus = failCount === 0 ? '✅ ALL PASSED' : `❌ ${failCount} FAILED`; - const body = `## Load Test Results ${overallStatus} - - **Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new) - **Old commit:** ${{ steps.pr.outputs.base_sha }} - **New commit:** ${{ steps.pr.outputs.head_sha }} - **Triggered by:** @${{ github.event.comment.user.login }} - - ### Summary - - | Scenario | Status | - |----------|--------| - ${summaries.join('\n')} - - **Total:** ${passCount} passed, ${failCount} failed - - ### Detailed Results - - ${results} - -
- 📦 Download full results - - Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}). -
- `; + const body = [ + `## Load Test Results ${overallStatus}`, + '', + `**Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new)`, + `**Old commit:** ${{ steps.pr.outputs.base_sha }}`, + `**New commit:** ${{ steps.pr.outputs.head_sha }}`, + `**Triggered by:** @${{ github.event.comment.user.login }}`, + '', + '### Summary', + '', + '| Scenario | Status |', + '|----------|--------|', + summaries.join('\n'), + '', + `**Total:** ${passCount} passed, ${failCount} failed`, + '', + '### Detailed Results', + '', + results, + '', + '
', + '📦 Download full results', + '', + `Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).`, + '
', + ].join('\n'); await github.rest.issues.createComment({ owner: context.repo.owner, From 27f49ecc0fea8e0cd4a976f1ac22236f2304a429 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:07:38 +0100 Subject: [PATCH 04/21] fix: Use go 1.25 for load tests in CI --- .github/workflows/loadtest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index db01e7ed2..a67744445 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -48,7 +48,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version: '1.25' cache: false - name: Set up Docker Buildx From eb3bc2447e2fefc9add1ef2b7113af88db7ca01b Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:12:10 +0100 Subject: [PATCH 05/21] refactor(upgrade.go): simplify retryOnConflict to return matched status and error --- internal/pkg/handler/upgrade.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index e355d5fd0..932fade20 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -242,16 +242,15 @@ func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs call matchedCount := 0 for _, item := range items { - err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { - matched, err := upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) - if matched { - matchedCount++ - } - return matched, err + matched, err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { + return upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) }) if err != nil { return err } + if matched { + matchedCount++ + } } // Record workloads matched @@ -260,11 +259,13 @@ func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs call return nil } -func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error { +func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) (bool, error) { var lastError error + var matched bool fetchResource := false // do not fetch resource on first attempt, already done by ItemsFunc err := wait.ExponentialBackoff(backoff, func() (bool, error) { - _, err := fn(fetchResource) + var err error + matched, err = fn(fetchResource) fetchResource = true switch { case err == nil: @@ -279,7 +280,7 @@ func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error if wait.Interrupted(err) { err = lastError } - return err + return matched, err } func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) (bool, error) { From f7210204d40e43b08f37a237997a7063da4ae023 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:41:44 +0100 Subject: [PATCH 06/21] fix: Safe parsing for duration --- test/loadtest/cmd/loadtest/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index 19f7b1dba..572509007 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -142,9 +142,13 @@ func parseArgs(args []string) Config { case strings.HasPrefix(arg, "--scenario="): cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") case strings.HasPrefix(arg, "--duration="): - fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration) + if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration); n != 1 { + log.Printf("Warning: invalid --duration value, using default (%d)", cfg.Duration) + } case strings.HasPrefix(arg, "--parallelism="): - fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism) + if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism); n != 1 { + log.Printf("Warning: invalid --parallelism value, using default (%d)", cfg.Parallelism) + } case arg == "--skip-cluster": cfg.SkipCluster = true case strings.HasPrefix(arg, "--results-dir="): From 193f64c0ec3fc9c6225a2004b8f01fc2b68e3d58 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 14:22:43 +0100 Subject: [PATCH 07/21] fix: Missing reloader.go --- .gitignore | 1 + test/loadtest/internal/reloader/reloader.go | 250 ++++++++++++++++++ test/loadtest/internal/scenarios/scenarios.go | 22 +- 3 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 test/loadtest/internal/reloader/reloader.go diff --git a/.gitignore b/.gitignore index 73da63e55..a398472b1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ vendor dist Reloader !**/chart/reloader +!**/internal/reloader *.tgz styles/ site/ diff --git a/test/loadtest/internal/reloader/reloader.go b/test/loadtest/internal/reloader/reloader.go new file mode 100644 index 000000000..ff3cfdb3f --- /dev/null +++ b/test/loadtest/internal/reloader/reloader.go @@ -0,0 +1,250 @@ +package reloader + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Config holds configuration for a Reloader deployment. +type Config struct { + Version string + Image string + Namespace string + ReloadStrategy string +} + +// Manager handles Reloader deployment operations. +type Manager struct { + config Config + kubeContext string +} + +// NewManager creates a new Reloader manager. +func NewManager(config Config) *Manager { + return &Manager{ + config: config, + } +} + +// SetKubeContext sets the kubeconfig context to use. +func (m *Manager) SetKubeContext(kubeContext string) { + m.kubeContext = kubeContext +} + +// kubectl returns kubectl command with optional context. +func (m *Manager) kubectl(ctx context.Context, args ...string) *exec.Cmd { + if m.kubeContext != "" { + args = append([]string{"--context", m.kubeContext}, args...) + } + return exec.CommandContext(ctx, "kubectl", args...) +} + +// namespace returns the namespace for this reloader instance. +func (m *Manager) namespace() string { + if m.config.Namespace != "" { + return m.config.Namespace + } + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// releaseName returns the release name for this instance. +func (m *Manager) releaseName() string { + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// Job returns the Prometheus job name for this Reloader instance. +func (m *Manager) Job() string { + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// Deploy deploys Reloader to the cluster using raw manifests. +func (m *Manager) Deploy(ctx context.Context) error { + ns := m.namespace() + name := m.releaseName() + + fmt.Printf("Deploying Reloader (%s) with image %s...\n", m.config.Version, m.config.Image) + + manifest := m.buildManifest(ns, name) + + applyCmd := m.kubectl(ctx, "apply", "-f", "-") + applyCmd.Stdin = strings.NewReader(manifest) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying manifest: %w", err) + } + + fmt.Printf("Waiting for Reloader deployment to be ready...\n") + waitCmd := m.kubectl(ctx, "rollout", "status", "deployment", name, + "-n", ns, + "--timeout=120s") + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for deployment: %w", err) + } + + time.Sleep(2 * time.Second) + + fmt.Printf("Reloader (%s) deployed successfully\n", m.config.Version) + return nil +} + +// buildManifest creates the raw Kubernetes manifest for Reloader. +func (m *Manager) buildManifest(ns, name string) string { + var args []string + args = append(args, "--log-format=json") + if m.config.ReloadStrategy != "" && m.config.ReloadStrategy != "default" { + args = append(args, fmt.Sprintf("--reload-strategy=%s", m.config.ReloadStrategy)) + } + + argsYAML := "" + if len(args) > 0 { + argsYAML = " args:\n" + for _, arg := range args { + argsYAML += fmt.Sprintf(" - %q\n", arg) + } + } + + return fmt.Sprintf(`--- +apiVersion: v1 +kind: Namespace +metadata: + name: %[1]s +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: %[2]s + namespace: %[1]s +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: %[2]s +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: %[2]s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %[2]s +subjects: +- kind: ServiceAccount + name: %[2]s + namespace: %[1]s +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %[2]s + namespace: %[1]s + labels: + app: %[2]s + app.kubernetes.io/name: reloader + loadtest-version: %[3]s +spec: + replicas: 1 + selector: + matchLabels: + app: %[2]s + template: + metadata: + labels: + app: %[2]s + app.kubernetes.io/name: reloader + loadtest-version: %[3]s + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: %[2]s + securityContext: + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: reloader + image: %[4]s + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9090 +%[5]s resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true +`, ns, name, m.config.Version, m.config.Image, argsYAML) +} + +// Cleanup removes all Reloader resources from the cluster. +func (m *Manager) Cleanup(ctx context.Context) error { + ns := m.namespace() + name := m.releaseName() + + delDeploy := m.kubectl(ctx, "delete", "deployment", name, "-n", ns, "--ignore-not-found") + delDeploy.Run() + + delCRB := m.kubectl(ctx, "delete", "clusterrolebinding", name, "--ignore-not-found") + delCRB.Run() + + delCR := m.kubectl(ctx, "delete", "clusterrole", name, "--ignore-not-found") + delCR.Run() + + delNS := m.kubectl(ctx, "delete", "namespace", ns, "--wait=false", "--ignore-not-found") + if err := delNS.Run(); err != nil { + return fmt.Errorf("deleting namespace: %w", err) + } + + return nil +} + +// CollectLogs collects logs from the Reloader pod and writes them to the specified file. +func (m *Manager) CollectLogs(ctx context.Context, logPath string) error { + ns := m.namespace() + name := m.releaseName() + + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + return fmt.Errorf("creating log directory: %w", err) + } + + cmd := m.kubectl(ctx, "logs", + "-n", ns, + "-l", fmt.Sprintf("app=%s", name), + "--tail=-1") + + out, err := cmd.Output() + if err != nil { + cmd = m.kubectl(ctx, "logs", + "-n", ns, + "-l", "app.kubernetes.io/name=reloader", + "--tail=-1") + out, err = cmd.Output() + if err != nil { + return fmt.Errorf("collecting logs: %w", err) + } + } + + if err := os.WriteFile(logPath, out, 0644); err != nil { + return fmt.Errorf("writing logs: %w", err) + } + + return nil +} diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go index ed48a3bfa..794f1a688 100644 --- a/test/loadtest/internal/scenarios/scenarios.go +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -1137,7 +1137,7 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int for time.Now().Before(endTime) { select { case <-ctx.Done(): - return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil case <-ticker.C: if updateSecret { // Update a random Secret @@ -1169,21 +1169,25 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } log.Printf("S10: Completed %d Secret updates and %d ConfigMap updates", secretUpdateCount, cmUpdateCount) - return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil } -func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { - // Secret updates trigger: secret-only deploys + mixed deploys - secretTriggeredReloads := secretUpdates * (secretOnlyDeploys + mixedDeploys) - // ConfigMap updates trigger: cm-only deploys + mixed deploys - cmTriggeredReloads := cmUpdates * (cmOnlyDeploys + mixedDeploys) +func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, numSecrets, numConfigMaps, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { + // Average deploys triggered per random secret update + avgSecretReloads := float64(secretOnlyDeploys)/float64(numSecrets) + float64(mixedDeploys)/float64(numSecrets) + secretTriggeredReloads := int(float64(secretUpdates) * avgSecretReloads) + + // Average deploys triggered per random CM update + avgCMReloads := float64(cmOnlyDeploys)/float64(numConfigMaps) + float64(mixedDeploys)/float64(numConfigMaps) + cmTriggeredReloads := int(float64(cmUpdates) * avgCMReloads) + totalExpectedReloads := secretTriggeredReloads + cmTriggeredReloads return ExpectedMetrics{ ActionTotal: totalExpectedReloads, ReloadExecutedTotal: totalExpectedReloads, - Description: fmt.Sprintf("S10: %d Secret updates (→%d reloads) + %d CM updates (→%d reloads) = %d total", - secretUpdates, secretTriggeredReloads, cmUpdates, cmTriggeredReloads, totalExpectedReloads), + Description: fmt.Sprintf("S10: %d Secret updates (→%d reloads, avg %.1f/update) + %d CM updates (→%d reloads, avg %.1f/update) = %d total", + secretUpdates, secretTriggeredReloads, avgSecretReloads, cmUpdates, cmTriggeredReloads, avgCMReloads, totalExpectedReloads), } } From 922cac120aa5bf3f5fc9c979d57d5e4e68d38bbd Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 22:15:21 +0100 Subject: [PATCH 08/21] fix: Update gitignore to include results and nfs files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a398472b1..3f28c3f59 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,8 @@ styles/ site/ /mkdocs.yml yq -bin \ No newline at end of file +bin +test/loadtest/results +test/loadtest/loadtest +# Temporary NFS files +.nfs* From 958c6c2be77a433d91b638419f030761d09faa5f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 22:52:07 +0100 Subject: [PATCH 09/21] feat(ci): Separate action for loadtests --- .github/actions/loadtest/action.yml | 256 ++++++++++++++++++++ .github/workflows/loadtest.yml | 165 ++----------- .github/workflows/pull_request.yaml | 12 + Makefile | 40 ++++ test/loadtest/cmd/loadtest/main.go | 354 +++++++++++++++++++++++++++- 5 files changed, 677 insertions(+), 150 deletions(-) create mode 100644 .github/actions/loadtest/action.yml diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml new file mode 100644 index 000000000..3b91957ec --- /dev/null +++ b/.github/actions/loadtest/action.yml @@ -0,0 +1,256 @@ +name: 'Reloader Load Test' +description: 'Run Reloader load tests with A/B comparison support' + +inputs: + old-ref: + description: 'Git ref for "old" version (optional, enables A/B comparison)' + required: false + default: '' + new-ref: + description: 'Git ref for "new" version (defaults to current checkout)' + required: false + default: '' + old-image: + description: 'Pre-built container image for "old" version (alternative to old-ref)' + required: false + default: '' + new-image: + description: 'Pre-built container image for "new" version (alternative to new-ref)' + required: false + default: '' + scenarios: + description: 'Scenarios to run: S1,S4,S6 or all' + required: false + default: 'S1,S4,S6' + test-type: + description: 'Test type label for summary: quick or full' + required: false + default: 'quick' + duration: + description: 'Test duration in seconds' + required: false + default: '60' + kind-cluster: + description: 'Name of existing Kind cluster (if empty, creates new one)' + required: false + default: '' + post-comment: + description: 'Post results as PR comment' + required: false + default: 'false' + pr-number: + description: 'PR number for commenting (required if post-comment is true)' + required: false + default: '' + github-token: + description: 'GitHub token for posting comments' + required: false + default: ${{ github.token }} + comment-header: + description: 'Optional header text for the comment' + required: false + default: '' + +outputs: + status: + description: 'Overall test status: pass or fail' + value: ${{ steps.run.outputs.status }} + summary: + description: 'Markdown summary of results' + value: ${{ steps.summary.outputs.summary }} + pass-count: + description: 'Number of passed scenarios' + value: ${{ steps.summary.outputs.pass_count }} + fail-count: + description: 'Number of failed scenarios' + value: ${{ steps.summary.outputs.fail_count }} + +runs: + using: 'composite' + steps: + - name: Determine images to use + id: images + shell: bash + run: | + # Determine old image + if [ -n "${{ inputs.old-image }}" ]; then + echo "old=${{ inputs.old-image }}" >> $GITHUB_OUTPUT + elif [ -n "${{ inputs.old-ref }}" ]; then + echo "old=localhost/reloader:old" >> $GITHUB_OUTPUT + echo "build_old=true" >> $GITHUB_OUTPUT + else + echo "old=" >> $GITHUB_OUTPUT + fi + + # Determine new image + if [ -n "${{ inputs.new-image }}" ]; then + echo "new=${{ inputs.new-image }}" >> $GITHUB_OUTPUT + elif [ -n "${{ inputs.new-ref }}" ]; then + echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT + echo "build_new=true" >> $GITHUB_OUTPUT + else + # Default: build from current checkout + echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT + echo "build_new_current=true" >> $GITHUB_OUTPUT + fi + + - name: Build old image from ref + if: steps.images.outputs.build_old == 'true' + shell: bash + run: | + CURRENT_SHA=$(git rev-parse HEAD) + git checkout ${{ inputs.old-ref }} + docker build -t localhost/reloader:old . + echo "Built old image from ref: ${{ inputs.old-ref }}" + git checkout $CURRENT_SHA + + - name: Build new image from ref + if: steps.images.outputs.build_new == 'true' + shell: bash + run: | + CURRENT_SHA=$(git rev-parse HEAD) + git checkout ${{ inputs.new-ref }} + docker build -t localhost/reloader:new . + echo "Built new image from ref: ${{ inputs.new-ref }}" + git checkout $CURRENT_SHA + + - name: Build new image from current checkout + if: steps.images.outputs.build_new_current == 'true' + shell: bash + run: | + docker build -t localhost/reloader:new . + echo "Built new image from current checkout" + + - name: Build loadtest binary + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + go build -o loadtest ./cmd/loadtest + + - name: Determine cluster name + id: cluster + shell: bash + run: | + if [ -n "${{ inputs.kind-cluster }}" ]; then + echo "name=${{ inputs.kind-cluster }}" >> $GITHUB_OUTPUT + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "name=reloader-loadtest" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT + fi + + - name: Load images into Kind + shell: bash + run: | + CLUSTER="${{ steps.cluster.outputs.name }}" + + if [ -n "${{ steps.images.outputs.old }}" ]; then + echo "Loading old image: ${{ steps.images.outputs.old }}" + kind load docker-image "${{ steps.images.outputs.old }}" --name "$CLUSTER" || true + fi + + echo "Loading new image: ${{ steps.images.outputs.new }}" + kind load docker-image "${{ steps.images.outputs.new }}" --name "$CLUSTER" || true + + - name: Run load tests + id: run + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + + ARGS="--new-image=${{ steps.images.outputs.new }}" + ARGS="$ARGS --scenario=${{ inputs.scenarios }}" + ARGS="$ARGS --duration=${{ inputs.duration }}" + ARGS="$ARGS --cluster-name=${{ steps.cluster.outputs.name }}" + + if [ -n "${{ steps.images.outputs.old }}" ]; then + ARGS="$ARGS --old-image=${{ steps.images.outputs.old }}" + fi + + if [ "${{ steps.cluster.outputs.skip }}" = "true" ]; then + ARGS="$ARGS --skip-cluster" + fi + + echo "Running: ./loadtest run $ARGS" + if ./loadtest run $ARGS; then + echo "status=pass" >> $GITHUB_OUTPUT + else + echo "status=fail" >> $GITHUB_OUTPUT + fi + + - name: Generate summary + id: summary + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + + # Generate markdown summary + ./loadtest summary \ + --results-dir=./results \ + --test-type=${{ inputs.test-type }} \ + --format=markdown > summary.md 2>/dev/null || true + + # Output to GitHub Step Summary + cat summary.md >> $GITHUB_STEP_SUMMARY + + # Store summary for output (using heredoc for multiline) + { + echo 'summary<> $GITHUB_OUTPUT + + # Get pass/fail counts from JSON + COUNTS=$(./loadtest summary --format=json 2>/dev/null | head -20 || echo '{}') + echo "pass_count=$(echo "$COUNTS" | grep -o '"pass_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT + echo "fail_count=$(echo "$COUNTS" | grep -o '"fail_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT + + - name: Post PR comment + if: inputs.post-comment == 'true' && inputs.pr-number != '' + uses: actions/github-script@v7 + with: + github-token: ${{ inputs.github-token }} + script: | + const fs = require('fs'); + const summaryPath = '${{ github.workspace }}/test/loadtest/summary.md'; + let summary = 'No results available'; + try { + summary = fs.readFileSync(summaryPath, 'utf8'); + } catch (e) { + console.log('Could not read summary file:', e.message); + } + + const header = '${{ inputs.comment-header }}'; + const status = '${{ steps.run.outputs.status }}'; + const statusEmoji = status === 'pass' ? ':white_check_mark:' : ':x:'; + + const body = [ + header ? header : `## ${statusEmoji} Load Test Results (${{ inputs.test-type }})`, + '', + summary, + '', + '---', + `**Artifacts:** [Download](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`, + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + body: body + }); + + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: loadtest-${{ inputs.test-type }}-results + path: | + ${{ github.workspace }}/test/loadtest/results/ + retention-days: 30 + + - name: Cleanup Kind cluster (only if we created it) + if: always() && steps.cluster.outputs.skip == 'false' + shell: bash + run: | + kind delete cluster --name ${{ steps.cluster.outputs.name }} || true diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a67744445..f4eb32216 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -1,4 +1,4 @@ -name: Load Test +name: Load Test (Full) on: issue_comment: @@ -10,6 +10,7 @@ permissions: issues: write jobs: + # Full load test suite triggered by /loadtest command loadtest: # Only run on PR comments with /loadtest command if: | @@ -45,6 +46,12 @@ jobs: core.setOutput('base_sha', pr.data.base.sha); console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`); + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.head_sha }} + fetch-depth: 0 # Full history for building from base ref + - name: Set up Go uses: actions/setup-go@v5 with: @@ -66,151 +73,23 @@ jobs: chmod +x kubectl sudo mv kubectl /usr/local/bin/kubectl - # Build OLD image from base branch (e.g., main) - - name: Checkout base branch (old) - uses: actions/checkout@v4 - with: - ref: ${{ steps.pr.outputs.base_ref }} - path: old - - - name: Build old image - run: | - cd old - docker build -t localhost/reloader:old -f Dockerfile . - echo "Built old image from ${{ steps.pr.outputs.base_ref }} (${{ steps.pr.outputs.base_sha }})" - - # Build NEW image from PR branch - - name: Checkout PR branch (new) - uses: actions/checkout@v4 - with: - ref: ${{ steps.pr.outputs.head_ref }} - path: new - - - name: Build new image - run: | - cd new - docker build -t localhost/reloader:new -f Dockerfile . - echo "Built new image from ${{ steps.pr.outputs.head_ref }} (${{ steps.pr.outputs.head_sha }})" - - # Build and run loadtest from PR branch - - name: Build loadtest tool - run: | - cd new/test/loadtest - go build -o loadtest ./cmd/loadtest - - - name: Run A/B comparison load test + - name: Run full A/B comparison load test id: loadtest - run: | - cd new/test/loadtest - ./loadtest run \ - --old-image=localhost/reloader:old \ - --new-image=localhost/reloader:new \ - --scenario=all \ - --duration=60 2>&1 | tee loadtest-output.txt - echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: loadtest-results - path: | - new/test/loadtest/results/ - new/test/loadtest/loadtest-output.txt - retention-days: 30 - - - name: Post results comment - uses: actions/github-script@v7 - if: always() + uses: ./.github/actions/loadtest with: - script: | - const fs = require('fs'); - - let results = ''; - const resultsDir = 'new/test/loadtest/results'; - - // Collect summary of all scenarios - let passCount = 0; - let failCount = 0; - const summaries = []; - - if (fs.existsSync(resultsDir)) { - const scenarios = fs.readdirSync(resultsDir).sort(); - for (const scenario of scenarios) { - const reportPath = `${resultsDir}/${scenario}/report.txt`; - if (fs.existsSync(reportPath)) { - const report = fs.readFileSync(reportPath, 'utf8'); - - // Extract status from report - const statusMatch = report.match(/Status:\s+(PASS|FAIL)/); - const status = statusMatch ? statusMatch[1] : 'UNKNOWN'; - - if (status === 'PASS') passCount++; - else failCount++; - - // Extract key metrics for summary - const actionMatch = report.match(/action_total\s+[\d.]+\s+[\d.]+\s+[\d.]+/); - const errorsMatch = report.match(/errors_total\s+[\d.]+\s+[\d.]+/); - - summaries.push(`| ${scenario} | ${status === 'PASS' ? '✅' : '❌'} ${status} |`); - - results += `\n
\n${status === 'PASS' ? '✅' : '❌'} ${scenario}\n\n\`\`\`\n${report}\n\`\`\`\n
\n`; - } - } - } - - if (!results) { - // Read raw output if no reports - if (fs.existsSync('new/test/loadtest/loadtest-output.txt')) { - const output = fs.readFileSync('new/test/loadtest/loadtest-output.txt', 'utf8'); - const maxLen = 60000; - results = output.length > maxLen - ? output.substring(output.length - maxLen) - : output; - results = `\`\`\`\n${results}\n\`\`\``; - } else { - results = 'No results available'; - } - } - - const overallStatus = failCount === 0 ? '✅ ALL PASSED' : `❌ ${failCount} FAILED`; - - const body = [ - `## Load Test Results ${overallStatus}`, - '', - `**Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new)`, - `**Old commit:** ${{ steps.pr.outputs.base_sha }}`, - `**New commit:** ${{ steps.pr.outputs.head_sha }}`, - `**Triggered by:** @${{ github.event.comment.user.login }}`, - '', - '### Summary', - '', - '| Scenario | Status |', - '|----------|--------|', - summaries.join('\n'), - '', - `**Total:** ${passCount} passed, ${failCount} failed`, - '', - '### Detailed Results', - '', - results, - '', - '
', - '📦 Download full results', - '', - `Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).`, - '
', - ].join('\n'); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body - }); + old-ref: ${{ steps.pr.outputs.base_sha }} + new-ref: ${{ steps.pr.outputs.head_sha }} + scenarios: 'all' + test-type: 'full' + post-comment: 'true' + pr-number: ${{ github.event.issue.number }} + comment-header: | + ## Load Test Results (Full A/B Comparison) + **Comparing:** `${{ steps.pr.outputs.base_ref }}` → `${{ steps.pr.outputs.head_ref }}` + **Triggered by:** @${{ github.event.comment.user.login }} - name: Add success reaction - if: success() + if: steps.loadtest.outputs.status == 'pass' uses: actions/github-script@v7 with: script: | @@ -222,7 +101,7 @@ jobs: }); - name: Add failure reaction - if: failure() + if: steps.loadtest.outputs.status == 'fail' uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index e4b1c6f2c..fbf59abae 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -35,6 +35,7 @@ jobs: permissions: contents: read + pull-requests: write runs-on: ubuntu-latest name: Build @@ -109,6 +110,17 @@ jobs: - name: Test run: make test + - name: Run quick A/B load tests + uses: ./.github/actions/loadtest + with: + old-ref: ${{ github.event.pull_request.base.sha }} + # new-ref defaults to current checkout (PR branch) + scenarios: 'S1,S4,S6' + test-type: 'quick' + kind-cluster: 'kind' # Use the existing cluster created above + post-comment: 'true' + pr-number: ${{ github.event.pull_request.number }} + - name: Generate Tags id: generate_tag run: | diff --git a/Makefile b/Makefile index 8444e1f76..8c0aed8fa 100644 --- a/Makefile +++ b/Makefile @@ -169,3 +169,43 @@ yq-install: @curl -sL $(YQ_DOWNLOAD_URL) -o $(YQ_BIN) @chmod +x $(YQ_BIN) @echo "yq $(YQ_VERSION) installed at $(YQ_BIN)" + +# ============================================================================= +# Load Testing +# ============================================================================= + +LOADTEST_BIN = test/loadtest/loadtest +LOADTEST_OLD_IMAGE ?= localhost/reloader:old +LOADTEST_NEW_IMAGE ?= localhost/reloader:new +LOADTEST_DURATION ?= 60 +LOADTEST_SCENARIOS ?= all + +.PHONY: loadtest-build loadtest-quick loadtest-full loadtest loadtest-clean + +loadtest-build: ## Build loadtest binary + cd test/loadtest && $(GOCMD) build -o loadtest ./cmd/loadtest + +loadtest-quick: loadtest-build ## Run quick load tests (S1, S4, S6) + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=S1,S4,S6 \ + --duration=$(LOADTEST_DURATION) + +loadtest-full: loadtest-build ## Run full load test suite + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=all \ + --duration=$(LOADTEST_DURATION) + +loadtest: loadtest-build ## Run load tests with configurable scenarios (default: all) + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=$(LOADTEST_SCENARIOS) \ + --duration=$(LOADTEST_DURATION) + +loadtest-clean: ## Clean loadtest binary and results + rm -f $(LOADTEST_BIN) + rm -rf test/loadtest/results diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index 572509007..fb05db58e 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -30,6 +30,15 @@ const ( testNamespace = "reloader-test" ) +// OutputFormat defines the output format for reports. +type OutputFormat string + +const ( + OutputFormatText OutputFormat = "text" + OutputFormatJSON OutputFormat = "json" + OutputFormatMarkdown OutputFormat = "markdown" +) + // workerContext holds all resources for a single worker (cluster + prometheus). type workerContext struct { id int @@ -47,6 +56,7 @@ type Config struct { Scenario string Duration int SkipCluster bool + ClusterName string // Custom cluster name (default: reloader-loadtest) ResultsDir string ManifestsDir string Parallelism int // Number of parallel clusters (1 = sequential) @@ -64,6 +74,8 @@ func main() { runCommand(os.Args[2:]) case "report": reportCommand(os.Args[2:]) + case "summary": + summaryCommand(os.Args[2:]) case "help", "--help", "-h": printUsage() default: @@ -78,7 +90,8 @@ func printUsage() { Usage: loadtest run [options] Run A/B comparison tests - loadtest report [options] Generate comparison report + loadtest report [options] Generate comparison report for a scenario + loadtest summary [options] Generate summary across all scenarios (for CI) loadtest help Show this help Run Options: @@ -87,13 +100,21 @@ Run Options: --scenario=ID Test scenario: S1-S13 or "all" (default: all) --duration=SECONDS Test duration in seconds (default: 60) --parallelism=N Run N scenarios in parallel on N clusters (default: 1) - --skip-cluster Skip kind cluster creation (use existing, only for parallelism=1) + --skip-cluster Skip kind cluster creation (use existing) + --cluster-name=NAME Kind cluster name (default: reloader-loadtest) --results-dir=DIR Directory for results (default: ./results) Report Options: --scenario=ID Scenario to report on (required) --results-dir=DIR Directory containing results (default: ./results) --output=FILE Output file (default: stdout) + --format=FORMAT Output format: text, json, markdown (default: text) + +Summary Options: + --results-dir=DIR Directory containing results (default: ./results) + --output=FILE Output file (default: stdout) + --format=FORMAT Output format: text, json, markdown (default: markdown) + --test-type=TYPE Test type label: quick, full (default: full) Examples: # Compare two images @@ -111,8 +132,14 @@ Examples: # Run all 13 scenarios in parallel (one cluster per scenario) loadtest run --new-image=localhost/reloader:test --parallelism=13 - # Generate report + # Generate report for a scenario loadtest report --scenario=S2 --results-dir=./results + + # Generate JSON report + loadtest report --scenario=S2 --format=json + + # Generate markdown summary for CI + loadtest summary --results-dir=./results --format=markdown `) } @@ -122,6 +149,7 @@ func parseArgs(args []string) Config { Duration: 60, ResultsDir: "./results", Parallelism: 1, + ClusterName: clusterName, // default } // Find manifests dir relative to executable or current dir @@ -151,6 +179,8 @@ func parseArgs(args []string) Config { } case arg == "--skip-cluster": cfg.SkipCluster = true + case strings.HasPrefix(arg, "--cluster-name="): + cfg.ClusterName = strings.TrimPrefix(arg, "--cluster-name=") case strings.HasPrefix(arg, "--results-dir="): cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") case strings.HasPrefix(arg, "--manifests-dir="): @@ -234,15 +264,15 @@ func runCommand(args []string) { func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { // Create cluster manager clusterMgr := cluster.NewManager(cluster.Config{ - Name: clusterName, + Name: cfg.ClusterName, ContainerRuntime: runtime, }) // Create/verify cluster if cfg.SkipCluster { - log.Println("Skipping cluster creation (using existing)") + log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) if !clusterMgr.Exists() { - log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", clusterName) + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) } } else { log.Println("Creating kind cluster...") @@ -781,6 +811,7 @@ func cleanupReloader(ctx context.Context, version string, kubeContext string) { func reportCommand(args []string) { var scenarioID, resultsDir, outputFile string + format := OutputFormatText resultsDir = "./results" for _, arg := range args { @@ -791,6 +822,8 @@ func reportCommand(args []string) { resultsDir = strings.TrimPrefix(arg, "--results-dir=") case strings.HasPrefix(arg, "--output="): outputFile = strings.TrimPrefix(arg, "--output=") + case strings.HasPrefix(arg, "--format="): + format = OutputFormat(strings.TrimPrefix(arg, "--format=")) } } @@ -803,7 +836,15 @@ func reportCommand(args []string) { log.Fatalf("Failed to generate report: %v", err) } - output := renderScenarioReport(report) + var output string + switch format { + case OutputFormatJSON: + output = renderScenarioReportJSON(report) + case OutputFormatMarkdown: + output = renderScenarioReportMarkdown(report) + default: + output = renderScenarioReport(report) + } if outputFile != "" { if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { @@ -1584,3 +1625,302 @@ func renderScenarioReport(report *ScenarioReport) string { return sb.String() } + +// renderScenarioReportJSON renders a scenario report as JSON. +func renderScenarioReportJSON(report *ScenarioReport) string { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +// renderScenarioReportMarkdown renders a scenario report as concise markdown. +func renderScenarioReportMarkdown(report *ScenarioReport) string { + var sb strings.Builder + + // Status emoji + emoji := "✅" + if report.OverallStatus != "PASS" { + emoji = "❌" + } + + sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) + + if report.TestDescription != "" { + sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) + } + + // Key metrics table + sb.WriteString("| Metric | Value | Expected | Status |\n") + sb.WriteString("|--------|------:|:--------:|:------:|\n") + + // Show only key metrics + keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} + for _, name := range keyMetrics { + for _, c := range report.Comparisons { + if c.Name == name { + value := fmt.Sprintf("%.0f", c.NewValue) + expected := "-" + if c.Expected > 0 { + expected = fmt.Sprintf("%.0f", c.Expected) + } + status := "✅" + if c.Status == "fail" { + status = "❌" + } else if c.Status == "info" { + status = "ℹ️" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) + break + } + } + } + + return sb.String() +} + +// ============================================================================ +// SUMMARY COMMAND +// ============================================================================ + +// SummaryReport aggregates results from multiple scenarios. +type SummaryReport struct { + Timestamp time.Time `json:"timestamp"` + TestType string `json:"test_type"` + PassCount int `json:"pass_count"` + FailCount int `json:"fail_count"` + TotalCount int `json:"total_count"` + Scenarios []ScenarioSummary `json:"scenarios"` +} + +// ScenarioSummary provides a brief summary of a single scenario. +type ScenarioSummary struct { + ID string `json:"id"` + Status string `json:"status"` + Description string `json:"description"` + ActionTotal float64 `json:"action_total"` + ActionExp float64 `json:"action_expected"` + ErrorsTotal float64 `json:"errors_total"` +} + +func summaryCommand(args []string) { + var resultsDir, outputFile, testType string + format := OutputFormatMarkdown // Default to markdown for CI + resultsDir = "./results" + testType = "full" + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--results-dir="): + resultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--output="): + outputFile = strings.TrimPrefix(arg, "--output=") + case strings.HasPrefix(arg, "--format="): + format = OutputFormat(strings.TrimPrefix(arg, "--format=")) + case strings.HasPrefix(arg, "--test-type="): + testType = strings.TrimPrefix(arg, "--test-type=") + } + } + + summary, err := generateSummaryReport(resultsDir, testType) + if err != nil { + log.Fatalf("Failed to generate summary: %v", err) + } + + var output string + switch format { + case OutputFormatJSON: + output = renderSummaryJSON(summary) + case OutputFormatText: + output = renderSummaryText(summary) + default: + output = renderSummaryMarkdown(summary) + } + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Summary written to %s", outputFile) + } else { + fmt.Print(output) + } + + // Exit with non-zero status if any tests failed + if summary.FailCount > 0 { + os.Exit(1) + } +} + +func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { + summary := &SummaryReport{ + Timestamp: time.Now(), + TestType: testType, + } + + // Find all scenario directories + entries, err := os.ReadDir(resultsDir) + if err != nil { + return nil, fmt.Errorf("failed to read results directory: %w", err) + } + + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { + continue + } + + scenarioID := entry.Name() + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) + continue + } + + scenarioSummary := ScenarioSummary{ + ID: scenarioID, + Status: report.OverallStatus, + Description: report.TestDescription, + } + + // Extract key metrics + for _, c := range report.Comparisons { + switch c.Name { + case "action_total": + scenarioSummary.ActionTotal = c.NewValue + scenarioSummary.ActionExp = c.Expected + case "errors_total": + scenarioSummary.ErrorsTotal = c.NewValue + } + } + + summary.Scenarios = append(summary.Scenarios, scenarioSummary) + summary.TotalCount++ + if report.OverallStatus == "PASS" { + summary.PassCount++ + } else { + summary.FailCount++ + } + } + + // Sort scenarios by ID + sort.Slice(summary.Scenarios, func(i, j int) bool { + return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) + }) + + return summary, nil +} + +// naturalSort compares two scenario IDs (S1, S2, ..., S10, S11) +func naturalSort(a, b string) bool { + var aNum, bNum int + fmt.Sscanf(a, "S%d", &aNum) + fmt.Sscanf(b, "S%d", &bNum) + return aNum < bNum +} + +func renderSummaryJSON(summary *SummaryReport) string { + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderSummaryText(summary *SummaryReport) string { + var sb strings.Builder + + sb.WriteString("================================================================================\n") + sb.WriteString(" LOAD TEST SUMMARY\n") + sb.WriteString("================================================================================\n\n") + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + + fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) + fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) + + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") + + for _, s := range summary.Scenarios { + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) + } + + sb.WriteString("\n================================================================================\n") + return sb.String() +} + +func renderSummaryMarkdown(summary *SummaryReport) string { + var sb strings.Builder + + // Overall status + emoji := "✅" + title := "ALL TESTS PASSED" + if summary.FailCount > 0 { + emoji = "❌" + title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) + } else if summary.TotalCount == 0 { + emoji = "⚠️" + title = "NO RESULTS" + } + + sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) + + // Test type note + if summary.TestType == "quick" { + sb.WriteString("> 🚀 **Quick Test** (S1, S4, S6) — Use `/loadtest` for full suite\n\n") + } + + // Pass rate + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) + + // Results table + sb.WriteString("| | Scenario | Description | Actions | Errors |\n") + sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") + + for _, s := range summary.Scenarios { + icon := "✅" + if s.Status != "PASS" { + icon = "❌" + } + + // Truncate description + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + + // Format actions + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + + // Format errors + errors := fmt.Sprintf("%.0f", s.ErrorsTotal) + if s.ErrorsTotal > 0 { + errors = fmt.Sprintf("⚠️ %.0f", s.ErrorsTotal) + } + + sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) + } + + sb.WriteString("\n📦 **[Download detailed results](../artifacts)**\n") + + return sb.String() +} From 322c4bc13027c449fb26ce0ce69f30a33509fa2e Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:26:41 +0100 Subject: [PATCH 10/21] feat: Use cobra for loadtest CLI commands --- test/loadtest/cmd/loadtest/main.go | 1924 +------------------------ test/loadtest/go.mod | 4 +- test/loadtest/go.sum | 6 + test/loadtest/internal/cmd/report.go | 856 +++++++++++ test/loadtest/internal/cmd/root.go | 44 + test/loadtest/internal/cmd/run.go | 648 +++++++++ test/loadtest/internal/cmd/summary.go | 251 ++++ 7 files changed, 1811 insertions(+), 1922 deletions(-) create mode 100644 test/loadtest/internal/cmd/report.go create mode 100644 test/loadtest/internal/cmd/root.go create mode 100644 test/loadtest/internal/cmd/run.go create mode 100644 test/loadtest/internal/cmd/summary.go diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index fb05db58e..a8925bc37 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -1,1926 +1,8 @@ -// Package main implements the unified load test CLI for Reloader A/B comparison. +// Package main is the entrypoint for the load test CLI. package main -import ( - "context" - "encoding/json" - "fmt" - "log" - "math" - "os" - "os/exec" - "os/signal" - "path/filepath" - "sort" - "strings" - "sync" - "syscall" - "time" - - "github.com/stakater/Reloader/test/loadtest/internal/cluster" - "github.com/stakater/Reloader/test/loadtest/internal/prometheus" - "github.com/stakater/Reloader/test/loadtest/internal/reloader" - "github.com/stakater/Reloader/test/loadtest/internal/scenarios" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -const ( - clusterName = "reloader-loadtest" - testNamespace = "reloader-test" -) - -// OutputFormat defines the output format for reports. -type OutputFormat string - -const ( - OutputFormatText OutputFormat = "text" - OutputFormatJSON OutputFormat = "json" - OutputFormatMarkdown OutputFormat = "markdown" -) - -// workerContext holds all resources for a single worker (cluster + prometheus). -type workerContext struct { - id int - clusterMgr *cluster.Manager - promMgr *prometheus.Manager - kubeClient kubernetes.Interface - kubeContext string - runtime string -} - -// Config holds CLI configuration. -type Config struct { - OldImage string - NewImage string - Scenario string - Duration int - SkipCluster bool - ClusterName string // Custom cluster name (default: reloader-loadtest) - ResultsDir string - ManifestsDir string - Parallelism int // Number of parallel clusters (1 = sequential) -} +import "github.com/stakater/Reloader/test/loadtest/internal/cmd" func main() { - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - cmd := os.Args[1] - switch cmd { - case "run": - runCommand(os.Args[2:]) - case "report": - reportCommand(os.Args[2:]) - case "summary": - summaryCommand(os.Args[2:]) - case "help", "--help", "-h": - printUsage() - default: - fmt.Printf("Unknown command: %s\n", cmd) - printUsage() - os.Exit(1) - } -} - -func printUsage() { - fmt.Print(`Reloader Load Test CLI - -Usage: - loadtest run [options] Run A/B comparison tests - loadtest report [options] Generate comparison report for a scenario - loadtest summary [options] Generate summary across all scenarios (for CI) - loadtest help Show this help - -Run Options: - --old-image=IMAGE Container image for "old" version (required for comparison) - --new-image=IMAGE Container image for "new" version (required for comparison) - --scenario=ID Test scenario: S1-S13 or "all" (default: all) - --duration=SECONDS Test duration in seconds (default: 60) - --parallelism=N Run N scenarios in parallel on N clusters (default: 1) - --skip-cluster Skip kind cluster creation (use existing) - --cluster-name=NAME Kind cluster name (default: reloader-loadtest) - --results-dir=DIR Directory for results (default: ./results) - -Report Options: - --scenario=ID Scenario to report on (required) - --results-dir=DIR Directory containing results (default: ./results) - --output=FILE Output file (default: stdout) - --format=FORMAT Output format: text, json, markdown (default: text) - -Summary Options: - --results-dir=DIR Directory containing results (default: ./results) - --output=FILE Output file (default: stdout) - --format=FORMAT Output format: text, json, markdown (default: markdown) - --test-type=TYPE Test type label: quick, full (default: full) - -Examples: - # Compare two images - loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 - - # Run specific scenario - loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 - - # Test single image (no comparison) - loadtest run --new-image=localhost/reloader:test - - # Run all scenarios in parallel on 4 clusters - loadtest run --new-image=localhost/reloader:test --parallelism=4 - - # Run all 13 scenarios in parallel (one cluster per scenario) - loadtest run --new-image=localhost/reloader:test --parallelism=13 - - # Generate report for a scenario - loadtest report --scenario=S2 --results-dir=./results - - # Generate JSON report - loadtest report --scenario=S2 --format=json - - # Generate markdown summary for CI - loadtest summary --results-dir=./results --format=markdown -`) -} - -func parseArgs(args []string) Config { - cfg := Config{ - Scenario: "all", - Duration: 60, - ResultsDir: "./results", - Parallelism: 1, - ClusterName: clusterName, // default - } - - // Find manifests dir relative to executable or current dir - execPath, _ := os.Executable() - execDir := filepath.Dir(execPath) - cfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") - if _, err := os.Stat(cfg.ManifestsDir); os.IsNotExist(err) { - // Try relative to current dir - cfg.ManifestsDir = "./manifests" - } - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--old-image="): - cfg.OldImage = strings.TrimPrefix(arg, "--old-image=") - case strings.HasPrefix(arg, "--new-image="): - cfg.NewImage = strings.TrimPrefix(arg, "--new-image=") - case strings.HasPrefix(arg, "--scenario="): - cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") - case strings.HasPrefix(arg, "--duration="): - if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration); n != 1 { - log.Printf("Warning: invalid --duration value, using default (%d)", cfg.Duration) - } - case strings.HasPrefix(arg, "--parallelism="): - if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism); n != 1 { - log.Printf("Warning: invalid --parallelism value, using default (%d)", cfg.Parallelism) - } - case arg == "--skip-cluster": - cfg.SkipCluster = true - case strings.HasPrefix(arg, "--cluster-name="): - cfg.ClusterName = strings.TrimPrefix(arg, "--cluster-name=") - case strings.HasPrefix(arg, "--results-dir="): - cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--manifests-dir="): - cfg.ManifestsDir = strings.TrimPrefix(arg, "--manifests-dir=") - } - } - - // Validate parallelism - if cfg.Parallelism < 1 { - cfg.Parallelism = 1 - } - - return cfg -} - -func runCommand(args []string) { - cfg := parseArgs(args) - - // Validate required args - if cfg.OldImage == "" && cfg.NewImage == "" { - log.Fatal("At least one of --old-image or --new-image is required") - } - - // Determine mode - runOld := cfg.OldImage != "" - runNew := cfg.NewImage != "" - runBoth := runOld && runNew - - log.Printf("Configuration:") - log.Printf(" Scenario: %s", cfg.Scenario) - log.Printf(" Duration: %ds", cfg.Duration) - log.Printf(" Parallelism: %d", cfg.Parallelism) - if cfg.OldImage != "" { - log.Printf(" Old image: %s", cfg.OldImage) - } - if cfg.NewImage != "" { - log.Printf(" New image: %s", cfg.NewImage) - } - - // Detect container runtime - runtime, err := cluster.DetectContainerRuntime() - if err != nil { - log.Fatalf("Failed to detect container runtime: %v", err) - } - log.Printf(" Container runtime: %s", runtime) - - // Setup context with signal handling - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigCh - log.Println("Received shutdown signal...") - cancel() - }() - - // Determine scenarios to run - scenariosToRun := []string{cfg.Scenario} - if cfg.Scenario == "all" { - scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} - } - - // Skip-cluster only works for parallelism=1 - if cfg.SkipCluster && cfg.Parallelism > 1 { - log.Fatal("--skip-cluster is not supported with --parallelism > 1") - } - - // If parallelism > 1, use parallel execution - if cfg.Parallelism > 1 { - runParallel(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) - return - } - - // Sequential execution (parallelism == 1) - runSequential(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) -} - -// runSequential runs scenarios one by one on a single cluster. -func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { - // Create cluster manager - clusterMgr := cluster.NewManager(cluster.Config{ - Name: cfg.ClusterName, - ContainerRuntime: runtime, - }) - - // Create/verify cluster - if cfg.SkipCluster { - log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) - if !clusterMgr.Exists() { - log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) - } - } else { - log.Println("Creating kind cluster...") - if err := clusterMgr.Create(ctx); err != nil { - log.Fatalf("Failed to create cluster: %v", err) - } - } - - // Deploy Prometheus - promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") - promMgr := prometheus.NewManager(promManifest) - - log.Println("Installing Prometheus...") - if err := promMgr.Deploy(ctx); err != nil { - log.Fatalf("Failed to deploy Prometheus: %v", err) - } - - if err := promMgr.StartPortForward(ctx); err != nil { - log.Fatalf("Failed to start Prometheus port-forward: %v", err) - } - defer promMgr.StopPortForward() - - // Load images into kind - log.Println("Loading images into kind cluster...") - if runOld { - log.Printf("Loading old image: %s", cfg.OldImage) - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Fatalf("Failed to load old image: %v", err) - } - } - if runNew { - log.Printf("Loading new image: %s", cfg.NewImage) - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Fatalf("Failed to load new image: %v", err) - } - } - - // Pre-pull test images - log.Println("Pre-loading test images...") - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) // Ignore errors - - // Get kubernetes client - kubeClient, err := getKubeClient("") - if err != nil { - log.Fatalf("Failed to create kubernetes client: %v", err) - } - - for _, scenarioID := range scenariosToRun { - log.Printf("========================================") - log.Printf("=== Starting scenario %s ===", scenarioID) - log.Printf("========================================") - - // Clean up from previous scenario - cleanupTestNamespaces(ctx, "") - cleanupReloader(ctx, "old", "") - cleanupReloader(ctx, "new", "") - - // Reset Prometheus - if err := promMgr.Reset(ctx); err != nil { - log.Printf("Warning: failed to reset Prometheus: %v", err) - } - - // Create test namespace - createTestNamespace(ctx, "") - - if runOld { - // Test old version - oldMgr := reloader.NewManager(reloader.Config{ - Version: "old", - Image: cfg.OldImage, - }) - - if err := oldMgr.Deploy(ctx); err != nil { - log.Printf("Failed to deploy old Reloader: %v", err) - continue - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { - log.Printf("Warning: %v", err) - log.Println("Proceeding anyway, but metrics may be incomplete") - } - - runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) - collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) - - if runBoth { - // Clean up for new version - cleanupTestNamespaces(ctx, "") - oldMgr.Cleanup(ctx) - promMgr.Reset(ctx) - createTestNamespace(ctx, "") - } - } - - if runNew { - // Test new version - newMgr := reloader.NewManager(reloader.Config{ - Version: "new", - Image: cfg.NewImage, - }) - - if err := newMgr.Deploy(ctx); err != nil { - log.Printf("Failed to deploy new Reloader: %v", err) - continue - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { - log.Printf("Warning: %v", err) - log.Println("Proceeding anyway, but metrics may be incomplete") - } - - runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) - collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) - } - - // Generate report - generateReport(scenarioID, cfg.ResultsDir, runBoth) - - log.Printf("=== Scenario %s complete ===", scenarioID) - } - - log.Println("Load test complete!") - log.Printf("Results available in: %s", cfg.ResultsDir) -} - -// runParallel runs scenarios in parallel on N separate kind clusters. -func runParallel(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { - numWorkers := cfg.Parallelism - if numWorkers > len(scenariosToRun) { - numWorkers = len(scenariosToRun) - log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) - } - - log.Printf("Starting parallel execution with %d workers", numWorkers) - - // Create workers - workers := make([]*workerContext, numWorkers) - var setupWg sync.WaitGroup - setupErrors := make(chan error, numWorkers) - - log.Println("Setting up worker clusters...") - for i := 0; i < numWorkers; i++ { - setupWg.Add(1) - go func(workerID int) { - defer setupWg.Done() - worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) - if err != nil { - setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) - return - } - workers[workerID] = worker - }(i) - } - - setupWg.Wait() - close(setupErrors) - - // Check for setup errors - for err := range setupErrors { - log.Printf("Error: %v", err) - } - - // Verify all workers are ready - readyWorkers := 0 - for _, w := range workers { - if w != nil { - readyWorkers++ - } - } - if readyWorkers == 0 { - log.Fatal("No workers ready, aborting") - } - if readyWorkers < numWorkers { - log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) - } - - // Cleanup workers on exit - defer func() { - log.Println("Cleaning up worker clusters...") - for _, w := range workers { - if w != nil { - w.promMgr.StopPortForward() - } - } - }() - - // Create scenario channel - scenarioCh := make(chan string, len(scenariosToRun)) - for _, s := range scenariosToRun { - scenarioCh <- s - } - close(scenarioCh) - - // Results tracking - var resultsMu sync.Mutex - completedScenarios := make([]string, 0, len(scenariosToRun)) - - // Start workers - var wg sync.WaitGroup - for _, worker := range workers { - if worker == nil { - continue - } - wg.Add(1) - go func(w *workerContext) { - defer wg.Done() - for scenarioID := range scenarioCh { - select { - case <-ctx.Done(): - return - default: - } - - log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) - - // Clean up from previous scenario - cleanupTestNamespaces(ctx, w.kubeContext) - cleanupReloader(ctx, "old", w.kubeContext) - cleanupReloader(ctx, "new", w.kubeContext) - - // Reset Prometheus - if err := w.promMgr.Reset(ctx); err != nil { - log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) - } - - // Create test namespace - createTestNamespace(ctx, w.kubeContext) - - if runOld { - runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) - } - - if runNew { - runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) - } - - // Generate report - generateReport(scenarioID, cfg.ResultsDir, runBoth) - - resultsMu.Lock() - completedScenarios = append(completedScenarios, scenarioID) - resultsMu.Unlock() - - log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) - } - }(worker) - } - - wg.Wait() - - log.Println("Load test complete!") - log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) - log.Printf("Results available in: %s", cfg.ResultsDir) -} - -// setupWorker creates a cluster and deploys prometheus for a single worker. -func setupWorker(ctx context.Context, cfg Config, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { - workerName := fmt.Sprintf("%s-%d", clusterName, workerID) - promPort := 9091 + workerID - - log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) - - clusterMgr := cluster.NewManager(cluster.Config{ - Name: workerName, - ContainerRuntime: runtime, - PortOffset: workerID, // Each cluster gets unique ports - }) - - if err := clusterMgr.Create(ctx); err != nil { - return nil, fmt.Errorf("creating cluster: %w", err) - } - - kubeContext := clusterMgr.Context() - - // Deploy Prometheus - promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") - promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) - - log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) - if err := promMgr.Deploy(ctx); err != nil { - return nil, fmt.Errorf("deploying prometheus: %w", err) - } - - if err := promMgr.StartPortForward(ctx); err != nil { - return nil, fmt.Errorf("starting prometheus port-forward: %w", err) - } - - // Load images - log.Printf("[Worker %d] Loading images...", workerID) - if runOld { - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) - } - } - if runNew { - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) - } - } - - // Pre-pull test images - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) - - // Get kubernetes client for this context - kubeClient, err := getKubeClient(kubeContext) - if err != nil { - return nil, fmt.Errorf("creating kubernetes client: %w", err) - } - - log.Printf("[Worker %d] Ready", workerID) - return &workerContext{ - id: workerID, - clusterMgr: clusterMgr, - promMgr: promMgr, - kubeClient: kubeClient, - kubeContext: kubeContext, - runtime: runtime, - }, nil -} - -// runVersionOnWorker runs a single version test on a worker. -func runVersionOnWorker(ctx context.Context, w *workerContext, cfg Config, scenarioID, version, image string, cleanupAfter bool) { - mgr := reloader.NewManager(reloader.Config{ - Version: version, - Image: image, - }) - mgr.SetKubeContext(w.kubeContext) - - if err := mgr.Deploy(ctx); err != nil { - log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) - return - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { - log.Printf("[Worker %d] Warning: %v", w.id, err) - log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) - } - - runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) - collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) - - if cleanupAfter { - cleanupTestNamespaces(ctx, w.kubeContext) - mgr.Cleanup(ctx) - w.promMgr.Reset(ctx) - createTestNamespace(ctx, w.kubeContext) - } -} - -func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { - runner, ok := scenarios.Registry[scenarioID] - if !ok { - log.Printf("Unknown scenario: %s", scenarioID) - return - } - - // For S6, set the reloader version - if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { - s6.ReloaderVersion = version - } - - // For S11, set the image to deploy its own Reloader - if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { - s11.Image = image - } - - log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) - - // Debug: check parent context state - if ctx.Err() != nil { - log.Printf("WARNING: Parent context already done: %v", ctx.Err()) - } - - // Add extra time for scenario setup (creating deployments, waiting for ready state) - // Some scenarios like S2 create 50 deployments which can take 2-3 minutes - timeout := time.Duration(duration)*time.Second + 5*time.Minute - log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) - - scenarioCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - expected, err := runner.Run(scenarioCtx, client, testNamespace, time.Duration(duration)*time.Second) - if err != nil { - log.Printf("Scenario %s failed: %v", scenarioID, err) - } - - scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) -} - -func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { - log.Printf("Waiting 5s for Reloader to finish processing events...") - time.Sleep(5 * time.Second) - - log.Printf("Waiting 8s for Prometheus to scrape final metrics...") - time.Sleep(8 * time.Second) - - log.Printf("Collecting metrics for %s...", version) - outputDir := filepath.Join(resultsDir, scenarioID, version) - if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { - log.Printf("Failed to collect metrics: %v", err) - } -} - -func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { - log.Printf("Collecting logs for %s...", version) - logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") - if err := mgr.CollectLogs(ctx, logPath); err != nil { - log.Printf("Failed to collect logs: %v", err) - } -} - -func generateReport(scenarioID, resultsDir string, isComparison bool) { - if isComparison { - log.Println("Generating comparison report...") - } else { - log.Println("Generating single-version report...") - } - - reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") - - // Use the report command - cmd := exec.Command(os.Args[0], "report", - fmt.Sprintf("--scenario=%s", scenarioID), - fmt.Sprintf("--results-dir=%s", resultsDir), - fmt.Sprintf("--output=%s", reportPath)) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() - - // Also print to stdout - if data, err := os.ReadFile(reportPath); err == nil { - fmt.Println(string(data)) - } - - log.Printf("Report saved to: %s", reportPath) -} - -func getKubeClient(kubeContext string) (kubernetes.Interface, error) { - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - home, _ := os.UserHomeDir() - kubeconfig = filepath.Join(home, ".kube", "config") - } - - loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} - configOverrides := &clientcmd.ConfigOverrides{} - if kubeContext != "" { - configOverrides.CurrentContext = kubeContext - } - - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - config, err := kubeConfig.ClientConfig() - if err != nil { - return nil, err - } - - return kubernetes.NewForConfig(config) -} - -func createTestNamespace(ctx context.Context, kubeContext string) { - args := []string{"create", "namespace", testNamespace, "--dry-run=client", "-o", "yaml"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - cmd := exec.CommandContext(ctx, "kubectl", args...) - out, _ := cmd.Output() - - applyArgs := []string{"apply", "-f", "-"} - if kubeContext != "" { - applyArgs = append([]string{"--context", kubeContext}, applyArgs...) - } - applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) - applyCmd.Stdin = strings.NewReader(string(out)) - applyCmd.Run() -} - -func cleanupTestNamespaces(ctx context.Context, kubeContext string) { - log.Println("Cleaning up test resources...") - - // Main namespace + S3 extra namespaces - namespaces := []string{testNamespace} - for i := 0; i < 10; i++ { - namespaces = append(namespaces, fmt.Sprintf("%s-%d", testNamespace, i)) - } - - for _, ns := range namespaces { - args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - exec.CommandContext(ctx, "kubectl", args...).Run() - } - - // Wait a bit for cleanup - time.Sleep(2 * time.Second) - - // Force delete remaining pods - for _, ns := range namespaces { - args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - exec.CommandContext(ctx, "kubectl", args...).Run() - } -} - -func cleanupReloader(ctx context.Context, version string, kubeContext string) { - ns := fmt.Sprintf("reloader-%s", version) - - nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - - if kubeContext != "" { - nsArgs = append([]string{"--context", kubeContext}, nsArgs...) - crArgs = append([]string{"--context", kubeContext}, crArgs...) - crbArgs = append([]string{"--context", kubeContext}, crbArgs...) - } - - exec.CommandContext(ctx, "kubectl", nsArgs...).Run() - exec.CommandContext(ctx, "kubectl", crArgs...).Run() - exec.CommandContext(ctx, "kubectl", crbArgs...).Run() -} - -// ============================================================================ -// REPORT COMMAND -// ============================================================================ - -func reportCommand(args []string) { - var scenarioID, resultsDir, outputFile string - format := OutputFormatText - resultsDir = "./results" - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--scenario="): - scenarioID = strings.TrimPrefix(arg, "--scenario=") - case strings.HasPrefix(arg, "--results-dir="): - resultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--output="): - outputFile = strings.TrimPrefix(arg, "--output=") - case strings.HasPrefix(arg, "--format="): - format = OutputFormat(strings.TrimPrefix(arg, "--format=")) - } - } - - if scenarioID == "" { - log.Fatal("--scenario is required for report command") - } - - report, err := generateScenarioReport(scenarioID, resultsDir) - if err != nil { - log.Fatalf("Failed to generate report: %v", err) - } - - var output string - switch format { - case OutputFormatJSON: - output = renderScenarioReportJSON(report) - case OutputFormatMarkdown: - output = renderScenarioReportMarkdown(report) - default: - output = renderScenarioReport(report) - } - - if outputFile != "" { - if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { - log.Fatalf("Failed to write output file: %v", err) - } - log.Printf("Report written to %s", outputFile) - } else { - fmt.Println(output) - } -} - -// PrometheusResponse represents a Prometheus API response for report parsing. -type PrometheusResponse struct { - Status string `json:"status"` - Data struct { - ResultType string `json:"resultType"` - Result []struct { - Metric map[string]string `json:"metric"` - Value []interface{} `json:"value"` - } `json:"result"` - } `json:"data"` -} - -// MetricComparison represents the comparison of a single metric. -type MetricComparison struct { - Name string - DisplayName string - Unit string - IsCounter bool - OldValue float64 - NewValue float64 - Expected float64 - Difference float64 - DiffPct float64 - Status string - Threshold float64 - OldMeetsExpected string - NewMeetsExpected string -} - -type metricInfo struct { - unit string - isCounter bool -} - -var metricInfoMap = map[string]metricInfo{ - "reconcile_total": {unit: "count", isCounter: true}, - "reconcile_duration_p50": {unit: "s", isCounter: false}, - "reconcile_duration_p95": {unit: "s", isCounter: false}, - "reconcile_duration_p99": {unit: "s", isCounter: false}, - "action_total": {unit: "count", isCounter: true}, - "action_latency_p50": {unit: "s", isCounter: false}, - "action_latency_p95": {unit: "s", isCounter: false}, - "action_latency_p99": {unit: "s", isCounter: false}, - "errors_total": {unit: "count", isCounter: true}, - "reload_executed_total": {unit: "count", isCounter: true}, - "workloads_scanned_total": {unit: "count", isCounter: true}, - "workloads_matched_total": {unit: "count", isCounter: true}, - "skipped_total_no_data_change": {unit: "count", isCounter: true}, - "rest_client_requests_total": {unit: "count", isCounter: true}, - "rest_client_requests_get": {unit: "count", isCounter: true}, - "rest_client_requests_patch": {unit: "count", isCounter: true}, - "rest_client_requests_put": {unit: "count", isCounter: true}, - "rest_client_requests_errors": {unit: "count", isCounter: true}, - - // Resource consumption metrics (gauges, not counters) - "memory_rss_mb_avg": {unit: "MB", isCounter: false}, - "memory_rss_mb_max": {unit: "MB", isCounter: false}, - "memory_heap_mb_avg": {unit: "MB", isCounter: false}, - "memory_heap_mb_max": {unit: "MB", isCounter: false}, - "cpu_cores_avg": {unit: "cores", isCounter: false}, - "cpu_cores_max": {unit: "cores", isCounter: false}, - "goroutines_avg": {unit: "count", isCounter: false}, - "goroutines_max": {unit: "count", isCounter: false}, - "gc_pause_p99_ms": {unit: "ms", isCounter: false}, -} - -// ReportExpectedMetrics matches the expected metrics from test scenarios. -type ReportExpectedMetrics struct { - ActionTotal int `json:"action_total"` - ReloadExecutedTotal int `json:"reload_executed_total"` - ReconcileTotal int `json:"reconcile_total"` - WorkloadsScannedTotal int `json:"workloads_scanned_total"` - WorkloadsMatchedTotal int `json:"workloads_matched_total"` - SkippedTotal int `json:"skipped_total"` - Description string `json:"description"` -} - -// ScenarioReport represents the full report for a scenario. -type ScenarioReport struct { - Scenario string - Timestamp time.Time - Comparisons []MetricComparison - OverallStatus string - Summary string - PassCriteria []string - FailedCriteria []string - Expected ReportExpectedMetrics - TestDescription string -} - -// MetricType defines how to evaluate a metric. -type MetricType int - -const ( - LowerIsBetter MetricType = iota - ShouldMatch - HigherIsBetter - Informational // Reports values but doesn't affect pass/fail -) - -type ThresholdConfig struct { - maxDiff float64 - metricType MetricType - minAbsDiff float64 -} - -var thresholds = map[string]ThresholdConfig{ - "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, - "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, - "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, - "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, - "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, - // API metrics - use minAbsDiff to allow small differences - "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, - "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, // Pass if both < 100 - - // Resource consumption metrics - "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB - "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, // 50% or 30MB - "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, // 50% or 15MB - "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB - "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, // 100% or 0.1 cores - "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, // 100% or 0.2 cores - "goroutines_avg": {metricType: Informational}, // Info only - different architectures may use more goroutines - "goroutines_max": {metricType: Informational}, // Info only - different architectures may use more goroutines - "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, // 100% or 5ms -} - -func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { - oldDir := filepath.Join(resultsDir, scenario, "old") - newDir := filepath.Join(resultsDir, scenario, "new") - scenarioDir := filepath.Join(resultsDir, scenario) - - // Check which directories exist to determine mode - _, oldErr := os.Stat(oldDir) - _, newErr := os.Stat(newDir) - hasOld := oldErr == nil - hasNew := newErr == nil - isComparison := hasOld && hasNew - - // For single-version mode, determine which version we have - singleVersion := "" - singleDir := "" - if !isComparison { - if hasNew { - singleVersion = "new" - singleDir = newDir - } else if hasOld { - singleVersion = "old" - singleDir = oldDir - } else { - return nil, fmt.Errorf("no results found in %s", scenarioDir) - } - } - - report := &ScenarioReport{ - Scenario: scenario, - Timestamp: time.Now(), - } - - // Load expected metrics - expectedPath := filepath.Join(scenarioDir, "expected.json") - if data, err := os.ReadFile(expectedPath); err == nil { - if err := json.Unmarshal(data, &report.Expected); err != nil { - log.Printf("Warning: Could not parse expected metrics: %v", err) - } else { - report.TestDescription = report.Expected.Description - } - } - - // Handle single-version mode - if !isComparison { - return generateSingleVersionReport(report, singleDir, singleVersion, scenario) - } - - // Define metrics to compare - metricsToCompare := []struct { - name string - file string - selector func(data PrometheusResponse) float64 - }{ - {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, - {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, - {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, - {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, - {"action_total", "reloader_action_total.json", sumAllValues}, - {"action_latency_p50", "action_p50.json", getFirstValue}, - {"action_latency_p95", "action_p95.json", getFirstValue}, - {"action_latency_p99", "action_p99.json", getFirstValue}, - {"errors_total", "reloader_errors_total.json", sumAllValues}, - {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, - {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, - {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, - {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, - {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, - {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, - {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, - {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, - - // Resource consumption metrics - {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, - {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, - {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, - {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, - {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, - {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, - {"goroutines_avg", "goroutines_avg.json", getFirstValue}, - {"goroutines_max", "goroutines_max.json", getFirstValue}, - {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, - } - - // Build expected values map - expectedValues := map[string]float64{ - "action_total": float64(report.Expected.ActionTotal), - "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), - "reconcile_total": float64(report.Expected.ReconcileTotal), - "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), - "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), - "skipped_total": float64(report.Expected.SkippedTotal), - } - - // First pass: collect all metric values - metricValues := make(map[string]struct{ old, new, expected float64 }) - - for _, m := range metricsToCompare { - oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load old metric %s: %v", m.name, err) - continue - } - - newData, err := loadMetricFile(filepath.Join(newDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load new metric %s: %v", m.name, err) - continue - } - - oldValue := m.selector(oldData) - newValue := m.selector(newData) - expected := expectedValues[m.name] - - metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} - } - - // Check context for smart pass/fail decisions - newMeetsActionExpected := false - newReconcileIsZero := false - isChurnScenario := scenario == "S5" // Workload churn has special pass/fail rules - if v, ok := metricValues["action_total"]; ok && v.expected > 0 { - tolerance := v.expected * 0.15 - newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance - } - if v, ok := metricValues["reconcile_total"]; ok { - newReconcileIsZero = v.new == 0 - } - - // Second pass: generate comparisons with context awareness - for _, m := range metricsToCompare { - v, ok := metricValues[m.name] - if !ok { - continue - } - - comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) - - // Context-aware adjustments for API metrics - if strings.HasPrefix(m.name, "rest_client_requests") { - // If new correctly processed all expected reloads but old didn't, - // higher API calls in new is expected (it's doing the work correctly) - if newMeetsActionExpected && comparison.Status != "pass" { - if oldMeets, ok := metricValues["action_total"]; ok { - oldTolerance := oldMeets.expected * 0.15 - oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance - if oldMissed { - comparison.Status = "pass" - } - } - } - // If new has 0 reconciles (no-op scenario), API differences are fine - if newReconcileIsZero && comparison.Status != "pass" { - comparison.Status = "pass" - } - } - - // S5 (Workload Churn) specific adjustments: - // - "Not found" errors are expected when deployments are deleted during processing - // - No expected values for throughput, so compare old vs new (should be similar) - if isChurnScenario { - if m.name == "errors_total" { - // In churn scenarios, "not found" errors are expected when workloads - // are deleted while Reloader is processing them. Allow up to 50 errors. - if v.new < 50 && v.old < 50 { - comparison.Status = "pass" - } else if v.new <= v.old*1.5 { - // Also pass if new has similar or fewer errors than old - comparison.Status = "pass" - } - } - if m.name == "action_total" || m.name == "reload_executed_total" { - // No expected value for churn - compare old vs new - // Both should be similar (within 20% of each other) - if v.old > 0 { - diff := math.Abs(v.new-v.old) / v.old * 100 - if diff <= 20 { - comparison.Status = "pass" - } - } else if v.new > 0 { - // Old is 0, new has value - that's fine - comparison.Status = "pass" - } - } - } - - report.Comparisons = append(report.Comparisons, comparison) - - if comparison.Status == "pass" { - report.PassCriteria = append(report.PassCriteria, m.name) - } else if comparison.Status == "fail" { - report.FailedCriteria = append(report.FailedCriteria, m.name) - } - } - - // Determine overall status - if len(report.FailedCriteria) == 0 { - report.OverallStatus = "PASS" - report.Summary = "All metrics within acceptable thresholds" - } else { - report.OverallStatus = "FAIL" - report.Summary = fmt.Sprintf("%d metrics failed: %s", - len(report.FailedCriteria), - strings.Join(report.FailedCriteria, ", ")) - } - - return report, nil -} - -// generateSingleVersionReport creates a report for a single version (no comparison). -func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { - // Define metrics to collect - metricsToCollect := []struct { - name string - file string - selector func(data PrometheusResponse) float64 - }{ - {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, - {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, - {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, - {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, - {"action_total", "reloader_action_total.json", sumAllValues}, - {"action_latency_p50", "action_p50.json", getFirstValue}, - {"action_latency_p95", "action_p95.json", getFirstValue}, - {"action_latency_p99", "action_p99.json", getFirstValue}, - {"errors_total", "reloader_errors_total.json", sumAllValues}, - {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, - {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, - {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, - {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, - {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, - {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, - {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, - {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, - {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, - {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, - {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, - {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, - {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, - {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, - {"goroutines_avg", "goroutines_avg.json", getFirstValue}, - {"goroutines_max", "goroutines_max.json", getFirstValue}, - {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, - } - - // Build expected values map - expectedValues := map[string]float64{ - "action_total": float64(report.Expected.ActionTotal), - "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), - "reconcile_total": float64(report.Expected.ReconcileTotal), - "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), - "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), - "skipped_total": float64(report.Expected.SkippedTotal), - } - - for _, m := range metricsToCollect { - data, err := loadMetricFile(filepath.Join(dataDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load metric %s: %v", m.name, err) - continue - } - - value := m.selector(data) - expected := expectedValues[m.name] - - info := metricInfoMap[m.name] - if info.unit == "" { - info = metricInfo{unit: "count", isCounter: true} - } - - displayName := m.name - if info.unit != "count" { - displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) - } - - // For single-version, put the value in NewValue column - status := "info" - meetsExp := "-" - - // Check against expected if available - if expected > 0 { - meetsExp = meetsExpected(value, expected) - threshold, ok := thresholds[m.name] - if ok && threshold.metricType == ShouldMatch { - if meetsExp == "✓" { - status = "pass" - report.PassCriteria = append(report.PassCriteria, m.name) - } else { - status = "fail" - report.FailedCriteria = append(report.FailedCriteria, m.name) - } - } - } - - if info.isCounter { - value = math.Round(value) - } - - report.Comparisons = append(report.Comparisons, MetricComparison{ - Name: m.name, - DisplayName: displayName, - Unit: info.unit, - IsCounter: info.isCounter, - OldValue: 0, // No old value in single-version mode - NewValue: value, - Expected: expected, - OldMeetsExpected: "-", - NewMeetsExpected: meetsExp, - Status: status, - }) - } - - if len(report.FailedCriteria) == 0 { - report.OverallStatus = "PASS" - report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) - } else { - report.OverallStatus = "FAIL" - report.Summary = fmt.Sprintf("%d metrics failed: %s", - len(report.FailedCriteria), - strings.Join(report.FailedCriteria, ", ")) - } - - return report, nil -} - -func loadMetricFile(path string) (PrometheusResponse, error) { - var resp PrometheusResponse - data, err := os.ReadFile(path) - if err != nil { - return resp, err - } - err = json.Unmarshal(data, &resp) - return resp, err -} - -func sumAllValues(data PrometheusResponse) float64 { - var sum float64 - for _, result := range data.Data.Result { - if len(result.Value) >= 2 { - if v, ok := result.Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - sum += f - } - } - } - return sum -} - -func sumSuccessValues(data PrometheusResponse) float64 { - var sum float64 - for _, result := range data.Data.Result { - if result.Metric["success"] == "true" { - if len(result.Value) >= 2 { - if v, ok := result.Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - sum += f - } - } - } - } - return sum -} - -func getFirstValue(data PrometheusResponse) float64 { - if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { - if v, ok := data.Data.Result[0].Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - return f - } - } - return 0 -} - -// bytesToMB converts bytes to megabytes. -func bytesToMB(data PrometheusResponse) float64 { - bytes := getFirstValue(data) - return bytes / (1024 * 1024) -} - -// secondsToMs converts seconds to milliseconds. -func secondsToMs(data PrometheusResponse) float64 { - seconds := getFirstValue(data) - return seconds * 1000 -} - -func meetsExpected(value, expected float64) string { - if expected == 0 { - return "-" - } - tolerance := expected * 0.15 - if math.Abs(value-expected) <= tolerance { - return "✓" - } - return "✗" -} - -func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { - diff := newValue - oldValue - absDiff := math.Abs(diff) - var diffPct float64 - if oldValue != 0 { - diffPct = (diff / oldValue) * 100 - } else if newValue != 0 { - diffPct = 100 - } - - threshold, ok := thresholds[name] - if !ok { - threshold = ThresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} - } - - info := metricInfoMap[name] - if info.unit == "" { - info = metricInfo{unit: "count", isCounter: true} - } - displayName := name - if info.unit != "count" { - displayName = fmt.Sprintf("%s (%s)", name, info.unit) - } - - if info.isCounter { - oldValue = math.Round(oldValue) - newValue = math.Round(newValue) - } - - status := "pass" - oldMeetsExp := meetsExpected(oldValue, expected) - newMeetsExp := meetsExpected(newValue, expected) - - if expected > 0 && threshold.metricType == ShouldMatch { - if newMeetsExp == "✗" { - status = "fail" - } - } else { - switch threshold.metricType { - case LowerIsBetter: - if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { - status = "pass" - } else if diffPct > threshold.maxDiff { - status = "fail" - } - case HigherIsBetter: - if diffPct < -threshold.maxDiff { - status = "fail" - } - case ShouldMatch: - if math.Abs(diffPct) > threshold.maxDiff { - status = "fail" - } - case Informational: - status = "info" - } - } - - return MetricComparison{ - Name: name, - DisplayName: displayName, - Unit: info.unit, - IsCounter: info.isCounter, - Expected: expected, - OldMeetsExpected: oldMeetsExp, - NewMeetsExpected: newMeetsExp, - OldValue: oldValue, - NewValue: newValue, - Difference: diff, - DiffPct: diffPct, - Status: status, - Threshold: threshold.maxDiff, - } -} - -func renderScenarioReport(report *ScenarioReport) string { - var sb strings.Builder - - // Detect single-version mode by checking if all OldValues are 0 - isSingleVersion := true - for _, c := range report.Comparisons { - if c.OldValue != 0 { - isSingleVersion = false - break - } - } - - sb.WriteString("\n") - sb.WriteString("================================================================================\n") - if isSingleVersion { - sb.WriteString(" RELOADER TEST REPORT\n") - } else { - sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") - } - sb.WriteString("================================================================================\n\n") - - fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) - fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) - fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) - fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) - - if report.TestDescription != "" { - fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) - } - - if report.Expected.ActionTotal > 0 { - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" EXPECTED VALUES\n") - sb.WriteString("--------------------------------------------------------------------------------\n") - fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) - fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) - if report.Expected.SkippedTotal > 0 { - fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - if isSingleVersion { - sb.WriteString(" METRICS\n") - } else { - sb.WriteString(" METRIC COMPARISONS\n") - } - sb.WriteString("--------------------------------------------------------------------------------\n") - - if isSingleVersion { - sb.WriteString("(✓ = meets expected value within 15%)\n\n") - fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", - "Metric", "Value", "Expected", "Met?", "Status") - fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", - "------", "-----", "--------", "----", "------") - - for _, c := range report.Comparisons { - if c.IsCounter { - if c.Expected > 0 { - fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", - c.DisplayName, c.NewValue, c.Expected, - c.NewMeetsExpected, c.Status) - } else { - fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", - c.DisplayName, c.NewValue, "-", - c.NewMeetsExpected, c.Status) - } - } else { - fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", - c.DisplayName, c.NewValue, "-", - c.NewMeetsExpected, c.Status) - } - } - } else { - sb.WriteString("(Old✓/New✓ = meets expected value within 15%)\n\n") - - fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", - "Metric", "Old", "New", "Expected", "Old✓", "New✓", "Status") - fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", - "------", "---", "---", "--------", "----", "----", "------") - - for _, c := range report.Comparisons { - if c.IsCounter { - if c.Expected > 0 { - fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, c.Expected, - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } else { - fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, "-", - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } - } else { - fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, "-", - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" PASS/FAIL CRITERIA\n") - sb.WriteString("--------------------------------------------------------------------------------\n\n") - - fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) - for _, p := range report.PassCriteria { - fmt.Fprintf(&sb, " ✓ %s\n", p) - } - - if len(report.FailedCriteria) > 0 { - fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) - for _, f := range report.FailedCriteria { - fmt.Fprintf(&sb, " ✗ %s\n", f) - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" THRESHOLDS USED\n") - sb.WriteString("--------------------------------------------------------------------------------\n\n") - - fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", - "Metric", "Max Diff%", "Min Abs Diff", "Direction") - fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", - "------", "---------", "------------", "---------") - - // Sort threshold names - var names []string - for name := range thresholds { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - t := thresholds[name] - var direction string - switch t.metricType { - case LowerIsBetter: - direction = "lower is better" - case HigherIsBetter: - direction = "higher is better" - case ShouldMatch: - direction = "should match" - case Informational: - direction = "info only" - } - minAbsDiff := "-" - if t.minAbsDiff > 0 { - minAbsDiff = fmt.Sprintf("%.1fs", t.minAbsDiff) - } - fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", - name, t.maxDiff, minAbsDiff, direction) - } - - sb.WriteString("\n================================================================================\n") - - return sb.String() -} - -// renderScenarioReportJSON renders a scenario report as JSON. -func renderScenarioReportJSON(report *ScenarioReport) string { - data, err := json.MarshalIndent(report, "", " ") - if err != nil { - return fmt.Sprintf(`{"error": "%s"}`, err.Error()) - } - return string(data) -} - -// renderScenarioReportMarkdown renders a scenario report as concise markdown. -func renderScenarioReportMarkdown(report *ScenarioReport) string { - var sb strings.Builder - - // Status emoji - emoji := "✅" - if report.OverallStatus != "PASS" { - emoji = "❌" - } - - sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) - - if report.TestDescription != "" { - sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) - } - - // Key metrics table - sb.WriteString("| Metric | Value | Expected | Status |\n") - sb.WriteString("|--------|------:|:--------:|:------:|\n") - - // Show only key metrics - keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} - for _, name := range keyMetrics { - for _, c := range report.Comparisons { - if c.Name == name { - value := fmt.Sprintf("%.0f", c.NewValue) - expected := "-" - if c.Expected > 0 { - expected = fmt.Sprintf("%.0f", c.Expected) - } - status := "✅" - if c.Status == "fail" { - status = "❌" - } else if c.Status == "info" { - status = "ℹ️" - } - sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) - break - } - } - } - - return sb.String() -} - -// ============================================================================ -// SUMMARY COMMAND -// ============================================================================ - -// SummaryReport aggregates results from multiple scenarios. -type SummaryReport struct { - Timestamp time.Time `json:"timestamp"` - TestType string `json:"test_type"` - PassCount int `json:"pass_count"` - FailCount int `json:"fail_count"` - TotalCount int `json:"total_count"` - Scenarios []ScenarioSummary `json:"scenarios"` -} - -// ScenarioSummary provides a brief summary of a single scenario. -type ScenarioSummary struct { - ID string `json:"id"` - Status string `json:"status"` - Description string `json:"description"` - ActionTotal float64 `json:"action_total"` - ActionExp float64 `json:"action_expected"` - ErrorsTotal float64 `json:"errors_total"` -} - -func summaryCommand(args []string) { - var resultsDir, outputFile, testType string - format := OutputFormatMarkdown // Default to markdown for CI - resultsDir = "./results" - testType = "full" - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--results-dir="): - resultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--output="): - outputFile = strings.TrimPrefix(arg, "--output=") - case strings.HasPrefix(arg, "--format="): - format = OutputFormat(strings.TrimPrefix(arg, "--format=")) - case strings.HasPrefix(arg, "--test-type="): - testType = strings.TrimPrefix(arg, "--test-type=") - } - } - - summary, err := generateSummaryReport(resultsDir, testType) - if err != nil { - log.Fatalf("Failed to generate summary: %v", err) - } - - var output string - switch format { - case OutputFormatJSON: - output = renderSummaryJSON(summary) - case OutputFormatText: - output = renderSummaryText(summary) - default: - output = renderSummaryMarkdown(summary) - } - - if outputFile != "" { - if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { - log.Fatalf("Failed to write output file: %v", err) - } - log.Printf("Summary written to %s", outputFile) - } else { - fmt.Print(output) - } - - // Exit with non-zero status if any tests failed - if summary.FailCount > 0 { - os.Exit(1) - } -} - -func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { - summary := &SummaryReport{ - Timestamp: time.Now(), - TestType: testType, - } - - // Find all scenario directories - entries, err := os.ReadDir(resultsDir) - if err != nil { - return nil, fmt.Errorf("failed to read results directory: %w", err) - } - - for _, entry := range entries { - if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { - continue - } - - scenarioID := entry.Name() - report, err := generateScenarioReport(scenarioID, resultsDir) - if err != nil { - log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) - continue - } - - scenarioSummary := ScenarioSummary{ - ID: scenarioID, - Status: report.OverallStatus, - Description: report.TestDescription, - } - - // Extract key metrics - for _, c := range report.Comparisons { - switch c.Name { - case "action_total": - scenarioSummary.ActionTotal = c.NewValue - scenarioSummary.ActionExp = c.Expected - case "errors_total": - scenarioSummary.ErrorsTotal = c.NewValue - } - } - - summary.Scenarios = append(summary.Scenarios, scenarioSummary) - summary.TotalCount++ - if report.OverallStatus == "PASS" { - summary.PassCount++ - } else { - summary.FailCount++ - } - } - - // Sort scenarios by ID - sort.Slice(summary.Scenarios, func(i, j int) bool { - return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) - }) - - return summary, nil -} - -// naturalSort compares two scenario IDs (S1, S2, ..., S10, S11) -func naturalSort(a, b string) bool { - var aNum, bNum int - fmt.Sscanf(a, "S%d", &aNum) - fmt.Sscanf(b, "S%d", &bNum) - return aNum < bNum -} - -func renderSummaryJSON(summary *SummaryReport) string { - data, err := json.MarshalIndent(summary, "", " ") - if err != nil { - return fmt.Sprintf(`{"error": "%s"}`, err.Error()) - } - return string(data) -} - -func renderSummaryText(summary *SummaryReport) string { - var sb strings.Builder - - sb.WriteString("================================================================================\n") - sb.WriteString(" LOAD TEST SUMMARY\n") - sb.WriteString("================================================================================\n\n") - - passRate := 0 - if summary.TotalCount > 0 { - passRate = summary.PassCount * 100 / summary.TotalCount - } - - fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) - fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) - - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") - - for _, s := range summary.Scenarios { - desc := s.Description - if len(desc) > 45 { - desc = desc[:42] + "..." - } - actions := fmt.Sprintf("%.0f", s.ActionTotal) - if s.ActionExp > 0 { - actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) - } - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) - } - - sb.WriteString("\n================================================================================\n") - return sb.String() -} - -func renderSummaryMarkdown(summary *SummaryReport) string { - var sb strings.Builder - - // Overall status - emoji := "✅" - title := "ALL TESTS PASSED" - if summary.FailCount > 0 { - emoji = "❌" - title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) - } else if summary.TotalCount == 0 { - emoji = "⚠️" - title = "NO RESULTS" - } - - sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) - - // Test type note - if summary.TestType == "quick" { - sb.WriteString("> 🚀 **Quick Test** (S1, S4, S6) — Use `/loadtest` for full suite\n\n") - } - - // Pass rate - passRate := 0 - if summary.TotalCount > 0 { - passRate = summary.PassCount * 100 / summary.TotalCount - } - sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) - - // Results table - sb.WriteString("| | Scenario | Description | Actions | Errors |\n") - sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") - - for _, s := range summary.Scenarios { - icon := "✅" - if s.Status != "PASS" { - icon = "❌" - } - - // Truncate description - desc := s.Description - if len(desc) > 45 { - desc = desc[:42] + "..." - } - - // Format actions - actions := fmt.Sprintf("%.0f", s.ActionTotal) - if s.ActionExp > 0 { - actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) - } - - // Format errors - errors := fmt.Sprintf("%.0f", s.ErrorsTotal) - if s.ErrorsTotal > 0 { - errors = fmt.Sprintf("⚠️ %.0f", s.ErrorsTotal) - } - - sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) - } - - sb.WriteString("\n📦 **[Download detailed results](../artifacts)**\n") - - return sb.String() + cmd.Execute() } diff --git a/test/loadtest/go.mod b/test/loadtest/go.mod index ed5288210..e96ed763c 100644 --- a/test/loadtest/go.mod +++ b/test/loadtest/go.mod @@ -1,8 +1,9 @@ module github.com/stakater/Reloader/test/loadtest -go 1.22.0 +go 1.25 require ( + github.com/spf13/cobra v1.8.1 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 k8s.io/client-go v0.31.0 @@ -23,6 +24,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/test/loadtest/go.sum b/test/loadtest/go.sum index a8edbda5a..f4f0ad8d6 100644 --- a/test/loadtest/go.sum +++ b/test/loadtest/go.sum @@ -1,3 +1,4 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,6 +37,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -67,6 +70,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/test/loadtest/internal/cmd/report.go b/test/loadtest/internal/cmd/report.go new file mode 100644 index 000000000..aa273a517 --- /dev/null +++ b/test/loadtest/internal/cmd/report.go @@ -0,0 +1,856 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + reportScenario string + reportResultsDir string + reportOutputFile string + reportFormat string +) + +var reportCmd = &cobra.Command{ + Use: "report", + Short: "Generate comparison report for a scenario", + Long: `Generate a detailed report for a specific test scenario. + +Examples: + # Generate report for a scenario + loadtest report --scenario=S2 --results-dir=./results + + # Generate JSON report + loadtest report --scenario=S2 --format=json`, + Run: func(cmd *cobra.Command, args []string) { + reportCommand() + }, +} + +func init() { + reportCmd.Flags().StringVar(&reportScenario, "scenario", "", "Scenario to report on (required)") + reportCmd.Flags().StringVar(&reportResultsDir, "results-dir", "./results", "Directory containing results") + reportCmd.Flags().StringVar(&reportOutputFile, "output", "", "Output file (default: stdout)") + reportCmd.Flags().StringVar(&reportFormat, "format", "text", "Output format: text, json, markdown") + reportCmd.MarkFlagRequired("scenario") +} + +// PrometheusResponse represents a Prometheus API response for report parsing. +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// MetricComparison represents the comparison of a single metric. +type MetricComparison struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + Unit string `json:"unit"` + IsCounter bool `json:"is_counter"` + OldValue float64 `json:"old_value"` + NewValue float64 `json:"new_value"` + Expected float64 `json:"expected"` + Difference float64 `json:"difference"` + DiffPct float64 `json:"diff_pct"` + Status string `json:"status"` + Threshold float64 `json:"threshold"` + OldMeetsExpected string `json:"old_meets_expected"` + NewMeetsExpected string `json:"new_meets_expected"` +} + +type metricInfo struct { + unit string + isCounter bool +} + +var metricInfoMap = map[string]metricInfo{ + "reconcile_total": {unit: "count", isCounter: true}, + "reconcile_duration_p50": {unit: "s", isCounter: false}, + "reconcile_duration_p95": {unit: "s", isCounter: false}, + "reconcile_duration_p99": {unit: "s", isCounter: false}, + "action_total": {unit: "count", isCounter: true}, + "action_latency_p50": {unit: "s", isCounter: false}, + "action_latency_p95": {unit: "s", isCounter: false}, + "action_latency_p99": {unit: "s", isCounter: false}, + "errors_total": {unit: "count", isCounter: true}, + "reload_executed_total": {unit: "count", isCounter: true}, + "workloads_scanned_total": {unit: "count", isCounter: true}, + "workloads_matched_total": {unit: "count", isCounter: true}, + "skipped_total_no_data_change": {unit: "count", isCounter: true}, + "rest_client_requests_total": {unit: "count", isCounter: true}, + "rest_client_requests_get": {unit: "count", isCounter: true}, + "rest_client_requests_patch": {unit: "count", isCounter: true}, + "rest_client_requests_put": {unit: "count", isCounter: true}, + "rest_client_requests_errors": {unit: "count", isCounter: true}, + "memory_rss_mb_avg": {unit: "MB", isCounter: false}, + "memory_rss_mb_max": {unit: "MB", isCounter: false}, + "memory_heap_mb_avg": {unit: "MB", isCounter: false}, + "memory_heap_mb_max": {unit: "MB", isCounter: false}, + "cpu_cores_avg": {unit: "cores", isCounter: false}, + "cpu_cores_max": {unit: "cores", isCounter: false}, + "goroutines_avg": {unit: "count", isCounter: false}, + "goroutines_max": {unit: "count", isCounter: false}, + "gc_pause_p99_ms": {unit: "ms", isCounter: false}, +} + +// ReportExpectedMetrics matches the expected metrics from test scenarios. +type ReportExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// ScenarioReport represents the full report for a scenario. +type ScenarioReport struct { + Scenario string `json:"scenario"` + Timestamp time.Time `json:"timestamp"` + Comparisons []MetricComparison `json:"comparisons"` + OverallStatus string `json:"overall_status"` + Summary string `json:"summary"` + PassCriteria []string `json:"pass_criteria"` + FailedCriteria []string `json:"failed_criteria"` + Expected ReportExpectedMetrics `json:"expected"` + TestDescription string `json:"test_description"` +} + +// MetricType defines how to evaluate a metric. +type MetricType int + +const ( + LowerIsBetter MetricType = iota + ShouldMatch + HigherIsBetter + Informational +) + +type thresholdConfig struct { + maxDiff float64 + metricType MetricType + minAbsDiff float64 +} + +var thresholds = map[string]thresholdConfig{ + "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, + "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, + "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, + "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, + "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, + "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, + "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, + "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, + "goroutines_avg": {metricType: Informational}, + "goroutines_max": {metricType: Informational}, + "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, +} + +func reportCommand() { + if reportScenario == "" { + log.Fatal("--scenario is required for report command") + } + + report, err := generateScenarioReport(reportScenario, reportResultsDir) + if err != nil { + log.Fatalf("Failed to generate report: %v", err) + } + + var output string + switch OutputFormat(reportFormat) { + case OutputFormatJSON: + output = renderScenarioReportJSON(report) + case OutputFormatMarkdown: + output = renderScenarioReportMarkdown(report) + default: + output = renderScenarioReport(report) + } + + if reportOutputFile != "" { + if err := os.WriteFile(reportOutputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Report written to %s", reportOutputFile) + } else { + fmt.Println(output) + } +} + +func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { + oldDir := filepath.Join(resultsDir, scenario, "old") + newDir := filepath.Join(resultsDir, scenario, "new") + scenarioDir := filepath.Join(resultsDir, scenario) + + _, oldErr := os.Stat(oldDir) + _, newErr := os.Stat(newDir) + hasOld := oldErr == nil + hasNew := newErr == nil + isComparison := hasOld && hasNew + + singleVersion := "" + singleDir := "" + if !isComparison { + if hasNew { + singleVersion = "new" + singleDir = newDir + } else if hasOld { + singleVersion = "old" + singleDir = oldDir + } else { + return nil, fmt.Errorf("no results found in %s", scenarioDir) + } + } + + report := &ScenarioReport{ + Scenario: scenario, + Timestamp: time.Now(), + } + + expectedPath := filepath.Join(scenarioDir, "expected.json") + if data, err := os.ReadFile(expectedPath); err == nil { + if err := json.Unmarshal(data, &report.Expected); err != nil { + log.Printf("Warning: Could not parse expected metrics: %v", err) + } else { + report.TestDescription = report.Expected.Description + } + } + + if !isComparison { + return generateSingleVersionReport(report, singleDir, singleVersion, scenario) + } + + metricsToCompare := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + metricValues := make(map[string]struct{ old, new, expected float64 }) + + for _, m := range metricsToCompare { + oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load old metric %s: %v", m.name, err) + continue + } + + newData, err := loadMetricFile(filepath.Join(newDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load new metric %s: %v", m.name, err) + continue + } + + oldValue := m.selector(oldData) + newValue := m.selector(newData) + expected := expectedValues[m.name] + + metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} + } + + newMeetsActionExpected := false + newReconcileIsZero := false + isChurnScenario := scenario == "S5" + if v, ok := metricValues["action_total"]; ok && v.expected > 0 { + tolerance := v.expected * 0.15 + newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance + } + if v, ok := metricValues["reconcile_total"]; ok { + newReconcileIsZero = v.new == 0 + } + + for _, m := range metricsToCompare { + v, ok := metricValues[m.name] + if !ok { + continue + } + + comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) + + if strings.HasPrefix(m.name, "rest_client_requests") { + if newMeetsActionExpected && comparison.Status != "pass" { + if oldMeets, ok := metricValues["action_total"]; ok { + oldTolerance := oldMeets.expected * 0.15 + oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance + if oldMissed { + comparison.Status = "pass" + } + } + } + if newReconcileIsZero && comparison.Status != "pass" { + comparison.Status = "pass" + } + } + + if isChurnScenario { + if m.name == "errors_total" { + if v.new < 50 && v.old < 50 { + comparison.Status = "pass" + } else if v.new <= v.old*1.5 { + comparison.Status = "pass" + } + } + if m.name == "action_total" || m.name == "reload_executed_total" { + if v.old > 0 { + diff := math.Abs(v.new-v.old) / v.old * 100 + if diff <= 20 { + comparison.Status = "pass" + } + } else if v.new > 0 { + comparison.Status = "pass" + } + } + } + + report.Comparisons = append(report.Comparisons, comparison) + + if comparison.Status == "pass" { + report.PassCriteria = append(report.PassCriteria, m.name) + } else if comparison.Status == "fail" { + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = "All metrics within acceptable thresholds" + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { + metricsToCollect := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + for _, m := range metricsToCollect { + data, err := loadMetricFile(filepath.Join(dataDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load metric %s: %v", m.name, err) + continue + } + + value := m.selector(data) + expected := expectedValues[m.name] + + info := metricInfoMap[m.name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + + displayName := m.name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) + } + + status := "info" + meetsExp := "-" + + if expected > 0 { + meetsExp = meetsExpected(value, expected) + threshold, ok := thresholds[m.name] + if ok && threshold.metricType == ShouldMatch { + if meetsExp == "✓" { + status = "pass" + report.PassCriteria = append(report.PassCriteria, m.name) + } else { + status = "fail" + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + } + + if info.isCounter { + value = math.Round(value) + } + + report.Comparisons = append(report.Comparisons, MetricComparison{ + Name: m.name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + OldValue: 0, + NewValue: value, + Expected: expected, + OldMeetsExpected: "-", + NewMeetsExpected: meetsExp, + Status: status, + }) + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func loadMetricFile(path string) (PrometheusResponse, error) { + var resp PrometheusResponse + data, err := os.ReadFile(path) + if err != nil { + return resp, err + } + err = json.Unmarshal(data, &resp) + return resp, err +} + +func sumAllValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + return sum +} + +func sumSuccessValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if result.Metric["success"] == "true" { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + } + return sum +} + +func getFirstValue(data PrometheusResponse) float64 { + if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { + if v, ok := data.Data.Result[0].Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + return f + } + } + return 0 +} + +func bytesToMB(data PrometheusResponse) float64 { + bytes := getFirstValue(data) + return bytes / (1024 * 1024) +} + +func secondsToMs(data PrometheusResponse) float64 { + seconds := getFirstValue(data) + return seconds * 1000 +} + +func meetsExpected(value, expected float64) string { + if expected == 0 { + return "-" + } + tolerance := expected * 0.15 + if math.Abs(value-expected) <= tolerance { + return "✓" + } + return "✗" +} + +func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { + diff := newValue - oldValue + absDiff := math.Abs(diff) + var diffPct float64 + if oldValue != 0 { + diffPct = (diff / oldValue) * 100 + } else if newValue != 0 { + diffPct = 100 + } + + threshold, ok := thresholds[name] + if !ok { + threshold = thresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} + } + + info := metricInfoMap[name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + displayName := name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", name, info.unit) + } + + if info.isCounter { + oldValue = math.Round(oldValue) + newValue = math.Round(newValue) + } + + status := "pass" + oldMeetsExp := meetsExpected(oldValue, expected) + newMeetsExp := meetsExpected(newValue, expected) + + if expected > 0 && threshold.metricType == ShouldMatch { + if newMeetsExp == "✗" { + status = "fail" + } + } else { + switch threshold.metricType { + case LowerIsBetter: + if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { + status = "pass" + } else if diffPct > threshold.maxDiff { + status = "fail" + } + case HigherIsBetter: + if diffPct < -threshold.maxDiff { + status = "fail" + } + case ShouldMatch: + if math.Abs(diffPct) > threshold.maxDiff { + status = "fail" + } + case Informational: + status = "info" + } + } + + return MetricComparison{ + Name: name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + Expected: expected, + OldMeetsExpected: oldMeetsExp, + NewMeetsExpected: newMeetsExp, + OldValue: oldValue, + NewValue: newValue, + Difference: diff, + DiffPct: diffPct, + Status: status, + Threshold: threshold.maxDiff, + } +} + +func renderScenarioReport(report *ScenarioReport) string { + var sb strings.Builder + + isSingleVersion := true + for _, c := range report.Comparisons { + if c.OldValue != 0 { + isSingleVersion = false + break + } + } + + sb.WriteString("\n") + sb.WriteString("================================================================================\n") + if isSingleVersion { + sb.WriteString(" RELOADER TEST REPORT\n") + } else { + sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") + } + sb.WriteString("================================================================================\n\n") + + fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) + fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) + fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) + + if report.TestDescription != "" { + fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) + } + + if report.Expected.ActionTotal > 0 { + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" EXPECTED VALUES\n") + sb.WriteString("--------------------------------------------------------------------------------\n") + fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) + fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) + if report.Expected.SkippedTotal > 0 { + fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + if isSingleVersion { + sb.WriteString(" METRICS\n") + } else { + sb.WriteString(" METRIC COMPARISONS\n") + } + sb.WriteString("--------------------------------------------------------------------------------\n") + + if isSingleVersion { + sb.WriteString("(✓ = meets expected value within 15%)\n\n") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "Metric", "Value", "Expected", "Met?", "Status") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "------", "-----", "--------", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", + c.DisplayName, c.NewValue, c.Expected, + c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } + } else { + sb.WriteString("(Old✓/New✓ = meets expected value within 15%)\n\n") + + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "Metric", "Old", "New", "Expected", "Old✓", "New✓", "Status") + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "------", "---", "---", "--------", "----", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, c.Expected, + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" PASS/FAIL CRITERIA\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) + for _, p := range report.PassCriteria { + fmt.Fprintf(&sb, " ✓ %s\n", p) + } + + if len(report.FailedCriteria) > 0 { + fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) + for _, f := range report.FailedCriteria { + fmt.Fprintf(&sb, " ✗ %s\n", f) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" THRESHOLDS USED\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "Metric", "Max Diff%", "Min Abs Diff", "Direction") + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "------", "---------", "------------", "---------") + + var names []string + for name := range thresholds { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + t := thresholds[name] + var direction string + switch t.metricType { + case LowerIsBetter: + direction = "lower is better" + case HigherIsBetter: + direction = "higher is better" + case ShouldMatch: + direction = "should match" + case Informational: + direction = "info only" + } + minAbsDiff := "-" + if t.minAbsDiff > 0 { + minAbsDiff = fmt.Sprintf("%.1f", t.minAbsDiff) + } + fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", + name, t.maxDiff, minAbsDiff, direction) + } + + sb.WriteString("\n================================================================================\n") + + return sb.String() +} + +func renderScenarioReportJSON(report *ScenarioReport) string { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderScenarioReportMarkdown(report *ScenarioReport) string { + var sb strings.Builder + + emoji := "✅" + if report.OverallStatus != "PASS" { + emoji = "❌" + } + + sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) + + if report.TestDescription != "" { + sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) + } + + sb.WriteString("| Metric | Value | Expected | Status |\n") + sb.WriteString("|--------|------:|:--------:|:------:|\n") + + keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} + for _, name := range keyMetrics { + for _, c := range report.Comparisons { + if c.Name == name { + value := fmt.Sprintf("%.0f", c.NewValue) + expected := "-" + if c.Expected > 0 { + expected = fmt.Sprintf("%.0f", c.Expected) + } + status := "✅" + if c.Status == "fail" { + status = "❌" + } else if c.Status == "info" { + status = "ℹ️" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) + break + } + } + } + + return sb.String() +} diff --git a/test/loadtest/internal/cmd/root.go b/test/loadtest/internal/cmd/root.go new file mode 100644 index 000000000..406d57671 --- /dev/null +++ b/test/loadtest/internal/cmd/root.go @@ -0,0 +1,44 @@ +// Package cmd implements the CLI commands for the load test tool. +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +const ( + // DefaultClusterName is the default kind cluster name. + DefaultClusterName = "reloader-loadtest" + // TestNamespace is the namespace used for test resources. + TestNamespace = "reloader-test" +) + +// OutputFormat defines the output format for reports. +type OutputFormat string + +const ( + OutputFormatText OutputFormat = "text" + OutputFormatJSON OutputFormat = "json" + OutputFormatMarkdown OutputFormat = "markdown" +) + +// rootCmd is the base command. +var rootCmd = &cobra.Command{ + Use: "loadtest", + Short: "Reloader Load Test CLI", + Long: `A CLI tool for running A/B comparison load tests on Reloader.`, +} + +func init() { + rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(reportCmd) + rootCmd.AddCommand(summaryCmd) +} + +// Execute runs the root command. +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go new file mode 100644 index 000000000..8c88b5439 --- /dev/null +++ b/test/loadtest/internal/cmd/run.go @@ -0,0 +1,648 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + "github.com/stakater/Reloader/test/loadtest/internal/cluster" + "github.com/stakater/Reloader/test/loadtest/internal/prometheus" + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + "github.com/stakater/Reloader/test/loadtest/internal/scenarios" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// RunConfig holds CLI configuration for the run command. +type RunConfig struct { + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + ClusterName string + ResultsDir string + ManifestsDir string + Parallelism int +} + +// workerContext holds all resources for a single worker (cluster + prometheus). +type workerContext struct { + id int + clusterMgr *cluster.Manager + promMgr *prometheus.Manager + kubeClient kubernetes.Interface + kubeContext string + runtime string +} + +var runCfg RunConfig + +var runCmd = &cobra.Command{ + Use: "run", + Short: "Run A/B comparison tests", + Long: `Run load tests comparing old and new versions of Reloader. + +Examples: + # Compare two images + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 + + # Run specific scenario + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 + + # Test single image (no comparison) + loadtest run --new-image=localhost/reloader:test + + # Run all scenarios in parallel on 4 clusters + loadtest run --new-image=localhost/reloader:test --parallelism=4`, + Run: func(cmd *cobra.Command, args []string) { + runCommand() + }, +} + +func init() { + runCmd.Flags().StringVar(&runCfg.OldImage, "old-image", "", "Container image for \"old\" version (required for comparison)") + runCmd.Flags().StringVar(&runCfg.NewImage, "new-image", "", "Container image for \"new\" version (required for comparison)") + runCmd.Flags().StringVar(&runCfg.Scenario, "scenario", "all", "Test scenario: S1-S13 or \"all\"") + runCmd.Flags().IntVar(&runCfg.Duration, "duration", 60, "Test duration in seconds") + runCmd.Flags().IntVar(&runCfg.Parallelism, "parallelism", 1, "Run N scenarios in parallel on N clusters") + runCmd.Flags().BoolVar(&runCfg.SkipCluster, "skip-cluster", false, "Skip kind cluster creation (use existing)") + runCmd.Flags().StringVar(&runCfg.ClusterName, "cluster-name", DefaultClusterName, "Kind cluster name") + runCmd.Flags().StringVar(&runCfg.ResultsDir, "results-dir", "./results", "Directory for results") + runCmd.Flags().StringVar(&runCfg.ManifestsDir, "manifests-dir", "", "Directory containing manifests (auto-detected if not set)") +} + +func runCommand() { + if runCfg.ManifestsDir == "" { + execPath, _ := os.Executable() + execDir := filepath.Dir(execPath) + runCfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") + if _, err := os.Stat(runCfg.ManifestsDir); os.IsNotExist(err) { + runCfg.ManifestsDir = "./manifests" + } + } + + if runCfg.Parallelism < 1 { + runCfg.Parallelism = 1 + } + + if runCfg.OldImage == "" && runCfg.NewImage == "" { + log.Fatal("At least one of --old-image or --new-image is required") + } + + runOld := runCfg.OldImage != "" + runNew := runCfg.NewImage != "" + runBoth := runOld && runNew + + log.Printf("Configuration:") + log.Printf(" Scenario: %s", runCfg.Scenario) + log.Printf(" Duration: %ds", runCfg.Duration) + log.Printf(" Parallelism: %d", runCfg.Parallelism) + if runCfg.OldImage != "" { + log.Printf(" Old image: %s", runCfg.OldImage) + } + if runCfg.NewImage != "" { + log.Printf(" New image: %s", runCfg.NewImage) + } + + runtime, err := cluster.DetectContainerRuntime() + if err != nil { + log.Fatalf("Failed to detect container runtime: %v", err) + } + log.Printf(" Container runtime: %s", runtime) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Println("Received shutdown signal...") + cancel() + }() + + scenariosToRun := []string{runCfg.Scenario} + if runCfg.Scenario == "all" { + scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } + + if runCfg.SkipCluster && runCfg.Parallelism > 1 { + log.Fatal("--skip-cluster is not supported with --parallelism > 1") + } + + if runCfg.Parallelism > 1 { + runParallel(ctx, runCfg, scenariosToRun, runtime, runOld, runNew, runBoth) + return + } + + runSequential(ctx, runCfg, scenariosToRun, runtime, runOld, runNew, runBoth) +} + +func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + clusterMgr := cluster.NewManager(cluster.Config{ + Name: cfg.ClusterName, + ContainerRuntime: runtime, + }) + + if cfg.SkipCluster { + log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) + if !clusterMgr.Exists() { + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) + } + } else { + log.Println("Creating kind cluster...") + if err := clusterMgr.Create(ctx); err != nil { + log.Fatalf("Failed to create cluster: %v", err) + } + } + + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManager(promManifest) + + log.Println("Installing Prometheus...") + if err := promMgr.Deploy(ctx); err != nil { + log.Fatalf("Failed to deploy Prometheus: %v", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + log.Fatalf("Failed to start Prometheus port-forward: %v", err) + } + defer promMgr.StopPortForward() + + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } + } + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } + } + + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + kubeClient, err := getKubeClient("") + if err != nil { + log.Fatalf("Failed to create kubernetes client: %v", err) + } + + for _, scenarioID := range scenariosToRun { + log.Printf("========================================") + log.Printf("=== Starting scenario %s ===", scenarioID) + log.Printf("========================================") + + cleanupTestNamespaces(ctx, "") + cleanupReloader(ctx, "old", "") + cleanupReloader(ctx, "new", "") + + if err := promMgr.Reset(ctx); err != nil { + log.Printf("Warning: failed to reset Prometheus: %v", err) + } + + createTestNamespace(ctx, "") + + if runOld { + oldMgr := reloader.NewManager(reloader.Config{ + Version: "old", + Image: cfg.OldImage, + }) + + if err := oldMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy old Reloader: %v", err) + continue + } + + if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) + collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) + + if runBoth { + cleanupTestNamespaces(ctx, "") + oldMgr.Cleanup(ctx) + promMgr.Reset(ctx) + createTestNamespace(ctx, "") + } + } + + if runNew { + newMgr := reloader.NewManager(reloader.Config{ + Version: "new", + Image: cfg.NewImage, + }) + + if err := newMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy new Reloader: %v", err) + continue + } + + if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) + collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) + } + + generateReport(scenarioID, cfg.ResultsDir, runBoth) + log.Printf("=== Scenario %s complete ===", scenarioID) + } + + log.Println("Load test complete!") + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +func runParallel(ctx context.Context, cfg RunConfig, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + numWorkers := cfg.Parallelism + if numWorkers > len(scenariosToRun) { + numWorkers = len(scenariosToRun) + log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) + } + + log.Printf("Starting parallel execution with %d workers", numWorkers) + + workers := make([]*workerContext, numWorkers) + var setupWg sync.WaitGroup + setupErrors := make(chan error, numWorkers) + + log.Println("Setting up worker clusters...") + for i := range numWorkers { + setupWg.Add(1) + go func(workerID int) { + defer setupWg.Done() + worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) + if err != nil { + setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) + return + } + workers[workerID] = worker + }(i) + } + + setupWg.Wait() + close(setupErrors) + + for err := range setupErrors { + log.Printf("Error: %v", err) + } + + readyWorkers := 0 + for _, w := range workers { + if w != nil { + readyWorkers++ + } + } + if readyWorkers == 0 { + log.Fatal("No workers ready, aborting") + } + if readyWorkers < numWorkers { + log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) + } + + defer func() { + log.Println("Cleaning up worker clusters...") + for _, w := range workers { + if w != nil { + w.promMgr.StopPortForward() + } + } + }() + + scenarioCh := make(chan string, len(scenariosToRun)) + for _, s := range scenariosToRun { + scenarioCh <- s + } + close(scenarioCh) + + var resultsMu sync.Mutex + completedScenarios := make([]string, 0, len(scenariosToRun)) + + var wg sync.WaitGroup + for _, worker := range workers { + if worker == nil { + continue + } + wg.Add(1) + go func(w *workerContext) { + defer wg.Done() + for scenarioID := range scenarioCh { + select { + case <-ctx.Done(): + return + default: + } + + log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) + + cleanupTestNamespaces(ctx, w.kubeContext) + cleanupReloader(ctx, "old", w.kubeContext) + cleanupReloader(ctx, "new", w.kubeContext) + + if err := w.promMgr.Reset(ctx); err != nil { + log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) + } + + createTestNamespace(ctx, w.kubeContext) + + if runOld { + runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) + } + + if runNew { + runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) + } + + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + resultsMu.Lock() + completedScenarios = append(completedScenarios, scenarioID) + resultsMu.Unlock() + + log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) + } + }(worker) + } + + wg.Wait() + + log.Println("Load test complete!") + log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +func setupWorker(ctx context.Context, cfg RunConfig, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { + workerName := fmt.Sprintf("%s-%d", DefaultClusterName, workerID) + promPort := 9091 + workerID + + log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) + + clusterMgr := cluster.NewManager(cluster.Config{ + Name: workerName, + ContainerRuntime: runtime, + PortOffset: workerID, + }) + + if err := clusterMgr.Create(ctx); err != nil { + return nil, fmt.Errorf("creating cluster: %w", err) + } + + kubeContext := clusterMgr.Context() + + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) + + log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) + if err := promMgr.Deploy(ctx); err != nil { + return nil, fmt.Errorf("deploying prometheus: %w", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + return nil, fmt.Errorf("starting prometheus port-forward: %w", err) + } + + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } + } + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } + } + + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + kubeClient, err := getKubeClient(kubeContext) + if err != nil { + return nil, fmt.Errorf("creating kubernetes client: %w", err) + } + + log.Printf("[Worker %d] Ready", workerID) + return &workerContext{ + id: workerID, + clusterMgr: clusterMgr, + promMgr: promMgr, + kubeClient: kubeClient, + kubeContext: kubeContext, + runtime: runtime, + }, nil +} + +func runVersionOnWorker(ctx context.Context, w *workerContext, cfg RunConfig, scenarioID, version, image string, cleanupAfter bool) { + mgr := reloader.NewManager(reloader.Config{ + Version: version, + Image: image, + }) + mgr.SetKubeContext(w.kubeContext) + + if err := mgr.Deploy(ctx); err != nil { + log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) + return + } + + if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { + log.Printf("[Worker %d] Warning: %v", w.id, err) + log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) + } + + runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) + collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) + + if cleanupAfter { + cleanupTestNamespaces(ctx, w.kubeContext) + mgr.Cleanup(ctx) + w.promMgr.Reset(ctx) + createTestNamespace(ctx, w.kubeContext) + } +} + +func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { + runner, ok := scenarios.Registry[scenarioID] + if !ok { + log.Printf("Unknown scenario: %s", scenarioID) + return + } + + if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { + s6.ReloaderVersion = version + } + + if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { + s11.Image = image + } + + log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) + + if ctx.Err() != nil { + log.Printf("WARNING: Parent context already done: %v", ctx.Err()) + } + + timeout := time.Duration(duration)*time.Second + 5*time.Minute + log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) + + scenarioCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + expected, err := runner.Run(scenarioCtx, client, TestNamespace, time.Duration(duration)*time.Second) + if err != nil { + log.Printf("Scenario %s failed: %v", scenarioID, err) + } + + scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) +} + +func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { + log.Printf("Waiting 5s for Reloader to finish processing events...") + time.Sleep(5 * time.Second) + + log.Printf("Waiting 8s for Prometheus to scrape final metrics...") + time.Sleep(8 * time.Second) + + log.Printf("Collecting metrics for %s...", version) + outputDir := filepath.Join(resultsDir, scenarioID, version) + if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { + log.Printf("Failed to collect metrics: %v", err) + } +} + +func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { + log.Printf("Collecting logs for %s...", version) + logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") + if err := mgr.CollectLogs(ctx, logPath); err != nil { + log.Printf("Failed to collect logs: %v", err) + } +} + +func generateReport(scenarioID, resultsDir string, isComparison bool) { + if isComparison { + log.Println("Generating comparison report...") + } else { + log.Println("Generating single-version report...") + } + + reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") + + cmd := exec.Command(os.Args[0], "report", + fmt.Sprintf("--scenario=%s", scenarioID), + fmt.Sprintf("--results-dir=%s", resultsDir), + fmt.Sprintf("--output=%s", reportPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Run() + + if data, err := os.ReadFile(reportPath); err == nil { + fmt.Println(string(data)) + } + + log.Printf("Report saved to: %s", reportPath) +} + +func getKubeClient(kubeContext string) (kubernetes.Interface, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, _ := os.UserHomeDir() + kubeconfig = filepath.Join(home, ".kube", "config") + } + + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} + configOverrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + configOverrides.CurrentContext = kubeContext + } + + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(config) +} + +func createTestNamespace(ctx context.Context, kubeContext string) { + args := []string{"create", "namespace", TestNamespace, "--dry-run=client", "-o", "yaml"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, _ := cmd.Output() + + applyArgs := []string{"apply", "-f", "-"} + if kubeContext != "" { + applyArgs = append([]string{"--context", kubeContext}, applyArgs...) + } + applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) + applyCmd.Stdin = strings.NewReader(string(out)) + applyCmd.Run() +} + +func cleanupTestNamespaces(ctx context.Context, kubeContext string) { + log.Println("Cleaning up test resources...") + + namespaces := []string{TestNamespace} + for i := range 10 { + namespaces = append(namespaces, fmt.Sprintf("%s-%d", TestNamespace, i)) + } + + for _, ns := range namespaces { + args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } + + time.Sleep(2 * time.Second) + + for _, ns := range namespaces { + args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } +} + +func cleanupReloader(ctx context.Context, version string, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} diff --git a/test/loadtest/internal/cmd/summary.go b/test/loadtest/internal/cmd/summary.go new file mode 100644 index 000000000..bda40fb94 --- /dev/null +++ b/test/loadtest/internal/cmd/summary.go @@ -0,0 +1,251 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + summaryResultsDir string + summaryOutputFile string + summaryFormat string + summaryTestType string +) + +var summaryCmd = &cobra.Command{ + Use: "summary", + Short: "Generate summary across all scenarios (for CI)", + Long: `Generate an aggregated summary report across all test scenarios. + +Examples: + # Generate markdown summary for CI + loadtest summary --results-dir=./results --format=markdown`, + Run: func(cmd *cobra.Command, args []string) { + summaryCommand() + }, +} + +func init() { + summaryCmd.Flags().StringVar(&summaryResultsDir, "results-dir", "./results", "Directory containing results") + summaryCmd.Flags().StringVar(&summaryOutputFile, "output", "", "Output file (default: stdout)") + summaryCmd.Flags().StringVar(&summaryFormat, "format", "markdown", "Output format: text, json, markdown") + summaryCmd.Flags().StringVar(&summaryTestType, "test-type", "full", "Test type label: quick, full") +} + +// SummaryReport aggregates results from multiple scenarios. +type SummaryReport struct { + Timestamp time.Time `json:"timestamp"` + TestType string `json:"test_type"` + PassCount int `json:"pass_count"` + FailCount int `json:"fail_count"` + TotalCount int `json:"total_count"` + Scenarios []ScenarioSummary `json:"scenarios"` +} + +// ScenarioSummary provides a brief summary of a single scenario. +type ScenarioSummary struct { + ID string `json:"id"` + Status string `json:"status"` + Description string `json:"description"` + ActionTotal float64 `json:"action_total"` + ActionExp float64 `json:"action_expected"` + ErrorsTotal float64 `json:"errors_total"` +} + +func summaryCommand() { + summary, err := generateSummaryReport(summaryResultsDir, summaryTestType) + if err != nil { + log.Fatalf("Failed to generate summary: %v", err) + } + + var output string + switch OutputFormat(summaryFormat) { + case OutputFormatJSON: + output = renderSummaryJSON(summary) + case OutputFormatText: + output = renderSummaryText(summary) + default: + output = renderSummaryMarkdown(summary) + } + + if summaryOutputFile != "" { + if err := os.WriteFile(summaryOutputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Summary written to %s", summaryOutputFile) + } else { + fmt.Print(output) + } + + if summary.FailCount > 0 { + os.Exit(1) + } +} + +func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { + summary := &SummaryReport{ + Timestamp: time.Now(), + TestType: testType, + } + + entries, err := os.ReadDir(resultsDir) + if err != nil { + return nil, fmt.Errorf("failed to read results directory: %w", err) + } + + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { + continue + } + + scenarioID := entry.Name() + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) + continue + } + + scenarioSummary := ScenarioSummary{ + ID: scenarioID, + Status: report.OverallStatus, + Description: report.TestDescription, + } + + for _, c := range report.Comparisons { + switch c.Name { + case "action_total": + scenarioSummary.ActionTotal = c.NewValue + scenarioSummary.ActionExp = c.Expected + case "errors_total": + scenarioSummary.ErrorsTotal = c.NewValue + } + } + + summary.Scenarios = append(summary.Scenarios, scenarioSummary) + summary.TotalCount++ + if report.OverallStatus == "PASS" { + summary.PassCount++ + } else { + summary.FailCount++ + } + } + + sort.Slice(summary.Scenarios, func(i, j int) bool { + return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) + }) + + return summary, nil +} + +func naturalSort(a, b string) bool { + var aNum, bNum int + fmt.Sscanf(a, "S%d", &aNum) + fmt.Sscanf(b, "S%d", &bNum) + return aNum < bNum +} + +func renderSummaryJSON(summary *SummaryReport) string { + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderSummaryText(summary *SummaryReport) string { + var sb strings.Builder + + sb.WriteString("================================================================================\n") + sb.WriteString(" LOAD TEST SUMMARY\n") + sb.WriteString("================================================================================\n\n") + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + + fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) + fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) + + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") + + for _, s := range summary.Scenarios { + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) + } + + sb.WriteString("\n================================================================================\n") + return sb.String() +} + +func renderSummaryMarkdown(summary *SummaryReport) string { + var sb strings.Builder + + emoji := "✅" + title := "ALL TESTS PASSED" + if summary.FailCount > 0 { + emoji = "❌" + title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) + } else if summary.TotalCount == 0 { + emoji = "⚠️" + title = "NO RESULTS" + } + + sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) + + if summary.TestType == "quick" { + sb.WriteString("> 🚀 **Quick Test** (S1, S4, S6) — Use `/loadtest` for full suite\n\n") + } + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) + + sb.WriteString("| | Scenario | Description | Actions | Errors |\n") + sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") + + for _, s := range summary.Scenarios { + icon := "✅" + if s.Status != "PASS" { + icon = "❌" + } + + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + + errors := fmt.Sprintf("%.0f", s.ErrorsTotal) + if s.ErrorsTotal > 0 { + errors = fmt.Sprintf("⚠️ %.0f", s.ErrorsTotal) + } + + sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) + } + + sb.WriteString("\n📦 **[Download detailed results](../artifacts)**\n") + + return sb.String() +} From 76287e04202f6d590ad731368947cb0ebdaff9aa Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:36:02 +0100 Subject: [PATCH 11/21] refactor: Cleanup logic for reloader in loadtests --- test/loadtest/internal/cmd/run.go | 25 ++++----------------- test/loadtest/internal/reloader/reloader.go | 21 +++++++++++++++++ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 8c88b5439..4a0f731a2 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -208,8 +208,8 @@ func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, log.Printf("========================================") cleanupTestNamespaces(ctx, "") - cleanupReloader(ctx, "old", "") - cleanupReloader(ctx, "new", "") + reloader.CleanupByVersion(ctx, "old", "") + reloader.CleanupByVersion(ctx, "new", "") if err := promMgr.Reset(ctx); err != nil { log.Printf("Warning: failed to reset Prometheus: %v", err) @@ -357,8 +357,8 @@ func runParallel(ctx context.Context, cfg RunConfig, scenariosToRun []string, ru log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) cleanupTestNamespaces(ctx, w.kubeContext) - cleanupReloader(ctx, "old", w.kubeContext) - cleanupReloader(ctx, "new", w.kubeContext) + reloader.CleanupByVersion(ctx, "old", w.kubeContext) + reloader.CleanupByVersion(ctx, "new", w.kubeContext) if err := w.promMgr.Reset(ctx); err != nil { log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) @@ -629,20 +629,3 @@ func cleanupTestNamespaces(ctx context.Context, kubeContext string) { } } -func cleanupReloader(ctx context.Context, version string, kubeContext string) { - ns := fmt.Sprintf("reloader-%s", version) - - nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - - if kubeContext != "" { - nsArgs = append([]string{"--context", kubeContext}, nsArgs...) - crArgs = append([]string{"--context", kubeContext}, crArgs...) - crbArgs = append([]string{"--context", kubeContext}, crbArgs...) - } - - exec.CommandContext(ctx, "kubectl", nsArgs...).Run() - exec.CommandContext(ctx, "kubectl", crArgs...).Run() - exec.CommandContext(ctx, "kubectl", crbArgs...).Run() -} diff --git a/test/loadtest/internal/reloader/reloader.go b/test/loadtest/internal/reloader/reloader.go index ff3cfdb3f..2667cd474 100644 --- a/test/loadtest/internal/reloader/reloader.go +++ b/test/loadtest/internal/reloader/reloader.go @@ -216,6 +216,27 @@ func (m *Manager) Cleanup(ctx context.Context) error { return nil } +// CleanupByVersion removes Reloader resources for a specific version without needing a Manager instance. +// This is useful for cleaning up from previous runs before creating a new Manager. +func CleanupByVersion(ctx context.Context, version, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + name := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", name, "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", name, "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} + // CollectLogs collects logs from the Reloader pod and writes them to the specified file. func (m *Manager) CollectLogs(ctx context.Context, logPath string) error { ns := m.namespace() From 2442eddd811aebe11e1463464f3e24ae0f9146a6 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:34:09 +0100 Subject: [PATCH 12/21] fix: Skip loading images when already done --- .github/actions/loadtest/action.yml | 1 + test/loadtest/internal/cluster/kind.go | 20 +++++-- test/loadtest/internal/cmd/run.go | 78 +++++++++++++++----------- 3 files changed, 60 insertions(+), 39 deletions(-) diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml index 3b91957ec..86dcf49aa 100644 --- a/.github/actions/loadtest/action.yml +++ b/.github/actions/loadtest/action.yml @@ -162,6 +162,7 @@ runs: ARGS="$ARGS --scenario=${{ inputs.scenarios }}" ARGS="$ARGS --duration=${{ inputs.duration }}" ARGS="$ARGS --cluster-name=${{ steps.cluster.outputs.name }}" + ARGS="$ARGS --skip-image-load" if [ -n "${{ steps.images.outputs.old }}" ]; then ARGS="$ARGS --old-image=${{ steps.images.outputs.old }}" diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go index bcd5e19c6..557e628ba 100644 --- a/test/loadtest/internal/cluster/kind.go +++ b/test/loadtest/internal/cluster/kind.go @@ -29,14 +29,24 @@ func NewManager(cfg Config) *Manager { } // DetectContainerRuntime finds available container runtime. +// It checks if the runtime daemon is actually running, not just if the binary exists. func DetectContainerRuntime() (string, error) { - if _, err := exec.LookPath("podman"); err == nil { - return "podman", nil - } + // Prefer docker as it's more commonly used with kind if _, err := exec.LookPath("docker"); err == nil { - return "docker", nil + // Verify docker daemon is running + cmd := exec.Command("docker", "info") + if err := cmd.Run(); err == nil { + return "docker", nil + } + } + if _, err := exec.LookPath("podman"); err == nil { + // Verify podman is functional (check if we can run a basic command) + cmd := exec.Command("podman", "info") + if err := cmd.Run(); err == nil { + return "podman", nil + } } - return "", fmt.Errorf("neither docker nor podman found in PATH") + return "", fmt.Errorf("neither docker nor podman is running") } // Exists checks if the cluster already exists. diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 4a0f731a2..29bcb3424 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -24,15 +24,16 @@ import ( // RunConfig holds CLI configuration for the run command. type RunConfig struct { - OldImage string - NewImage string - Scenario string - Duration int - SkipCluster bool - ClusterName string - ResultsDir string - ManifestsDir string - Parallelism int + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + SkipImageLoad bool + ClusterName string + ResultsDir string + ManifestsDir string + Parallelism int } // workerContext holds all resources for a single worker (cluster + prometheus). @@ -76,6 +77,7 @@ func init() { runCmd.Flags().IntVar(&runCfg.Duration, "duration", 60, "Test duration in seconds") runCmd.Flags().IntVar(&runCfg.Parallelism, "parallelism", 1, "Run N scenarios in parallel on N clusters") runCmd.Flags().BoolVar(&runCfg.SkipCluster, "skip-cluster", false, "Skip kind cluster creation (use existing)") + runCmd.Flags().BoolVar(&runCfg.SkipImageLoad, "skip-image-load", false, "Skip loading images into kind (use when images already loaded)") runCmd.Flags().StringVar(&runCfg.ClusterName, "cluster-name", DefaultClusterName, "Kind cluster name") runCmd.Flags().StringVar(&runCfg.ResultsDir, "results-dir", "./results", "Directory for results") runCmd.Flags().StringVar(&runCfg.ManifestsDir, "manifests-dir", "", "Directory containing manifests (auto-detected if not set)") @@ -179,23 +181,27 @@ func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, } defer promMgr.StopPortForward() - log.Println("Loading images into kind cluster...") - if runOld { - log.Printf("Loading old image: %s", cfg.OldImage) - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Fatalf("Failed to load old image: %v", err) + if cfg.SkipImageLoad { + log.Println("Skipping image loading (--skip-image-load)") + } else { + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } } - } - if runNew { - log.Printf("Loading new image: %s", cfg.NewImage) - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Fatalf("Failed to load new image: %v", err) + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } } - } - log.Println("Pre-loading test images...") - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + } kubeClient, err := getKubeClient("") if err != nil { @@ -422,20 +428,24 @@ func setupWorker(ctx context.Context, cfg RunConfig, workerID int, runtime strin return nil, fmt.Errorf("starting prometheus port-forward: %w", err) } - log.Printf("[Worker %d] Loading images...", workerID) - if runOld { - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + if cfg.SkipImageLoad { + log.Printf("[Worker %d] Skipping image loading (--skip-image-load)", workerID) + } else { + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } } - } - if runNew { - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } } - } - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + } kubeClient, err := getKubeClient(kubeContext) if err != nil { From c4f3255c78cddecdaae6feec533576ea9f64a174 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:34:25 +0100 Subject: [PATCH 13/21] ci: Disable tests temporarily --- .github/workflows/push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index dda9a1c14..092da7cd1 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -73,8 +73,8 @@ jobs: kind create cluster kubectl cluster-info - - name: Test - run: make test + #- name: Test + # run: make test - name: Set up QEMU uses: docker/setup-qemu-action@v3 From e56323d582ba629d4de1e39455414b6c5b06b2ba Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:37:34 +0100 Subject: [PATCH 14/21] ci: Disable tests in PR --- .github/workflows/pull_request.yaml | 4 ++-- .github/workflows/push.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index fbf59abae..2160c48ad 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -107,8 +107,8 @@ jobs: kubectl cluster-info - - name: Test - run: make test + #- name: Test + # run: make test - name: Run quick A/B load tests uses: ./.github/actions/loadtest diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 092da7cd1..dda9a1c14 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -73,8 +73,8 @@ jobs: kind create cluster kubectl cluster-info - #- name: Test - # run: make test + - name: Test + run: make test - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 2674f405ce76f3e0944ec76b1c777eee3b6959ed Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:44:43 +0100 Subject: [PATCH 15/21] fix: Issue with not parsing multiple scenario in args --- test/loadtest/internal/cmd/run.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 29bcb3424..c78e57916 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -133,9 +133,16 @@ func runCommand() { cancel() }() - scenariosToRun := []string{runCfg.Scenario} + var scenariosToRun []string if runCfg.Scenario == "all" { scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } else { + // Split comma-separated scenarios (e.g., "S1,S4,S6") + for _, s := range strings.Split(runCfg.Scenario, ",") { + if trimmed := strings.TrimSpace(s); trimmed != "" { + scenariosToRun = append(scenariosToRun, trimmed) + } + } } if runCfg.SkipCluster && runCfg.Parallelism > 1 { From a132ed8deaffc7e22455e1098d5456f1e0ef65c7 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:48:05 +0100 Subject: [PATCH 16/21] ci: Don't comment on forked PRs automatically --- .github/actions/loadtest/action.yml | 22 ++++++++++++++++------ .github/workflows/pull_request.yaml | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml index 86dcf49aa..3f71ae9f5 100644 --- a/.github/actions/loadtest/action.yml +++ b/.github/actions/loadtest/action.yml @@ -208,6 +208,7 @@ runs: - name: Post PR comment if: inputs.post-comment == 'true' && inputs.pr-number != '' + continue-on-error: true uses: actions/github-script@v7 with: github-token: ${{ inputs.github-token }} @@ -234,12 +235,21 @@ runs: `**Artifacts:** [Download](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`, ].join('\n'); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ inputs.pr-number }}, - body: body - }); + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + body: body + }); + console.log('Comment posted successfully'); + } catch (error) { + if (error.status === 403) { + console.log('Could not post comment (fork PR with restricted permissions). Use /loadtest command to run with comment posting.'); + } else { + throw error; + } + } - name: Upload results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 2160c48ad..1ed89399f 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -36,6 +36,7 @@ jobs: permissions: contents: read pull-requests: write + issues: write runs-on: ubuntu-latest name: Build From ad6013adbf6eb456c88ac383fb04afb3f9fe5ce6 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:58:28 +0100 Subject: [PATCH 17/21] fix: Treat missing metrics as info --- test/loadtest/internal/cmd/report.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/loadtest/internal/cmd/report.go b/test/loadtest/internal/cmd/report.go index aa273a517..7bf4cc670 100644 --- a/test/loadtest/internal/cmd/report.go +++ b/test/loadtest/internal/cmd/report.go @@ -608,7 +608,11 @@ func compareMetricWithExpected(name string, oldValue, newValue, expected float64 oldMeetsExp := meetsExpected(oldValue, expected) newMeetsExp := meetsExpected(newValue, expected) - if expected > 0 && threshold.metricType == ShouldMatch { + isNewMetric := info.isCounter && oldValue == 0 && newValue > 0 && expected == 0 + + if isNewMetric { + status = "info" + } else if expected > 0 && threshold.metricType == ShouldMatch { if newMeetsExp == "✗" { status = "fail" } From 07f7365d63f16d9d67c3ef735eb955b12342d229 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:14:45 +0100 Subject: [PATCH 18/21] ci: Enable tests for PR again --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 1ed89399f..c428826ea 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -108,8 +108,8 @@ jobs: kubectl cluster-info - #- name: Test - # run: make test + - name: Test + run: make test - name: Run quick A/B load tests uses: ./.github/actions/loadtest From 1945a740d0fb6860f137f675b723f371a117c40f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:22:24 +0100 Subject: [PATCH 19/21] chore: Format files# --- internal/pkg/metrics/prometheus.go | 38 ++++++++++-------------------- pkg/kube/resourcemapper.go | 6 ++--- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index e6f2f356b..6d103a10c 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -57,38 +57,24 @@ func init() { // Collectors holds all Prometheus metrics collectors for Reloader. type Collectors struct { - // Existing metrics (preserved for backward compatibility) Reloaded *prometheus.CounterVec ReloadedByNamespace *prometheus.CounterVec countByNamespace bool - // Reconcile/Handler metrics ReconcileTotal *prometheus.CounterVec // Total reconcile calls by result ReconcileDuration *prometheus.HistogramVec // Time spent in reconcile/handler - - // Action metrics - ActionTotal *prometheus.CounterVec // Total actions by workload kind and result - ActionLatency *prometheus.HistogramVec // Time from event to action applied - - // Skip metrics - SkippedTotal *prometheus.CounterVec // Skipped operations by reason - - // Queue metrics - QueueDepth prometheus.Gauge // Current queue depth - QueueAdds prometheus.Counter // Total items added to queue - QueueLatency *prometheus.HistogramVec // Time spent in queue - - // Error and retry metrics - ErrorsTotal *prometheus.CounterVec // Errors by type - RetriesTotal prometheus.Counter // Total retries - - // Event processing metrics - EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) - EventsProcessed *prometheus.CounterVec // Events processed by type and result - - // Resource discovery metrics - WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind - WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind + ActionTotal *prometheus.CounterVec // Total actions by workload kind and result + ActionLatency *prometheus.HistogramVec // Time from event to action applied + SkippedTotal *prometheus.CounterVec // Skipped operations by reason + QueueDepth prometheus.Gauge // Current queue depth + QueueAdds prometheus.Counter // Total items added to queue + QueueLatency *prometheus.HistogramVec // Time spent in queue + ErrorsTotal *prometheus.CounterVec // Errors by type + RetriesTotal prometheus.Counter // Total retries + EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) + EventsProcessed *prometheus.CounterVec // Events processed by type and result + WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind + WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind } // RecordReload records a reload event with the given success status and namespace. diff --git a/pkg/kube/resourcemapper.go b/pkg/kube/resourcemapper.go index 286d40813..bdb7858ba 100644 --- a/pkg/kube/resourcemapper.go +++ b/pkg/kube/resourcemapper.go @@ -8,8 +8,8 @@ import ( // ResourceMap are resources from where changes are going to be detected var ResourceMap = map[string]runtime.Object{ - "configmaps": &v1.ConfigMap{}, - "secrets": &v1.Secret{}, - "namespaces": &v1.Namespace{}, + "configmaps": &v1.ConfigMap{}, + "secrets": &v1.Secret{}, + "namespaces": &v1.Namespace{}, "secretproviderclasspodstatuses": &csiv1.SecretProviderClassPodStatus{}, } From 1be910749b91728fdfd6d9573f10580e9925b454 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:34:16 +0100 Subject: [PATCH 20/21] chore: A lot of cleanup --- .github/workflows/loadtest.yml | 1 - internal/pkg/controller/controller.go | 12 +--- internal/pkg/metrics/prometheus.go | 3 - test/loadtest/cmd/loadtest/main.go | 1 - test/loadtest/internal/cluster/kind.go | 9 --- test/loadtest/internal/cmd/root.go | 1 - .../internal/prometheus/prometheus.go | 29 +-------- test/loadtest/internal/scenarios/scenarios.go | 65 +------------------ 8 files changed, 9 insertions(+), 112 deletions(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index f4eb32216..c997e13d0 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -10,7 +10,6 @@ permissions: issues: write jobs: - # Full load test suite triggered by /loadtest command loadtest: # Only run on PR comments with /loadtest command if: | diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index da2b2c789..1a51d9a59 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -110,7 +110,6 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { - // Record event received c.collectors.RecordEventReceived("add", c.resource) switch object := obj.(type) { @@ -127,7 +126,7 @@ func (c *Controller) Add(obj interface{}) { Resource: obj, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -186,7 +185,6 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { - // Record event received c.collectors.RecordEventReceived("update", c.resource) switch new.(type) { @@ -200,7 +198,7 @@ func (c *Controller) Update(old interface{}, new interface{}) { OldResource: old, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -209,7 +207,6 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { - // Record event received c.collectors.RecordEventReceived("delete", c.resource) if _, ok := old.(*csiv1.SecretProviderClassPodStatus); ok { @@ -222,7 +219,7 @@ func (c *Controller) Delete(old interface{}) { Resource: old, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -285,7 +282,6 @@ func (c *Controller) processNextItem() bool { return false } - // Update queue depth after getting item c.collectors.SetQueueDepth(c.queue.Len()) // Tell the queue that we are done with processing this key. This unblocks the key for other workers @@ -307,7 +303,6 @@ func (c *Controller) processNextItem() bool { duration := time.Since(startTime) - // Record reconcile metrics if err != nil { c.collectors.RecordReconcile("error", duration) } else { @@ -355,7 +350,6 @@ func (c *Controller) handleErr(err error, key interface{}) { logrus.Errorf("Dropping key out of the queue: %v", err) logrus.Debugf("Dropping the key %q out of the queue: %v", key, err) - // Record failed event processing c.collectors.RecordEventProcessed("unknown", c.resource, "dropped") } diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index 6d103a10c..43103935e 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -219,8 +219,6 @@ func NewCollectors() Collectors { []string{"success", "namespace"}, ) - // === NEW: Comprehensive metrics === - reconcileTotal := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", @@ -372,7 +370,6 @@ func NewCollectors() Collectors { func SetupPrometheusEndpoint() Collectors { collectors := NewCollectors() - // Register all metrics prometheus.MustRegister(collectors.Reloaded) prometheus.MustRegister(collectors.ReconcileTotal) prometheus.MustRegister(collectors.ReconcileDuration) diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index a8925bc37..510ce0b43 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -1,4 +1,3 @@ -// Package main is the entrypoint for the load test CLI. package main import "github.com/stakater/Reloader/test/loadtest/internal/cmd" diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go index 557e628ba..1fde31428 100644 --- a/test/loadtest/internal/cluster/kind.go +++ b/test/loadtest/internal/cluster/kind.go @@ -31,16 +31,13 @@ func NewManager(cfg Config) *Manager { // DetectContainerRuntime finds available container runtime. // It checks if the runtime daemon is actually running, not just if the binary exists. func DetectContainerRuntime() (string, error) { - // Prefer docker as it's more commonly used with kind if _, err := exec.LookPath("docker"); err == nil { - // Verify docker daemon is running cmd := exec.Command("docker", "info") if err := cmd.Run(); err == nil { return "docker", nil } } if _, err := exec.LookPath("podman"); err == nil { - // Verify podman is functional (check if we can run a basic command) cmd := exec.Command("podman", "info") if err := cmd.Run(); err == nil { return "podman", nil @@ -85,7 +82,6 @@ func (m *Manager) Create(ctx context.Context) error { } } - // Calculate unique ports based on offset (for parallel clusters) httpPort := 8080 + m.cfg.PortOffset httpsPort := 8443 + m.cfg.PortOffset @@ -231,7 +227,6 @@ func (m *Manager) Name() string { // LoadImage loads a container image into the kind cluster. func (m *Manager) LoadImage(ctx context.Context, image string) error { - // First check if image exists locally if !m.imageExistsLocally(image) { fmt.Printf(" Image not found locally, pulling: %s\n", image) pullCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) @@ -247,7 +242,6 @@ func (m *Manager) LoadImage(ctx context.Context, image string) error { fmt.Printf(" Copying image to kind cluster...\n") if m.cfg.ContainerRuntime == "podman" { - // For podman, save to archive and load tmpFile := fmt.Sprintf("/tmp/kind-image-%d.tar", time.Now().UnixNano()) defer os.Remove(tmpFile) @@ -276,19 +270,16 @@ func (m *Manager) LoadImage(ctx context.Context, image string) error { // imageExistsLocally checks if an image exists in the local container runtime. func (m *Manager) imageExistsLocally(image string) bool { - // Try "image exists" command (works for podman) cmd := exec.Command(m.cfg.ContainerRuntime, "image", "exists", image) if err := cmd.Run(); err == nil { return true } - // Try "image inspect" (works for both docker and podman) cmd = exec.Command(m.cfg.ContainerRuntime, "image", "inspect", image) if err := cmd.Run(); err == nil { return true } - // Try listing images and grep cmd = exec.Command(m.cfg.ContainerRuntime, "images", "--format", "{{.Repository}}:{{.Tag}}") out, err := cmd.Output() if err == nil { diff --git a/test/loadtest/internal/cmd/root.go b/test/loadtest/internal/cmd/root.go index 406d57671..46e9be552 100644 --- a/test/loadtest/internal/cmd/root.go +++ b/test/loadtest/internal/cmd/root.go @@ -1,4 +1,3 @@ -// Package cmd implements the CLI commands for the load test tool. package cmd import ( diff --git a/test/loadtest/internal/prometheus/prometheus.go b/test/loadtest/internal/prometheus/prometheus.go index f16df782b..b9bf7555a 100644 --- a/test/loadtest/internal/prometheus/prometheus.go +++ b/test/loadtest/internal/prometheus/prometheus.go @@ -21,14 +21,14 @@ type Manager struct { manifestPath string portForward *exec.Cmd localPort int - kubeContext string // Optional: use specific kubeconfig context + kubeContext string } // NewManager creates a new Prometheus manager. func NewManager(manifestPath string) *Manager { return &Manager{ manifestPath: manifestPath, - localPort: 9091, // Use 9091 to avoid conflicts + localPort: 9091, } } @@ -51,7 +51,6 @@ func (m *Manager) kubectl(args ...string) []string { // Deploy deploys Prometheus to the cluster. func (m *Manager) Deploy(ctx context.Context) error { - // Create namespace cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("create", "namespace", "monitoring", "--dry-run=client", "-o", "yaml")...) out, err := cmd.Output() if err != nil { @@ -64,7 +63,6 @@ func (m *Manager) Deploy(ctx context.Context) error { return fmt.Errorf("applying namespace: %w", err) } - // Apply Prometheus manifest applyCmd = exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", m.manifestPath)...) applyCmd.Stdout = os.Stdout applyCmd.Stderr = os.Stderr @@ -72,7 +70,6 @@ func (m *Manager) Deploy(ctx context.Context) error { return fmt.Errorf("applying prometheus manifest: %w", err) } - // Wait for Prometheus to be ready fmt.Println("Waiting for Prometheus to be ready...") waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) @@ -89,7 +86,6 @@ func (m *Manager) Deploy(ctx context.Context) error { func (m *Manager) StartPortForward(ctx context.Context) error { m.StopPortForward() - // Start port-forward m.portForward = exec.CommandContext(ctx, "kubectl", m.kubectl("port-forward", "-n", "monitoring", "svc/prometheus", fmt.Sprintf("%d:9090", m.localPort))...) @@ -97,7 +93,6 @@ func (m *Manager) StartPortForward(ctx context.Context) error { return fmt.Errorf("starting port-forward: %w", err) } - // Wait for port-forward to be ready for i := 0; i < 30; i++ { time.Sleep(time.Second) if m.isAccessible() { @@ -115,7 +110,6 @@ func (m *Manager) StopPortForward() { m.portForward.Process.Kill() m.portForward = nil } - // Also kill any lingering port-forwards exec.Command("pkill", "-f", fmt.Sprintf("kubectl port-forward.*prometheus.*%d", m.localPort)).Run() } @@ -123,12 +117,10 @@ func (m *Manager) StopPortForward() { func (m *Manager) Reset(ctx context.Context) error { m.StopPortForward() - // Delete Prometheus pod to reset metrics cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("delete", "pod", "-n", "monitoring", "-l", "app=prometheus", "--grace-period=0", "--force")...) - cmd.Run() // Ignore errors + cmd.Run() - // Wait for new pod fmt.Println("Waiting for Prometheus to restart...") waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) @@ -136,12 +128,10 @@ func (m *Manager) Reset(ctx context.Context) error { return fmt.Errorf("waiting for prometheus restart: %w", err) } - // Restart port-forward if err := m.StartPortForward(ctx); err != nil { return err } - // Wait for scraping to initialize fmt.Println("Waiting 5s for Prometheus to initialize scraping...") time.Sleep(5 * time.Second) @@ -155,7 +145,6 @@ func (m *Manager) isAccessible() bool { } conn.Close() - // Also try HTTP resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/status/config", m.localPort)) if err != nil { return false @@ -186,7 +175,6 @@ func (m *Manager) WaitForTarget(ctx context.Context, job string, timeout time.Du } } - // Print debug info on timeout m.printTargetStatus(job) return fmt.Errorf("timeout waiting for Prometheus to scrape job '%s'", job) } @@ -338,7 +326,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s // For S6 (restart scenario), use increase() to handle counter resets useIncrease := scenario == "S6" - // Counter metrics counterMetrics := []string{ "reloader_reconcile_total", "reloader_action_total", @@ -363,7 +350,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // Histogram percentiles histogramMetrics := []struct { name string prefix string @@ -384,7 +370,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // REST client metrics restQueries := map[string]string{ "rest_client_requests_total.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s"})`, job), "rest_client_requests_get.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="GET"})`, job), @@ -399,30 +384,23 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // Resource consumption metrics (memory, CPU, goroutines) resourceQueries := map[string]string{ - // Memory metrics (in bytes) "memory_rss_bytes_avg.json": fmt.Sprintf(`avg_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), "memory_rss_bytes_max.json": fmt.Sprintf(`max_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), "memory_rss_bytes_cur.json": fmt.Sprintf(`process_resident_memory_bytes{job="%s"}`, job), - // Heap memory (Go runtime) "memory_heap_bytes_avg.json": fmt.Sprintf(`avg_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), "memory_heap_bytes_max.json": fmt.Sprintf(`max_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), - // CPU metrics (rate of CPU seconds used) "cpu_usage_cores_avg.json": fmt.Sprintf(`rate(process_cpu_seconds_total{job="%s"}[%s])`, job, timeRange), "cpu_usage_cores_max.json": fmt.Sprintf(`max_over_time(rate(process_cpu_seconds_total{job="%s"}[1m])[%s:1m])`, job, timeRange), - // Goroutines (concurrency indicator) "goroutines_avg.json": fmt.Sprintf(`avg_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), "goroutines_max.json": fmt.Sprintf(`max_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), "goroutines_cur.json": fmt.Sprintf(`go_goroutines{job="%s"}`, job), - // GC metrics "gc_duration_seconds_p99.json": fmt.Sprintf(`histogram_quantile(0.99, sum(rate(go_gc_duration_seconds_bucket{job="%s"}[%s])) by (le))`, job, timeRange), - // Threads "threads_cur.json": fmt.Sprintf(`go_threads{job="%s"}`, job), } @@ -438,7 +416,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s func (m *Manager) queryAndSave(ctx context.Context, query, outputPath string) error { result, err := m.Query(ctx, query) if err != nil { - // Write empty result on error emptyResult := `{"status":"success","data":{"resultType":"vector","result":[]}}` return os.WriteFile(outputPath, []byte(emptyResult), 0644) } diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go index 794f1a688..4909feb10 100644 --- a/test/loadtest/internal/scenarios/scenarios.go +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -197,7 +197,6 @@ func (s *FanOutScenario) Run(ctx context.Context, client kubernetes.Interface, n log.Println("S2: Updating shared ConfigMap...") - // Check context state before starting update loop if ctx.Err() != nil { log.Printf("S2: WARNING - Context already done before update loop: %v", ctx.Err()) } @@ -503,14 +502,7 @@ func (s *WorkloadChurnScenario) Run(ctx context.Context, client kubernetes.Inter wg.Wait() log.Printf("S5: Created %d, deleted %d deployments, %d CM updates", deployCounter, deleteCounter, cmUpdateCount) - // S5 does NOT set expected values for action_total/reload_executed_total because: - // - There are ~10 active deployments at any time (creates new, deletes old) - // - Each CM update triggers reloads on ALL active deployments - // - Exact counts depend on timing of creates/deletes vs CM updates - // - "Not found" errors are expected when a deployment is deleted during processing - // Instead, S5 pass/fail compares old vs new (both should be similar) return ExpectedMetrics{ - // No expected values - churn makes exact counts unpredictable Description: fmt.Sprintf("S5: Churn test - %d deploys created, %d deleted, %d CM updates, ~10 active deploys at any time", deployCounter, deleteCounter, cmUpdateCount), }, nil } @@ -765,8 +757,6 @@ func (s *LargeObjectScenario) Run(ctx context.Context, client kubernetes.Interfa }, nil } -// Helper functions - func waitForDeploymentsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { log.Printf("Waiting for all deployments in %s to be ready (timeout: %v)...", namespace, timeout) @@ -1051,7 +1041,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int setupCtx := context.Background() - // Create Secrets for i := 0; i < numSecrets; i++ { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -1067,7 +1056,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1083,7 +1071,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create Secret-only deployments for i := 0; i < numSecretOnlyDeploys; i++ { deploy := createDeploymentWithSecret( fmt.Sprintf("secret-only-deploy-%d", i), @@ -1095,7 +1082,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create ConfigMap-only deployments for i := 0; i < numConfigMapOnlyDeploys; i++ { deploy := createDeployment( fmt.Sprintf("cm-only-deploy-%d", i), @@ -1107,7 +1093,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create mixed deployments (using both Secret and ConfigMap) for i := 0; i < numMixedDeploys; i++ { deploy := createDeploymentWithBoth( fmt.Sprintf("mixed-deploy-%d", i), @@ -1131,7 +1116,7 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() - updateSecret := true // Alternate between Secret and ConfigMap updates + updateSecret := true endTime := time.Now().Add(duration - 5*time.Second) for time.Now().Before(endTime) { @@ -1140,7 +1125,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil case <-ticker.C: if updateSecret { - // Update a random Secret secretIndex := rand.Intn(numSecrets) secret, err := client.CoreV1().Secrets(namespace).Get(setupCtx, fmt.Sprintf("mixed-secret-%d", secretIndex), metav1.GetOptions{}) if err != nil { @@ -1153,7 +1137,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int secretUpdateCount++ } } else { - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("mixed-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1173,11 +1156,9 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, numSecrets, numConfigMaps, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { - // Average deploys triggered per random secret update avgSecretReloads := float64(secretOnlyDeploys)/float64(numSecrets) + float64(mixedDeploys)/float64(numSecrets) secretTriggeredReloads := int(float64(secretUpdates) * avgSecretReloads) - // Average deploys triggered per random CM update avgCMReloads := float64(cmOnlyDeploys)/float64(numConfigMaps) + float64(mixedDeploys)/float64(numConfigMaps) cmTriggeredReloads := int(float64(cmUpdates) * avgCMReloads) @@ -1208,7 +1189,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I setupCtx := context.Background() - // Create shared ConfigMap cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "multi-type-cm", @@ -1220,7 +1200,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) } - // Create Deployments for i := 0; i < numDeployments; i++ { deploy := createDeployment(fmt.Sprintf("multi-deploy-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { @@ -1228,7 +1207,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Create StatefulSets for i := 0; i < numStatefulSets; i++ { sts := createStatefulSet(fmt.Sprintf("multi-sts-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().StatefulSets(namespace).Create(setupCtx, sts, metav1.CreateOptions{}); err != nil { @@ -1236,7 +1214,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Create DaemonSets for i := 0; i < numDaemonSets; i++ { ds := createDaemonSet(fmt.Sprintf("multi-ds-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().DaemonSets(namespace).Create(setupCtx, ds, metav1.CreateOptions{}); err != nil { @@ -1244,7 +1221,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Wait for workloads to be ready if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { log.Printf("Warning: %v - continuing anyway", err) } @@ -1286,7 +1262,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } func (s *MultiWorkloadTypeScenario) calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets int) ExpectedMetrics { - // Each CM update triggers reload on all workloads totalWorkloads := numDeployments + numStatefulSets + numDaemonSets expectedReloads := updateCount * totalWorkloads @@ -1376,7 +1351,6 @@ func createDaemonSet(name, namespace, configMapName string) *appsv1.DaemonSet { }, Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: &terminationGracePeriod, - // Use tolerations to run on all nodes including control-plane Tolerations: []corev1.Toleration{ { Key: "node-role.kubernetes.io/control-plane", @@ -1509,7 +1483,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I setupCtx := context.Background() - // Create ConfigMaps with multiple keys for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1527,9 +1500,7 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I } } - // Create complex deployments with various reference types for i := 0; i < numDeployments; i++ { - // Each deployment references multiple ConfigMaps in different ways primaryCM := fmt.Sprintf("complex-cm-%d", i) secondaryCM := fmt.Sprintf("complex-cm-%d", (i+1)%numConfigMaps) @@ -1560,7 +1531,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I case <-ctx.Done(): return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("complex-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1582,11 +1552,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I } func (s *ComplexReferencesScenario) calculateExpected(updateCount, numConfigMaps, numDeployments int) ExpectedMetrics { - // Each ConfigMap is referenced by: - // - 1 deployment as primary (envFrom in init + valueFrom in main + volume mount) - // - 1 deployment as secondary (projected volume) - // So each CM update triggers 2 deployments (on average with random updates) - // But since we're randomly updating, each update affects those 2 deployments expectedReloadsPerUpdate := 2 expectedReloads := updateCount * expectedReloadsPerUpdate @@ -1616,7 +1581,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa setupCtx := context.Background() - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1630,7 +1594,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa } } - // Create Deployments with pause-period annotation for i := 0; i < numDeployments; i++ { deploy := createDeploymentWithPause( fmt.Sprintf("pause-deploy-%d", i), @@ -1659,7 +1622,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa case <-ctx.Done(): return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("pause-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1679,12 +1641,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa } func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updateInterval, pausePeriod time.Duration) ExpectedMetrics { - // With pause-period, we expect fewer reloads than updates - // Each deployment gets updates at random, and pause-period prevents rapid consecutive reloads - // The exact count depends on the distribution of updates across ConfigMaps - // Rough estimate: each CM gets updated ~(updateCount/10) times - // With 15s pause and 2s interval, we get roughly 1 reload per pause period per CM - // So expected reloads ≈ duration / pausePeriod per deployment = (duration/pausePeriod) * numDeployments // This is an approximation - the actual value depends on random distribution expectedCycles := int(duration / pausePeriod) @@ -1693,8 +1649,6 @@ func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updat } return ExpectedMetrics{ - // Don't set exact expected values since pause-period makes counts unpredictable - // The scenario validates that reloads << updates due to pause behavior Description: fmt.Sprintf("S12: %d updates with %v pause-period (expect ~%d reload cycles, actual reloads << updates)", updateCount, pausePeriod, expectedCycles), } @@ -1703,7 +1657,6 @@ func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updat // AnnotationStrategyScenario - Tests annotation-based reload strategy. // This scenario deploys its own Reloader instance with --reload-strategy=annotations. type AnnotationStrategyScenario struct { - // Image is the Reloader image to use. Must be set before running. Image string } @@ -1719,7 +1672,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. log.Println("S11: Deploying Reloader with --reload-strategy=annotations...") - // Deploy S11's own Reloader instance reloaderNS := "reloader-s11" mgr := reloader.NewManager(reloader.Config{ Version: "s11", @@ -1732,7 +1684,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. return ExpectedMetrics{}, fmt.Errorf("deploying S11 reloader: %w", err) } - // Ensure cleanup on exit defer func() { log.Println("S11: Cleaning up S11-specific Reloader...") cleanupCtx := context.Background() @@ -1748,7 +1699,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. setupCtx := context.Background() - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1762,7 +1712,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. } } - // Create Deployments for i := 0; i < numDeployments; i++ { deploy := createDeployment(fmt.Sprintf("annot-deploy-%d", i), namespace, fmt.Sprintf("annot-cm-%d", i)) if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { @@ -1781,13 +1730,12 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() - endTime := time.Now().Add(duration - 10*time.Second) // Extra time for cleanup + endTime := time.Now().Add(duration - 10*time.Second) for time.Now().Before(endTime) { select { case <-ctx.Done(): return s.calculateExpected(updateCount, annotationUpdatesSeen), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("annot-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1800,7 +1748,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. updateCount++ } - // Periodically check for annotation updates on deployments if updateCount%10 == 0 { deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", cmIndex), metav1.GetOptions{}) if err == nil { @@ -1812,9 +1759,8 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. } } - // Final check: verify annotation strategy is working log.Println("S11: Verifying annotation-based reload...") - time.Sleep(5 * time.Second) // Allow time for final updates to propagate + time.Sleep(5 * time.Second) deploysWithAnnotation := 0 for i := 0; i < numDeployments; i++ { @@ -1945,7 +1891,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: &terminationGracePeriod, - // Init container using envFrom InitContainers: []corev1.Container{ { Name: "init", @@ -1973,7 +1918,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, Containers: []corev1.Container{ - // Main container using valueFrom (individual keys) { Name: "main", Image: "gcr.io/google-containers/busybox:1.27", @@ -2013,7 +1957,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, }, - // Sidecar using volume mount { Name: "sidecar", Image: "gcr.io/google-containers/busybox:1.27", @@ -2041,7 +1984,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, Volumes: []corev1.Volume{ - // Regular ConfigMap volume { Name: "config-volume", VolumeSource: corev1.VolumeSource{ @@ -2052,7 +1994,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, }, - // Projected volume combining multiple ConfigMaps { Name: "projected-volume", VolumeSource: corev1.VolumeSource{ From 16ff7f6ac972eb01577fc96d37dce3d2eb8febe7 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 15:27:19 +0100 Subject: [PATCH 21/21] fix: Default reconcile metric result to error for panic safety --- internal/pkg/handler/create.go | 33 +++++++++++++------------- internal/pkg/handler/delete.go | 33 +++++++++++++------------- internal/pkg/handler/update.go | 42 +++++++++++++++++----------------- 3 files changed, 53 insertions(+), 55 deletions(-) diff --git a/internal/pkg/handler/create.go b/internal/pkg/handler/create.go index 5fd3014e0..d67661007 100644 --- a/internal/pkg/handler/create.go +++ b/internal/pkg/handler/create.go @@ -27,7 +27,7 @@ func (r ResourceCreatedHandler) GetEnqueueTime() time.Time { // Handle processes the newly created resource func (r ResourceCreatedHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -35,25 +35,24 @@ func (r ResourceCreatedHandler) Handle() error { if r.Resource == nil { logrus.Errorf("Resource creation handler received nil resource") - result = "error" - } else { - config, _ := r.GetConfig() - // Send webhook - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) - if err != nil { - result = "error" + return nil + } + + config, _ := r.GetConfig() + // Send webhook + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err } - return nil + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err == nil { + result = "success" + } + return err } // GetConfig gets configurations containing SHA, annotations, namespace and resource name diff --git a/internal/pkg/handler/delete.go b/internal/pkg/handler/delete.go index 243602c57..34e032b7f 100644 --- a/internal/pkg/handler/delete.go +++ b/internal/pkg/handler/delete.go @@ -35,7 +35,7 @@ func (r ResourceDeleteHandler) GetEnqueueTime() time.Time { // Handle processes resources being deleted func (r ResourceDeleteHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -43,25 +43,24 @@ func (r ResourceDeleteHandler) Handle() error { if r.Resource == nil { logrus.Errorf("Resource delete handler received nil resource") - result = "error" - } else { - config, _ := r.GetConfig() - // Send webhook - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) - if err != nil { - result = "error" + return nil + } + + config, _ := r.GetConfig() + // Send webhook + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err } - return nil + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + if err == nil { + result = "success" + } + return err } // GetConfig gets configurations containing SHA, annotations, namespace and resource name diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index 1d0403b17..3fde98e3a 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -30,7 +30,7 @@ func (r ResourceUpdatedHandler) GetEnqueueTime() time.Time { // Handle processes the updated resource func (r ResourceUpdatedHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -38,30 +38,30 @@ func (r ResourceUpdatedHandler) Handle() error { if r.Resource == nil || r.OldResource == nil { logrus.Errorf("Resource update handler received nil resource") - result = "error" - } else { - config, oldSHAData := r.GetConfig() - if config.SHAValue != oldSHAData { - // Send a webhook if update - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) - if err != nil { - result = "error" + return nil + } + + config, oldSHAData := r.GetConfig() + if config.SHAValue != oldSHAData { + // Send a webhook if update + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err - } else { - // No data change - skip - result = "skipped" - r.Collectors.RecordSkipped("no_data_change") } + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err == nil { + result = "success" + } + return err } + + // No data change - skip + result = "skipped" + r.Collectors.RecordSkipped("no_data_change") return nil }