diff --git a/hack/benchmark-external-certs/00-run-all.sh b/hack/benchmark-external-certs/00-run-all.sh new file mode 100755 index 0000000000..6d41ee058d --- /dev/null +++ b/hack/benchmark-external-certs/00-run-all.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# 00-run-all.sh +# Full OCPBUGS-77056 benchmark orchestrator. +# +# Phases (each starts with a wipe for idempotency): +# 0. quay login + preflight checks +# 1. Wipe + create test routes (extcert-bench namespace) +# 2. BASELINE: restart existing cluster router, measure startup +# 3. PATCHED: deploy pre-built patched image, restart, measure startup +# 4. Print comparison table +# 5. Cleanup prompt +# +# !! DO NOT COMMIT with QUAY_PASSWORD set !! +# Usage: ./00-run-all.sh [--count 200] [--namespace extcert-bench] + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Config ──────────────────────────────────────────────────────────────────── +ROUTE_COUNT="${ROUTE_COUNT:-200}" +NAMESPACE="${BENCH_NAMESPACE:-extcert-bench}" +REGISTRY="${ROUTER_IMAGE_REGISTRY:-quay.io/btofel/router}" +PATCHED_IMAGE="${REGISTRY}:patched" +QUAY_USER="${QUAY_USER:-btofel}" +QUAY_PASSWORD="${QUAY_PASSWORD:-UZF9gy4jrRp0l+/gh0zqK6HJx39MEZ3k57m3CDs2I8BrnGkmAttI63KiXFyn7/XV}" + +while [[ $# -gt 0 ]]; do + case $1 in + --count) ROUTE_COUNT="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +banner() { + echo "" + echo "══════════════════════════════════════════════════════════════" + printf " %s\n" "$*" + echo "══════════════════════════════════════════════════════════════" +} + +read_result() { grep "^${2}=" "${1}" 2>/dev/null | cut -d= -f2 || echo "N/A"; } + +# ── Phase 0: Preflight ──────────────────────────────────────────────────────── +banner "PHASE 0: Preflight" + +echo "==> Checking oc connectivity" +oc whoami || { echo "ERROR: not logged into a cluster"; exit 1; } +oc get nodes --no-headers | head -3 + +echo "==> Logging into quay.io" +podman login -u="${QUAY_USER}" -p="${QUAY_PASSWORD}" quay.io + +echo "==> Config: count=${ROUTE_COUNT} namespace=${NAMESPACE} image=${PATCHED_IMAGE}" + +# ── Phase 1: Wipe + setup routes ───────────────────────────────────────────── +banner "PHASE 1: Route setup (wipe + create ${ROUTE_COUNT} external-cert routes)" +bash "${SCRIPT_DIR}/01-setup-routes.sh" \ + --count "$ROUTE_COUNT" \ + --namespace "$NAMESPACE" + +# ── Phase 2: Baseline benchmark (existing cluster router) ──────────────────── +banner "PHASE 2: BASELINE benchmark (existing cluster router — bug present)" +# Ensure we are actually evaluating the baseline buggy image by restoring it +bash "${SCRIPT_DIR}/03-run-benchmark.sh" --restore +bash "${SCRIPT_DIR}/03-run-benchmark.sh" --existing +BASELINE_FILE=$(ls -t "${SCRIPT_DIR}"/benchmark-results-existing-*.txt 2>/dev/null | head -1 || \ + ls -t benchmark-results-existing-*.txt 2>/dev/null | head -1 || echo "") + +# ── Phase 3: Patched benchmark ─────────────────────────────────────────────── +banner "PHASE 3: PATCHED benchmark (fixed library-go)" +# Wipe and recreate routes so the router sees the same load as the baseline run +bash "${SCRIPT_DIR}/01-setup-routes.sh" \ + --count "$ROUTE_COUNT" \ + --namespace "$NAMESPACE" +bash "${SCRIPT_DIR}/03-run-benchmark.sh" --image "$PATCHED_IMAGE" +PATCHED_FILE=$(ls -t "${SCRIPT_DIR}"/benchmark-results-patched-*.txt 2>/dev/null | head -1 || \ + ls -t benchmark-results-patched-*.txt 2>/dev/null | head -1 || echo "") + +# ── Phase 4: Comparison ─────────────────────────────────────────────────────── +banner "PHASE 4: Results comparison" +if [[ -f "${BASELINE_FILE:-}" && -f "${PATCHED_FILE:-}" ]]; then + B_SECRETS=$(read_result "$BASELINE_FILE" secret_count) + B_LOAD=$(read_result "$BASELINE_FILE" secret_load_seconds) + B_READY=$(read_result "$BASELINE_FILE" pod_to_ready_seconds) + P_SECRETS=$(read_result "$PATCHED_FILE" secret_count) + P_LOAD=$(read_result "$PATCHED_FILE" secret_load_seconds) + P_READY=$(read_result "$PATCHED_FILE" pod_to_ready_seconds) + + printf "\n %-32s %-16s %-16s\n" "Metric" "BASELINE (bug)" "PATCHED (fix)" + printf " %-32s %-16s %-16s\n" "--------------------------------" "----------------" "----------------" + printf " %-32s %-16s %-16s\n" "Secrets loaded" "$B_SECRETS" "$P_SECRETS" + printf " %-32s %-16s %-16s\n" "Secret loading span (s)" "$B_LOAD" "$P_LOAD" + printf " %-32s %-16s %-16s\n" "Pod → Ready wall time (s)" "$B_READY" "$P_READY" + echo "" + + python3 -c " +b, p = '${B_LOAD}', '${P_LOAD}' +try: + ratio = float(b)/float(p) + print(f' Speedup (secret loading): {ratio:.1f}x faster with the fix') + print(f' {float(b):.0f}s → {float(p):.0f}s') +except: pass +" 2>/dev/null || true + + echo "" + echo " Baseline file : $BASELINE_FILE" + echo " Patched file : $PATCHED_FILE" +else + echo " (result files not found; check benchmark-results-*.txt)" +fi + +# ── Phase 5: Cleanup ───────────────────────────────────────────────────────── +banner "PHASE 5: Cleanup" +read -rp " Clean up test namespace and restore original router? [y/N] " CONFIRM +if [[ "${CONFIRM,,}" == "y" ]]; then + bash "${SCRIPT_DIR}/04-cleanup.sh" --namespace "$NAMESPACE" +else + echo " Skipped. Run: ./04-cleanup.sh --namespace $NAMESPACE" +fi diff --git a/hack/benchmark-external-certs/01-setup-routes.sh b/hack/benchmark-external-certs/01-setup-routes.sh new file mode 100755 index 0000000000..c020927864 --- /dev/null +++ b/hack/benchmark-external-certs/01-setup-routes.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# 01-setup-routes.sh +# Idempotent: always wipes and recreates the namespace, then creates N +# TLS secrets + routes with spec.tls.externalCertificate. +# +# Usage: ./01-setup-routes.sh [--count 200] [--namespace extcert-bench] + +set -euo pipefail + +COUNT=200 +NAMESPACE=extcert-bench +APP_DOMAIN="" + +while [[ $# -gt 0 ]]; do + case $1 in + --count) COUNT="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + --domain) APP_DOMAIN="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +# ── Wipe existing namespace (idempotent) ────────────────────────────────────── +echo "==> Wiping namespace $NAMESPACE (if exists)" +oc delete ns "$NAMESPACE" --ignore-not-found --wait=true 2>/dev/null || true + +echo "==> Creating namespace $NAMESPACE" +oc create ns "$NAMESPACE" + +# ── Auto-detect cluster app domain ─────────────────────────────────────────── +if [[ -z "$APP_DOMAIN" ]]; then + APP_DOMAIN=$(oc get ingresses.config.openshift.io cluster \ + -o jsonpath='{.spec.domain}' 2>/dev/null || echo "apps.example.com") + echo "==> Detected app domain: $APP_DOMAIN" +fi + +# ── RBAC: custom Role granting the router SA access to secrets ─────────────── +# NOTE: system:router clusterrole does NOT include secrets — must use custom Role. +echo "==> Creating RBAC for router secret access" +oc apply -n "$NAMESPACE" -f - < Generating test TLS certificate" +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT +openssl req -x509 -newkey rsa:2048 \ + -keyout "$TMPDIR/tls.key" -out "$TMPDIR/tls.crt" \ + -days 365 -nodes \ + -subj "/CN=bench.${APP_DOMAIN}" \ + -addext "subjectAltName=DNS:bench.${APP_DOMAIN}" \ + 2>/dev/null + +# ── Create secrets, services, and routes ───────────────────────────────────── +echo "==> Creating $COUNT secrets and routes in namespace $NAMESPACE" +CREATED=0 +for i in $(seq 1 "$COUNT"); do + NAME="extcert-bench-${i}" + + oc create secret tls "$NAME" \ + --cert="$TMPDIR/tls.crt" \ + --key="$TMPDIR/tls.key" \ + -n "$NAMESPACE" 2>/dev/null + + oc create service clusterip "$NAME" --tcp=8080:8080 \ + -n "$NAMESPACE" 2>/dev/null + + oc apply -n "$NAMESPACE" -f - 2>/dev/null < Done: $COUNT external-cert routes in namespace $NAMESPACE" +echo " Secrets: $(oc get secrets -n $NAMESPACE --no-headers | wc -l | tr -d ' ')" +echo " Routes: $(oc get routes -n $NAMESPACE --no-headers | wc -l | tr -d ' ')" diff --git a/hack/benchmark-external-certs/02-build-patched.sh b/hack/benchmark-external-certs/02-build-patched.sh new file mode 100755 index 0000000000..c77280d04d --- /dev/null +++ b/hack/benchmark-external-certs/02-build-patched.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# 02-build-patched.sh +# Builds the router binary with the patched library-go (OCPBUGS-77056 fix) +# and pushes to quay.io/btofel/router:patched. +# +# The BASELINE is the existing cluster router — no build needed for that. +# +# Usage: ./02-build-patched.sh [--registry quay.io/btofel/router] + +set -euo pipefail + +REGISTRY="${ROUTER_IMAGE_REGISTRY:-quay.io/btofel/router}" +IMAGE="${REGISTRY}:patched" +ROUTER_DIR="/Users/btofel/workspace/router" +LIBRARYGO_DIR="/Users/btofel/workspace/library-go" +CONTAINER_TOOL="${CONTAINER_TOOL:-podman}" + +while [[ $# -gt 0 ]]; do + case $1 in + --registry) REGISTRY="$2"; IMAGE="${REGISTRY}:patched"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +echo "==> Building patched router image: $IMAGE" +echo " Router: $ROUTER_DIR" +echo " Library-go: $LIBRARYGO_DIR (fixed branch: $(git -C $LIBRARYGO_DIR branch --show-current))" + +# Work in a temp copy to avoid dirtying the real workspace +WORKDIR=$(mktemp -d) +trap "rm -rf $WORKDIR" EXIT +cp -a "$ROUTER_DIR/." "$WORKDIR/" +cd "$WORKDIR" + +# ── Apply local library-go replace directive ────────────────────────────────── +echo "==> Applying library-go replace directive in go.mod" +LIBGO_MOD=$(grep '^module ' "$LIBRARYGO_DIR/go.mod" | awk '{print $2}') +if grep -q "replace.*library-go" go.mod 2>/dev/null; then + # Update existing replace line (macOS-compatible sed) + sed -i '' "s|replace ${LIBGO_MOD} =>.*|replace ${LIBGO_MOD} => ${LIBRARYGO_DIR}|" go.mod +else + printf '\nreplace %s => %s\n' "$LIBGO_MOD" "$LIBRARYGO_DIR" >> go.mod +fi + +echo "==> Running go mod tidy & vendor..." +go mod tidy 2>&1 | tail -5 || true +go mod vendor 2>&1 | tail -5 + +# Verify the fix is present in vendor +if grep -q "secret cache not synced yet" vendor/github.com/openshift/library-go/pkg/secret/secret_monitor.go; then + echo "==> Verified: patched secret_monitor.go in vendor" +else + echo "ERROR: patched library-go not found in vendor" >&2; exit 1 +fi + +# ── Build binary ────────────────────────────────────────────────────────────── +echo "==> Compiling openshift-router (linux/amd64)..." +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -o openshift-router -ldflags '-w -s' ./cmd/openshift-router +echo "==> Binary: $(ls -lh openshift-router | awk '{print $5}')" + +# ── Build container image ───────────────────────────────────────────────────── +BASE_IMAGE=$(oc get deployment router-default -n openshift-ingress -o jsonpath='{.spec.template.spec.containers[0].image}') +echo "==> Building container from $BASE_IMAGE" +IMGCTX=$(mktemp -d) +cp openshift-router "$IMGCTX/" +cat > "$IMGCTX/Dockerfile" < Pushing $IMAGE" +$CONTAINER_TOOL push "$IMAGE" +echo "==> Done: $IMAGE" diff --git a/hack/benchmark-external-certs/03-run-benchmark.sh b/hack/benchmark-external-certs/03-run-benchmark.sh new file mode 100755 index 0000000000..4c63849c6f --- /dev/null +++ b/hack/benchmark-external-certs/03-run-benchmark.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# 03-run-benchmark.sh +# Restarts the router, waits for it to become Ready, and measures: +# - Secret loading span (from router logs: first→last informer sync) +# - Pod→Ready wall-clock time +# +# Two modes: +# --existing Benchmark the cluster's current router (baseline, no image change) +# --image IMG Deploy a custom image first, then benchmark (patched) +# +# Usage: +# ./03-run-benchmark.sh --existing # baseline +# ./03-run-benchmark.sh --image quay.io/btofel/router:patched +# ./03-run-benchmark.sh --restore # put original image back + +set -euo pipefail + +MODE="" +IMAGE="" +RESTORE=false +INGRESS_NS=openshift-ingress +IC_NS=openshift-ingress-operator +TIMEOUT=600 +LABEL="ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default" + +while [[ $# -gt 0 ]]; do + case $1 in + --existing) MODE=existing; shift ;; + --image) MODE=patched; IMAGE="$2"; shift 2 ;; + --restore) RESTORE=true; shift ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +# ── Restore ─────────────────────────────────────────────────────────────────── +if [[ "$RESTORE" == "true" ]]; then + echo "==> Restoring original router image" + oc scale --replicas=1 deployment/ingress-operator -n "$IC_NS" + + # Wait a moment for the operator to wake up and trigger a rollout + sleep 5 + echo "==> Removing CVO override for Ingress Operator" + oc patch clusterversion version --type=json -p='[{"op":"remove","path":"/spec/overrides"}]' 2>/dev/null || true + + echo "==> Waiting for router to recover..." + oc rollout status deployment/router-default -n "$INGRESS_NS" --timeout="${TIMEOUT}s" + echo "==> Cleanup done." + exit 0 +fi + +if [[ -z "$MODE" ]]; then + echo "ERROR: specify --existing or --image " + exit 1 +fi + +TAG="$MODE" + +echo "" +echo "══════════════════════════════════════════════════════════════" +echo " OCPBUGS-77056 Benchmark — $TAG" +echo "══════════════════════════════════════════════════════════════" + +# ── Optionally swap the image ───────────────────────────────────────────────── +if [[ "$MODE" == "patched" ]]; then + echo "==> Pausing Ingress Operator to prevent image revert..." + oc patch clusterversion version --type=merge -p '{"spec":{"overrides":[{"kind":"Deployment","group":"apps","name":"ingress-operator","namespace":"openshift-ingress-operator","unmanaged":true}]}}' + oc scale --replicas=0 deployment/ingress-operator -n "$IC_NS" + sleep 3 # Give it a moment to terminate + + echo "==> Deploying image: $IMAGE" + oc set image deployment/router-default "router=${IMAGE}" -n "$INGRESS_NS" + + echo "==> Forcing imagePullPolicy to Always" + oc patch deployment/router-default -n "$INGRESS_NS" --type=strategic -p '{"spec":{"template":{"spec":{"containers":[{"name":"router","imagePullPolicy":"Always"}]}}}}' +fi + +# ── Restart and time it ─────────────────────────────────────────────────────── +echo "==> Restarting router-default" +oc rollout restart deployment/router-default -n "$INGRESS_NS" + +READY_START=$(date +%s) + +# Wait for the new pod to appear +echo -n "==> Waiting for new pod to be created: " +NEW_POD="" +for i in $(seq 1 90); do + sleep 2 + echo -n "." + # Get the most-recently-created running/pending pod + NEW_POD=$(oc get pods -n "$INGRESS_NS" -l "$LABEL" \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{.items[-1:].metadata.name}' 2>/dev/null || true) + if [[ -n "$NEW_POD" ]]; then + POD_PHASE=$(oc get pod "$NEW_POD" -n "$INGRESS_NS" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "$POD_PHASE" == "Running" || "$POD_PHASE" == "Pending" ]]; then + echo " Found: $NEW_POD (Phase: $POD_PHASE)" + break + fi + fi +done + +if [[ -z "$NEW_POD" || "$POD_PHASE" == "" ]]; then + echo " ERROR: Timed out waiting for a new pod to appear!" + exit 1 +fi + +echo "==> Tracking pod: $NEW_POD" + +# Wait for rollout ready +echo "==> Waiting for deployment rollout to finish (timeout ${TIMEOUT}s)..." +if ! oc rollout status deployment/router-default -n "$INGRESS_NS" --timeout="${TIMEOUT}s"; then + echo "ERROR: router did not become ready in ${TIMEOUT}s" + oc logs "$NEW_POD" -n "$INGRESS_NS" 2>/dev/null | tail -20 + exit 1 +fi + +READY_END=$(date +%s) +READY_ELAPSED=$((READY_END - READY_START)) + +# ── Parse log-based timing ──────────────────────────────────────────────────── +sleep 3 # let logs flush +LOG=$(oc logs "$NEW_POD" -n "$INGRESS_NS" 2>/dev/null || true) + +FIRST_TS=$(echo "$LOG" | grep -E 'starting informer|Starting informer' | head -1 | awk '{print $1, $2}' || true) +LAST_TS=$(echo "$LOG" | grep -E 'secret informer started|secret handler added' | tail -1 | awk '{print $1, $2}' || true) +SECRET_COUNT=$(echo "$LOG" | grep -c 'secret informer started' || true) + +SECRET_LOAD_SECS="N/A" +if [[ -n "$FIRST_TS" && -n "$LAST_TS" ]]; then + SECRET_LOAD_SECS=$(python3 -c " +from datetime import datetime +import sys +try: + ts1 = '${FIRST_TS}' # e.g. I0302 15:26:54.629672 + ts2 = '${LAST_TS}' + if not ts1 or not ts2: sys.exit(0) + + y = datetime.now().year + # remove 'I', 'W', 'E' prefix from month + d1 = f\"{y}{ts1[1:5]} {ts1.split()[1]}\" + d2 = f\"{y}{ts2[1:5]} {ts2.split()[1]}\" + + a = datetime.strptime(d1, '%Y%m%d %H:%M:%S.%f') + b = datetime.strptime(d2, '%Y%m%d %H:%M:%S.%f') + print(f'{(b-a).total_seconds():.2f}') +except Exception as e: + print('N/A') +" 2>/dev/null || echo "N/A") +fi + +# ── Print results ───────────────────────────────────────────────────────────── +echo "" +echo "══════════════════════════════════════════════════════════════" +echo " RESULTS — $TAG" +echo "══════════════════════════════════════════════════════════════" +printf " %-30s %s\n" "Pod:" "$NEW_POD" +printf " %-30s %s\n" "Image:" "${IMAGE:-}" +printf " %-30s %s\n" "Secrets loaded:" "$SECRET_COUNT" +printf " %-30s %ss\n" "Secret loading span:" "$SECRET_LOAD_SECS" +printf " %-30s %ss\n" "Pod → Ready (wall):" "$READY_ELAPSED" +echo "══════════════════════════════════════════════════════════════" + +# Save result file +RESULT_FILE="benchmark-results-${TAG}-$(date +%Y%m%d-%H%M%S).txt" +cat > "$RESULT_FILE" < Results saved: $RESULT_FILE" diff --git a/hack/benchmark-external-certs/04-cleanup.sh b/hack/benchmark-external-certs/04-cleanup.sh new file mode 100755 index 0000000000..850cf9621b --- /dev/null +++ b/hack/benchmark-external-certs/04-cleanup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# 04-cleanup.sh +# Removes test namespace and restores the original router image. +# Usage: ./04-cleanup.sh [--namespace extcert-bench] [--skip-router-restore] + +set -euo pipefail + +NAMESPACE=extcert-bench +SKIP_RESTORE=false +INGRESS_NS=openshift-ingress +IC_NS=openshift-ingress-operator + +while [[ $# -gt 0 ]]; do + case $1 in + --namespace) NAMESPACE="$2"; shift 2 ;; + --skip-router-restore) SKIP_RESTORE=true; shift ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +echo "==> Deleting namespace $NAMESPACE" +oc delete ns "$NAMESPACE" --ignore-not-found --wait=true 2>/dev/null || true + +if [[ "$SKIP_RESTORE" == "false" ]]; then + echo "==> Restoring original router image" + oc scale --replicas=1 deployment/ingress-operator -n "$IC_NS" + + echo "==> Removing CVO override for Ingress Operator" + oc patch clusterversion version --type=json -p='[{"op":"remove","path":"/spec/overrides"}]' 2>/dev/null || true + + echo "==> Waiting for router to recover..." + sleep 5 + oc rollout status deployment/router-default -n "$INGRESS_NS" --timeout=600s || true + + echo "==> Cleanup done." +fi diff --git a/hack/benchmark-external-certs/README.md b/hack/benchmark-external-certs/README.md new file mode 100644 index 0000000000..afc182dc03 --- /dev/null +++ b/hack/benchmark-external-certs/README.md @@ -0,0 +1,87 @@ +# External Certificate Route Benchmark Harness + +Benchmarks OpenShift router startup time with `spec.tls.externalCertificate` routes +to validate the fix for [OCPBUGS-77056](https://issues.redhat.com/browse/OCPBUGS-77056) +(sequential secret loading causing router startup times of 20–40 minutes at scale). + +## Background + +When a router starts with N external-cert routes, it registers a secret informer for each. +The bug: each registration holds a global write lock while waiting for the informer to sync +via etcd, serializing all N registrations. The fix (library-go PR #2132) releases the lock +before `WaitForCacheSync`, allowing concurrent loading. + +| Script | Purpose | +|---|---| +| `00-run-all.sh` | **Orchestrator** — runs all phases end-to-end | +| `01-setup-routes.sh` | Creates N TLS secrets + routes with `externalCertificate` (idempotent: wipes first) | +| `03-run-benchmark.sh` | Restarts router, measures secret-loading span + pod-ready wall time | +| `04-cleanup.sh` | Deletes test namespace, restores original router image | + +> **Note:** `00-run-all.sh` contains quay credentials — it is `.gitignore`d. +> **Note:** The patched image has been pre-built and pushed to `quay.io/btofel/router:patched`. + +## Quick Start (just run the orchestrator) + +```bash +./00-run-all.sh --count 200 +``` + +This runs four phases automatically: + +1. **Preflight** — verify `oc` login, login to quay.io +2. **Setup** — wipe + create 200 external-cert routes +3. **Baseline** — restart the existing cluster router, measure startup (this shows the bug) +4. **Patched** — deploy pre-built patched image, restart, measure startup (shows the fix) +5. **Compare** — print side-by-side results table + +``` + Metric BASELINE (bug) PATCHED (fix) + -------------------------------- ---------------- ---------------- + Secrets loaded 200 200 + Secret loading span (s) ~120s ~2s + Pod → Ready wall time (s) ~150s ~10s + + Speedup (secret loading): ~60x faster with the fix +``` + +## Options + +``` +--count N Number of routes to create (default: 200) +--namespace NAME Namespace for test routes (default: extcert-bench) +``` + +## Requirements + +- `oc` CLI logged into a cluster as `cluster-admin` +- `podman` with a running podman machine (`podman machine start`) +- `go` 1.22+ (for building the patched router) +- `openssl` (for self-signed test certs) +- Access to `openshift/router` at `/Users/btofel/workspace/router` +- Access to `openshift/library-go` at `/Users/btofel/workspace/library-go` + (on branch `fix-external-cert-serialization` with the OCPBUGS-77056 fix) +- `quay.io/btofel/router` repository created on quay.io and accessible from cluster nodes + +## What is measured + +- **Secret loading span**: time between first `starting informer` and last + `secret informer started` log line in the router pod — the precise window + where the serial vs. concurrent lock behavior matters +- **Pod → Ready wall time**: end-to-end time from restart to `Ready=True` + +## Running individual phases + +```bash +# Create routes only +./01-setup-routes.sh --count 200 + +# Benchmark existing router (baseline) +./03-run-benchmark.sh --existing + +# Benchmark custom image +./03-run-benchmark.sh --image quay.io/btofel/router:patched + +# Restore original router image and delete test namespace +./04-cleanup.sh +``` diff --git a/hack/benchmark-external-certs/benchmark-results-existing-20260303-162614.txt b/hack/benchmark-external-certs/benchmark-results-existing-20260303-162614.txt new file mode 100644 index 0000000000..8176067e46 --- /dev/null +++ b/hack/benchmark-external-certs/benchmark-results-existing-20260303-162614.txt @@ -0,0 +1,7 @@ +mode=existing +image=cluster-default +pod=router-default-97b7b594b-9c7pd +secret_count=100 +secret_load_seconds=58.10 +pod_to_ready_seconds=137 +timestamp=2026-03-03T21:26:14Z diff --git a/hack/benchmark-external-certs/benchmark-results-patched-20260303-162947.txt b/hack/benchmark-external-certs/benchmark-results-patched-20260303-162947.txt new file mode 100644 index 0000000000..5ff35086e0 --- /dev/null +++ b/hack/benchmark-external-certs/benchmark-results-patched-20260303-162947.txt @@ -0,0 +1,7 @@ +mode=patched +image=quay.io/btofel/router:patched +pod=router-default-7ff6c6557f-z7724 +secret_count=100 +secret_load_seconds=0.07 +pod_to_ready_seconds=110 +timestamp=2026-03-03T21:29:47Z diff --git a/hack/benchmark-external-certs/benchmark-run.log b/hack/benchmark-external-certs/benchmark-run.log new file mode 100644 index 0000000000..2f2b48faa6 --- /dev/null +++ b/hack/benchmark-external-certs/benchmark-run.log @@ -0,0 +1,769 @@ + +══════════════════════════════════════════════════════════════ + PHASE 0: Preflight +══════════════════════════════════════════════════════════════ +==> Checking oc connectivity +kube:admin +ip-10-0-10-7.ec2.internal Ready worker 5h57m v1.32.8 +ip-10-0-2-116.ec2.internal Ready control-plane,master 6h4m v1.32.8 +ip-10-0-53-34.ec2.internal Ready worker 5h57m v1.32.8 +==> Logging into quay.io +Login Succeeded! +==> Config: count=100 namespace=extcert-bench image=quay.io/btofel/router:patched + +══════════════════════════════════════════════════════════════ + PHASE 1: Route setup (wipe + create 100 external-cert routes) +══════════════════════════════════════════════════════════════ +==> Wiping namespace extcert-bench (if exists) +namespace "extcert-bench" deleted +==> Creating namespace extcert-bench +namespace/extcert-bench created +==> Detected app domain: apps.btofel-netedg-260303.devcluster.openshift.com +==> Creating RBAC for router secret access +role.rbac.authorization.k8s.io/router-secret-reader created +rolebinding.rbac.authorization.k8s.io/router-secret-reader created +==> Generating test TLS certificate +==> Creating 100 secrets and routes in namespace extcert-bench +secret/extcert-bench-1 created +service/extcert-bench-1 created +route.route.openshift.io/extcert-bench-1 created +secret/extcert-bench-2 created +service/extcert-bench-2 created +route.route.openshift.io/extcert-bench-2 created +secret/extcert-bench-3 created +service/extcert-bench-3 created +route.route.openshift.io/extcert-bench-3 created +secret/extcert-bench-4 created +service/extcert-bench-4 created +route.route.openshift.io/extcert-bench-4 created +secret/extcert-bench-5 created +service/extcert-bench-5 created +route.route.openshift.io/extcert-bench-5 created +secret/extcert-bench-6 created +service/extcert-bench-6 created +route.route.openshift.io/extcert-bench-6 created +secret/extcert-bench-7 created +service/extcert-bench-7 created +route.route.openshift.io/extcert-bench-7 created +secret/extcert-bench-8 created +service/extcert-bench-8 created +route.route.openshift.io/extcert-bench-8 created +secret/extcert-bench-9 created +service/extcert-bench-9 created +route.route.openshift.io/extcert-bench-9 created +secret/extcert-bench-10 created +service/extcert-bench-10 created +route.route.openshift.io/extcert-bench-10 created + ... created 10/100 routes +secret/extcert-bench-11 created +service/extcert-bench-11 created +route.route.openshift.io/extcert-bench-11 created +secret/extcert-bench-12 created +service/extcert-bench-12 created +route.route.openshift.io/extcert-bench-12 created +secret/extcert-bench-13 created +service/extcert-bench-13 created +route.route.openshift.io/extcert-bench-13 created +secret/extcert-bench-14 created +service/extcert-bench-14 created +route.route.openshift.io/extcert-bench-14 created +secret/extcert-bench-15 created +service/extcert-bench-15 created +route.route.openshift.io/extcert-bench-15 created +secret/extcert-bench-16 created +service/extcert-bench-16 created +route.route.openshift.io/extcert-bench-16 created +secret/extcert-bench-17 created +service/extcert-bench-17 created +route.route.openshift.io/extcert-bench-17 created +secret/extcert-bench-18 created +service/extcert-bench-18 created +route.route.openshift.io/extcert-bench-18 created +secret/extcert-bench-19 created +service/extcert-bench-19 created +route.route.openshift.io/extcert-bench-19 created +secret/extcert-bench-20 created +service/extcert-bench-20 created +route.route.openshift.io/extcert-bench-20 created + ... created 20/100 routes +secret/extcert-bench-21 created +service/extcert-bench-21 created +route.route.openshift.io/extcert-bench-21 created +secret/extcert-bench-22 created +service/extcert-bench-22 created +route.route.openshift.io/extcert-bench-22 created +secret/extcert-bench-23 created +service/extcert-bench-23 created +route.route.openshift.io/extcert-bench-23 created +secret/extcert-bench-24 created +service/extcert-bench-24 created +route.route.openshift.io/extcert-bench-24 created +secret/extcert-bench-25 created +service/extcert-bench-25 created +route.route.openshift.io/extcert-bench-25 created +secret/extcert-bench-26 created +service/extcert-bench-26 created +route.route.openshift.io/extcert-bench-26 created +secret/extcert-bench-27 created +service/extcert-bench-27 created +route.route.openshift.io/extcert-bench-27 created +secret/extcert-bench-28 created +service/extcert-bench-28 created +route.route.openshift.io/extcert-bench-28 created +secret/extcert-bench-29 created +service/extcert-bench-29 created +route.route.openshift.io/extcert-bench-29 created +secret/extcert-bench-30 created +service/extcert-bench-30 created +route.route.openshift.io/extcert-bench-30 created + ... created 30/100 routes +secret/extcert-bench-31 created +service/extcert-bench-31 created +route.route.openshift.io/extcert-bench-31 created +secret/extcert-bench-32 created +service/extcert-bench-32 created +route.route.openshift.io/extcert-bench-32 created +secret/extcert-bench-33 created +service/extcert-bench-33 created +route.route.openshift.io/extcert-bench-33 created +secret/extcert-bench-34 created +service/extcert-bench-34 created +route.route.openshift.io/extcert-bench-34 created +secret/extcert-bench-35 created +service/extcert-bench-35 created +route.route.openshift.io/extcert-bench-35 created +secret/extcert-bench-36 created +service/extcert-bench-36 created +route.route.openshift.io/extcert-bench-36 created +secret/extcert-bench-37 created +service/extcert-bench-37 created +route.route.openshift.io/extcert-bench-37 created +secret/extcert-bench-38 created +service/extcert-bench-38 created +route.route.openshift.io/extcert-bench-38 created +secret/extcert-bench-39 created +service/extcert-bench-39 created +route.route.openshift.io/extcert-bench-39 created +secret/extcert-bench-40 created +service/extcert-bench-40 created +route.route.openshift.io/extcert-bench-40 created + ... created 40/100 routes +secret/extcert-bench-41 created +service/extcert-bench-41 created +route.route.openshift.io/extcert-bench-41 created +secret/extcert-bench-42 created +service/extcert-bench-42 created +route.route.openshift.io/extcert-bench-42 created +secret/extcert-bench-43 created +service/extcert-bench-43 created +route.route.openshift.io/extcert-bench-43 created +secret/extcert-bench-44 created +service/extcert-bench-44 created +route.route.openshift.io/extcert-bench-44 created +secret/extcert-bench-45 created +service/extcert-bench-45 created +route.route.openshift.io/extcert-bench-45 created +secret/extcert-bench-46 created +service/extcert-bench-46 created +route.route.openshift.io/extcert-bench-46 created +secret/extcert-bench-47 created +service/extcert-bench-47 created +route.route.openshift.io/extcert-bench-47 created +secret/extcert-bench-48 created +service/extcert-bench-48 created +route.route.openshift.io/extcert-bench-48 created +secret/extcert-bench-49 created +service/extcert-bench-49 created +route.route.openshift.io/extcert-bench-49 created +secret/extcert-bench-50 created +service/extcert-bench-50 created +route.route.openshift.io/extcert-bench-50 created + ... created 50/100 routes +secret/extcert-bench-51 created +service/extcert-bench-51 created +route.route.openshift.io/extcert-bench-51 created +secret/extcert-bench-52 created +service/extcert-bench-52 created +route.route.openshift.io/extcert-bench-52 created +secret/extcert-bench-53 created +service/extcert-bench-53 created +route.route.openshift.io/extcert-bench-53 created +secret/extcert-bench-54 created +service/extcert-bench-54 created +route.route.openshift.io/extcert-bench-54 created +secret/extcert-bench-55 created +service/extcert-bench-55 created +route.route.openshift.io/extcert-bench-55 created +secret/extcert-bench-56 created +service/extcert-bench-56 created +route.route.openshift.io/extcert-bench-56 created +secret/extcert-bench-57 created +service/extcert-bench-57 created +route.route.openshift.io/extcert-bench-57 created +secret/extcert-bench-58 created +service/extcert-bench-58 created +route.route.openshift.io/extcert-bench-58 created +secret/extcert-bench-59 created +service/extcert-bench-59 created +route.route.openshift.io/extcert-bench-59 created +secret/extcert-bench-60 created +service/extcert-bench-60 created +route.route.openshift.io/extcert-bench-60 created + ... created 60/100 routes +secret/extcert-bench-61 created +service/extcert-bench-61 created +route.route.openshift.io/extcert-bench-61 created +secret/extcert-bench-62 created +service/extcert-bench-62 created +route.route.openshift.io/extcert-bench-62 created +secret/extcert-bench-63 created +service/extcert-bench-63 created +route.route.openshift.io/extcert-bench-63 created +secret/extcert-bench-64 created +service/extcert-bench-64 created +route.route.openshift.io/extcert-bench-64 created +secret/extcert-bench-65 created +service/extcert-bench-65 created +route.route.openshift.io/extcert-bench-65 created +secret/extcert-bench-66 created +service/extcert-bench-66 created +route.route.openshift.io/extcert-bench-66 created +secret/extcert-bench-67 created +service/extcert-bench-67 created +route.route.openshift.io/extcert-bench-67 created +secret/extcert-bench-68 created +service/extcert-bench-68 created +route.route.openshift.io/extcert-bench-68 created +secret/extcert-bench-69 created +service/extcert-bench-69 created +route.route.openshift.io/extcert-bench-69 created +secret/extcert-bench-70 created +service/extcert-bench-70 created +route.route.openshift.io/extcert-bench-70 created + ... created 70/100 routes +secret/extcert-bench-71 created +service/extcert-bench-71 created +route.route.openshift.io/extcert-bench-71 created +secret/extcert-bench-72 created +service/extcert-bench-72 created +route.route.openshift.io/extcert-bench-72 created +secret/extcert-bench-73 created +service/extcert-bench-73 created +route.route.openshift.io/extcert-bench-73 created +secret/extcert-bench-74 created +service/extcert-bench-74 created +route.route.openshift.io/extcert-bench-74 created +secret/extcert-bench-75 created +service/extcert-bench-75 created +route.route.openshift.io/extcert-bench-75 created +secret/extcert-bench-76 created +service/extcert-bench-76 created +route.route.openshift.io/extcert-bench-76 created +secret/extcert-bench-77 created +service/extcert-bench-77 created +route.route.openshift.io/extcert-bench-77 created +secret/extcert-bench-78 created +service/extcert-bench-78 created +route.route.openshift.io/extcert-bench-78 created +secret/extcert-bench-79 created +service/extcert-bench-79 created +route.route.openshift.io/extcert-bench-79 created +secret/extcert-bench-80 created +service/extcert-bench-80 created +route.route.openshift.io/extcert-bench-80 created + ... created 80/100 routes +secret/extcert-bench-81 created +service/extcert-bench-81 created +route.route.openshift.io/extcert-bench-81 created +secret/extcert-bench-82 created +service/extcert-bench-82 created +route.route.openshift.io/extcert-bench-82 created +secret/extcert-bench-83 created +service/extcert-bench-83 created +route.route.openshift.io/extcert-bench-83 created +secret/extcert-bench-84 created +service/extcert-bench-84 created +route.route.openshift.io/extcert-bench-84 created +secret/extcert-bench-85 created +service/extcert-bench-85 created +route.route.openshift.io/extcert-bench-85 created +secret/extcert-bench-86 created +service/extcert-bench-86 created +route.route.openshift.io/extcert-bench-86 created +secret/extcert-bench-87 created +service/extcert-bench-87 created +route.route.openshift.io/extcert-bench-87 created +secret/extcert-bench-88 created +service/extcert-bench-88 created +route.route.openshift.io/extcert-bench-88 created +secret/extcert-bench-89 created +service/extcert-bench-89 created +route.route.openshift.io/extcert-bench-89 created +secret/extcert-bench-90 created +service/extcert-bench-90 created +route.route.openshift.io/extcert-bench-90 created + ... created 90/100 routes +secret/extcert-bench-91 created +service/extcert-bench-91 created +route.route.openshift.io/extcert-bench-91 created +secret/extcert-bench-92 created +service/extcert-bench-92 created +route.route.openshift.io/extcert-bench-92 created +secret/extcert-bench-93 created +service/extcert-bench-93 created +route.route.openshift.io/extcert-bench-93 created +secret/extcert-bench-94 created +service/extcert-bench-94 created +route.route.openshift.io/extcert-bench-94 created +secret/extcert-bench-95 created +service/extcert-bench-95 created +route.route.openshift.io/extcert-bench-95 created +secret/extcert-bench-96 created +service/extcert-bench-96 created +route.route.openshift.io/extcert-bench-96 created +secret/extcert-bench-97 created +service/extcert-bench-97 created +route.route.openshift.io/extcert-bench-97 created +secret/extcert-bench-98 created +service/extcert-bench-98 created +route.route.openshift.io/extcert-bench-98 created +secret/extcert-bench-99 created +service/extcert-bench-99 created +route.route.openshift.io/extcert-bench-99 created +secret/extcert-bench-100 created +service/extcert-bench-100 created +route.route.openshift.io/extcert-bench-100 created + ... created 100/100 routes + +==> Done: 100 external-cert routes in namespace extcert-bench + Secrets: 103 + Routes: 100 + +══════════════════════════════════════════════════════════════ + PHASE 2: BASELINE benchmark (existing cluster router — bug present) +══════════════════════════════════════════════════════════════ +==> Restoring original router image +Warning: spec.template.spec.nodeSelector[node-role.kubernetes.io/master]: use "node-role.kubernetes.io/control-plane" instead +deployment.apps/ingress-operator scaled +==> Removing CVO override for Ingress Operator +clusterversion.config.openshift.io/version patched +==> Waiting for router to recover... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 of 2 updated replicas are available... +Waiting for deployment "router-default" rollout to finish: 1 of 2 updated replicas are available... +deployment "router-default" successfully rolled out +==> Cleanup done. + +══════════════════════════════════════════════════════════════ + OCPBUGS-77056 Benchmark — existing +══════════════════════════════════════════════════════════════ +==> Restarting router-default +deployment.apps/router-default restarted +==> Waiting for new pod to be created: . Found: router-default-97b7b594b-9c7pd (Phase: Pending) +==> Tracking pod: router-default-97b7b594b-9c7pd +==> Waiting for deployment rollout to finish (timeout 600s)... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 of 2 updated replicas are available... +Waiting for deployment "router-default" rollout to finish: 1 of 2 updated replicas are available... +deployment "router-default" successfully rolled out + +══════════════════════════════════════════════════════════════ + RESULTS — existing +══════════════════════════════════════════════════════════════ + Pod: router-default-97b7b594b-9c7pd + Image: + Secrets loaded: 100 + Secret loading span: 58.10s + Pod → Ready (wall): 137s +══════════════════════════════════════════════════════════════ +==> Results saved: benchmark-results-existing-20260303-162614.txt + +══════════════════════════════════════════════════════════════ + PHASE 3: PATCHED benchmark (fixed library-go) +══════════════════════════════════════════════════════════════ +==> Wiping namespace extcert-bench (if exists) +namespace "extcert-bench" deleted +==> Creating namespace extcert-bench +namespace/extcert-bench created +==> Detected app domain: apps.btofel-netedg-260303.devcluster.openshift.com +==> Creating RBAC for router secret access +role.rbac.authorization.k8s.io/router-secret-reader created +rolebinding.rbac.authorization.k8s.io/router-secret-reader created +==> Generating test TLS certificate +==> Creating 100 secrets and routes in namespace extcert-bench +secret/extcert-bench-1 created +service/extcert-bench-1 created +route.route.openshift.io/extcert-bench-1 created +secret/extcert-bench-2 created +service/extcert-bench-2 created +route.route.openshift.io/extcert-bench-2 created +secret/extcert-bench-3 created +service/extcert-bench-3 created +route.route.openshift.io/extcert-bench-3 created +secret/extcert-bench-4 created +service/extcert-bench-4 created +route.route.openshift.io/extcert-bench-4 created +secret/extcert-bench-5 created +service/extcert-bench-5 created +route.route.openshift.io/extcert-bench-5 created +secret/extcert-bench-6 created +service/extcert-bench-6 created +route.route.openshift.io/extcert-bench-6 created +secret/extcert-bench-7 created +service/extcert-bench-7 created +route.route.openshift.io/extcert-bench-7 created +secret/extcert-bench-8 created +service/extcert-bench-8 created +route.route.openshift.io/extcert-bench-8 created +secret/extcert-bench-9 created +service/extcert-bench-9 created +route.route.openshift.io/extcert-bench-9 created +secret/extcert-bench-10 created +service/extcert-bench-10 created +route.route.openshift.io/extcert-bench-10 created + ... created 10/100 routes +secret/extcert-bench-11 created +service/extcert-bench-11 created +route.route.openshift.io/extcert-bench-11 created +secret/extcert-bench-12 created +service/extcert-bench-12 created +route.route.openshift.io/extcert-bench-12 created +secret/extcert-bench-13 created +service/extcert-bench-13 created +route.route.openshift.io/extcert-bench-13 created +secret/extcert-bench-14 created +service/extcert-bench-14 created +route.route.openshift.io/extcert-bench-14 created +secret/extcert-bench-15 created +service/extcert-bench-15 created +route.route.openshift.io/extcert-bench-15 created +secret/extcert-bench-16 created +service/extcert-bench-16 created +route.route.openshift.io/extcert-bench-16 created +secret/extcert-bench-17 created +service/extcert-bench-17 created +route.route.openshift.io/extcert-bench-17 created +secret/extcert-bench-18 created +service/extcert-bench-18 created +route.route.openshift.io/extcert-bench-18 created +secret/extcert-bench-19 created +service/extcert-bench-19 created +route.route.openshift.io/extcert-bench-19 created +secret/extcert-bench-20 created +service/extcert-bench-20 created +route.route.openshift.io/extcert-bench-20 created + ... created 20/100 routes +secret/extcert-bench-21 created +service/extcert-bench-21 created +route.route.openshift.io/extcert-bench-21 created +secret/extcert-bench-22 created +service/extcert-bench-22 created +route.route.openshift.io/extcert-bench-22 created +secret/extcert-bench-23 created +service/extcert-bench-23 created +route.route.openshift.io/extcert-bench-23 created +secret/extcert-bench-24 created +service/extcert-bench-24 created +route.route.openshift.io/extcert-bench-24 created +secret/extcert-bench-25 created +service/extcert-bench-25 created +route.route.openshift.io/extcert-bench-25 created +secret/extcert-bench-26 created +service/extcert-bench-26 created +route.route.openshift.io/extcert-bench-26 created +secret/extcert-bench-27 created +service/extcert-bench-27 created +route.route.openshift.io/extcert-bench-27 created +secret/extcert-bench-28 created +service/extcert-bench-28 created +route.route.openshift.io/extcert-bench-28 created +secret/extcert-bench-29 created +service/extcert-bench-29 created +route.route.openshift.io/extcert-bench-29 created +secret/extcert-bench-30 created +service/extcert-bench-30 created +route.route.openshift.io/extcert-bench-30 created + ... created 30/100 routes +secret/extcert-bench-31 created +service/extcert-bench-31 created +route.route.openshift.io/extcert-bench-31 created +secret/extcert-bench-32 created +service/extcert-bench-32 created +route.route.openshift.io/extcert-bench-32 created +secret/extcert-bench-33 created +service/extcert-bench-33 created +route.route.openshift.io/extcert-bench-33 created +secret/extcert-bench-34 created +service/extcert-bench-34 created +route.route.openshift.io/extcert-bench-34 created +secret/extcert-bench-35 created +service/extcert-bench-35 created +route.route.openshift.io/extcert-bench-35 created +secret/extcert-bench-36 created +service/extcert-bench-36 created +route.route.openshift.io/extcert-bench-36 created +secret/extcert-bench-37 created +service/extcert-bench-37 created +route.route.openshift.io/extcert-bench-37 created +secret/extcert-bench-38 created +service/extcert-bench-38 created +route.route.openshift.io/extcert-bench-38 created +secret/extcert-bench-39 created +service/extcert-bench-39 created +route.route.openshift.io/extcert-bench-39 created +secret/extcert-bench-40 created +service/extcert-bench-40 created +route.route.openshift.io/extcert-bench-40 created + ... created 40/100 routes +secret/extcert-bench-41 created +service/extcert-bench-41 created +route.route.openshift.io/extcert-bench-41 created +secret/extcert-bench-42 created +service/extcert-bench-42 created +route.route.openshift.io/extcert-bench-42 created +secret/extcert-bench-43 created +service/extcert-bench-43 created +route.route.openshift.io/extcert-bench-43 created +secret/extcert-bench-44 created +service/extcert-bench-44 created +route.route.openshift.io/extcert-bench-44 created +secret/extcert-bench-45 created +service/extcert-bench-45 created +route.route.openshift.io/extcert-bench-45 created +secret/extcert-bench-46 created +service/extcert-bench-46 created +route.route.openshift.io/extcert-bench-46 created +secret/extcert-bench-47 created +service/extcert-bench-47 created +route.route.openshift.io/extcert-bench-47 created +secret/extcert-bench-48 created +service/extcert-bench-48 created +route.route.openshift.io/extcert-bench-48 created +secret/extcert-bench-49 created +service/extcert-bench-49 created +route.route.openshift.io/extcert-bench-49 created +secret/extcert-bench-50 created +service/extcert-bench-50 created +route.route.openshift.io/extcert-bench-50 created + ... created 50/100 routes +secret/extcert-bench-51 created +service/extcert-bench-51 created +route.route.openshift.io/extcert-bench-51 created +secret/extcert-bench-52 created +service/extcert-bench-52 created +route.route.openshift.io/extcert-bench-52 created +secret/extcert-bench-53 created +service/extcert-bench-53 created +route.route.openshift.io/extcert-bench-53 created +secret/extcert-bench-54 created +service/extcert-bench-54 created +route.route.openshift.io/extcert-bench-54 created +secret/extcert-bench-55 created +service/extcert-bench-55 created +route.route.openshift.io/extcert-bench-55 created +secret/extcert-bench-56 created +service/extcert-bench-56 created +route.route.openshift.io/extcert-bench-56 created +secret/extcert-bench-57 created +service/extcert-bench-57 created +route.route.openshift.io/extcert-bench-57 created +secret/extcert-bench-58 created +service/extcert-bench-58 created +route.route.openshift.io/extcert-bench-58 created +secret/extcert-bench-59 created +service/extcert-bench-59 created +route.route.openshift.io/extcert-bench-59 created +secret/extcert-bench-60 created +service/extcert-bench-60 created +route.route.openshift.io/extcert-bench-60 created + ... created 60/100 routes +secret/extcert-bench-61 created +service/extcert-bench-61 created +route.route.openshift.io/extcert-bench-61 created +secret/extcert-bench-62 created +service/extcert-bench-62 created +route.route.openshift.io/extcert-bench-62 created +secret/extcert-bench-63 created +service/extcert-bench-63 created +route.route.openshift.io/extcert-bench-63 created +secret/extcert-bench-64 created +service/extcert-bench-64 created +route.route.openshift.io/extcert-bench-64 created +secret/extcert-bench-65 created +service/extcert-bench-65 created +route.route.openshift.io/extcert-bench-65 created +secret/extcert-bench-66 created +service/extcert-bench-66 created +route.route.openshift.io/extcert-bench-66 created +secret/extcert-bench-67 created +service/extcert-bench-67 created +route.route.openshift.io/extcert-bench-67 created +secret/extcert-bench-68 created +service/extcert-bench-68 created +route.route.openshift.io/extcert-bench-68 created +secret/extcert-bench-69 created +service/extcert-bench-69 created +route.route.openshift.io/extcert-bench-69 created +secret/extcert-bench-70 created +service/extcert-bench-70 created +route.route.openshift.io/extcert-bench-70 created + ... created 70/100 routes +secret/extcert-bench-71 created +service/extcert-bench-71 created +route.route.openshift.io/extcert-bench-71 created +secret/extcert-bench-72 created +service/extcert-bench-72 created +route.route.openshift.io/extcert-bench-72 created +secret/extcert-bench-73 created +service/extcert-bench-73 created +route.route.openshift.io/extcert-bench-73 created +secret/extcert-bench-74 created +service/extcert-bench-74 created +route.route.openshift.io/extcert-bench-74 created +secret/extcert-bench-75 created +service/extcert-bench-75 created +route.route.openshift.io/extcert-bench-75 created +secret/extcert-bench-76 created +service/extcert-bench-76 created +route.route.openshift.io/extcert-bench-76 created +secret/extcert-bench-77 created +service/extcert-bench-77 created +route.route.openshift.io/extcert-bench-77 created +secret/extcert-bench-78 created +service/extcert-bench-78 created +route.route.openshift.io/extcert-bench-78 created +secret/extcert-bench-79 created +service/extcert-bench-79 created +route.route.openshift.io/extcert-bench-79 created +secret/extcert-bench-80 created +service/extcert-bench-80 created +route.route.openshift.io/extcert-bench-80 created + ... created 80/100 routes +secret/extcert-bench-81 created +service/extcert-bench-81 created +route.route.openshift.io/extcert-bench-81 created +secret/extcert-bench-82 created +service/extcert-bench-82 created +route.route.openshift.io/extcert-bench-82 created +secret/extcert-bench-83 created +service/extcert-bench-83 created +route.route.openshift.io/extcert-bench-83 created +secret/extcert-bench-84 created +service/extcert-bench-84 created +route.route.openshift.io/extcert-bench-84 created +secret/extcert-bench-85 created +service/extcert-bench-85 created +route.route.openshift.io/extcert-bench-85 created +secret/extcert-bench-86 created +service/extcert-bench-86 created +route.route.openshift.io/extcert-bench-86 created +secret/extcert-bench-87 created +service/extcert-bench-87 created +route.route.openshift.io/extcert-bench-87 created +secret/extcert-bench-88 created +service/extcert-bench-88 created +route.route.openshift.io/extcert-bench-88 created +secret/extcert-bench-89 created +service/extcert-bench-89 created +route.route.openshift.io/extcert-bench-89 created +secret/extcert-bench-90 created +service/extcert-bench-90 created +route.route.openshift.io/extcert-bench-90 created + ... created 90/100 routes +secret/extcert-bench-91 created +service/extcert-bench-91 created +route.route.openshift.io/extcert-bench-91 created +secret/extcert-bench-92 created +service/extcert-bench-92 created +route.route.openshift.io/extcert-bench-92 created +secret/extcert-bench-93 created +service/extcert-bench-93 created +route.route.openshift.io/extcert-bench-93 created +secret/extcert-bench-94 created +service/extcert-bench-94 created +route.route.openshift.io/extcert-bench-94 created +secret/extcert-bench-95 created +service/extcert-bench-95 created +route.route.openshift.io/extcert-bench-95 created +secret/extcert-bench-96 created +service/extcert-bench-96 created +route.route.openshift.io/extcert-bench-96 created +secret/extcert-bench-97 created +service/extcert-bench-97 created +route.route.openshift.io/extcert-bench-97 created +secret/extcert-bench-98 created +service/extcert-bench-98 created +route.route.openshift.io/extcert-bench-98 created +secret/extcert-bench-99 created +service/extcert-bench-99 created +route.route.openshift.io/extcert-bench-99 created +secret/extcert-bench-100 created +service/extcert-bench-100 created +route.route.openshift.io/extcert-bench-100 created + ... created 100/100 routes + +==> Done: 100 external-cert routes in namespace extcert-bench + Secrets: 103 + Routes: 100 + +══════════════════════════════════════════════════════════════ + OCPBUGS-77056 Benchmark — patched +══════════════════════════════════════════════════════════════ +==> Pausing Ingress Operator to prevent image revert... +clusterversion.config.openshift.io/version patched +Warning: spec.template.spec.nodeSelector[node-role.kubernetes.io/master]: use "node-role.kubernetes.io/control-plane" instead +deployment.apps/ingress-operator scaled +==> Deploying image: quay.io/btofel/router:patched +deployment.apps/router-default image updated +==> Forcing imagePullPolicy to Always +deployment.apps/router-default patched (no change) +==> Restarting router-default +deployment.apps/router-default restarted +==> Waiting for new pod to be created: . Found: router-default-7ff6c6557f-z7724 (Phase: Pending) +==> Tracking pod: router-default-7ff6c6557f-z7724 +==> Waiting for deployment rollout to finish (timeout 600s)... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 old replicas are pending termination... +Waiting for deployment "router-default" rollout to finish: 1 of 2 updated replicas are available... +deployment "router-default" successfully rolled out + +══════════════════════════════════════════════════════════════ + RESULTS — patched +══════════════════════════════════════════════════════════════ + Pod: router-default-7ff6c6557f-z7724 + Image: quay.io/btofel/router:patched + Secrets loaded: 100 + Secret loading span: 0.07s + Pod → Ready (wall): 110s +══════════════════════════════════════════════════════════════ +==> Results saved: benchmark-results-patched-20260303-162947.txt + +══════════════════════════════════════════════════════════════ + PHASE 4: Results comparison +══════════════════════════════════════════════════════════════ + + Metric BASELINE (bug) PATCHED (fix) + -------------------------------- ---------------- ---------------- + Secrets loaded 100 100 + Secret loading span (s) 58.10 0.07 + Pod → Ready wall time (s) 137 110 + + Speedup (secret loading): 830.0x faster with the fix + 58s → 0s + + Baseline file : /Users/btofel/workspace/cluster-ingress-operator/hack/benchmark-external-certs/benchmark-results-existing-20260303-162614.txt + Patched file : /Users/btofel/workspace/cluster-ingress-operator/hack/benchmark-external-certs/benchmark-results-patched-20260303-162947.txt + +══════════════════════════════════════════════════════════════ + PHASE 5: Cleanup +══════════════════════════════════════════════════════════════ + Clean up test namespace and restore original router? [y/N] \ No newline at end of file