openshift · bentito · Mar 3, 2026
diff --git a/hack/benchmark-external-certs/00-run-all.sh b/hack/benchmark-external-certs/00-run-all.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# 00-run-all.sh
+# Full OCPBUGS-77056 benchmark orchestrator.
+#
+# Phases (each starts with a wipe for idempotency):
+#   0. quay login + preflight checks
+#   1. Wipe + create test routes (extcert-bench namespace)
+#   2. BASELINE: restart existing cluster router, measure startup
+#   3. PATCHED: deploy pre-built patched image, restart, measure startup
+#   4. Print comparison table
+#   5. Cleanup prompt
+#
+# !! DO NOT COMMIT with QUAY_PASSWORD set !!
+# Usage: ./00-run-all.sh [--count 200] [--namespace extcert-bench]
+
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# ── Config ────────────────────────────────────────────────────────────────────
+ROUTE_COUNT="${ROUTE_COUNT:-200}"
+NAMESPACE="${BENCH_NAMESPACE:-extcert-bench}"
+REGISTRY="${ROUTER_IMAGE_REGISTRY:-quay.io/btofel/router}"
+PATCHED_IMAGE="${REGISTRY}:patched"
+QUAY_USER="${QUAY_USER:-btofel}"
+QUAY_PASSWORD="${QUAY_PASSWORD:-UZF9gy4jrRp0l+/gh0zqK6HJx39MEZ3k57m3CDs2I8BrnGkmAttI63KiXFyn7/XV}"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --count)      ROUTE_COUNT="$2"; shift 2 ;;
+    --namespace)  NAMESPACE="$2";   shift 2 ;;
+    *) echo "Unknown arg: $1"; exit 1 ;;
+  esac
+done
+
+banner() {
+  echo ""
+  echo "══════════════════════════════════════════════════════════════"
+  printf "  %s\n" "$*"
+  echo "══════════════════════════════════════════════════════════════"
+}
+
+read_result() { grep "^${2}=" "${1}" 2>/dev/null | cut -d= -f2 || echo "N/A"; }
+
+# ── Phase 0: Preflight ────────────────────────────────────────────────────────
+banner "PHASE 0: Preflight"
+
+echo "==> Checking oc connectivity"
+oc whoami || { echo "ERROR: not logged into a cluster"; exit 1; }
+oc get nodes --no-headers | head -3
+
+echo "==> Logging into quay.io"
+podman login -u="${QUAY_USER}" -p="${QUAY_PASSWORD}" quay.io
+
+echo "==> Config: count=${ROUTE_COUNT} namespace=${NAMESPACE} image=${PATCHED_IMAGE}"
+
+# ── Phase 1: Wipe + setup routes ─────────────────────────────────────────────
+banner "PHASE 1: Route setup (wipe + create ${ROUTE_COUNT} external-cert routes)"
+bash "${SCRIPT_DIR}/01-setup-routes.sh" \
+  --count "$ROUTE_COUNT" \
+  --namespace "$NAMESPACE"
+
+# ── Phase 2: Baseline benchmark (existing cluster router) ────────────────────
+banner "PHASE 2: BASELINE benchmark (existing cluster router — bug present)"
+# Ensure we are actually evaluating the baseline buggy image by restoring it
+bash "${SCRIPT_DIR}/03-run-benchmark.sh" --restore
+bash "${SCRIPT_DIR}/03-run-benchmark.sh" --existing
+BASELINE_FILE=$(ls -t "${SCRIPT_DIR}"/benchmark-results-existing-*.txt 2>/dev/null | head -1 || \
+                ls -t benchmark-results-existing-*.txt 2>/dev/null | head -1 || echo "")
+
+# ── Phase 3: Patched benchmark ───────────────────────────────────────────────
+banner "PHASE 3: PATCHED benchmark (fixed library-go)"
+# Wipe and recreate routes so the router sees the same load as the baseline run
+bash "${SCRIPT_DIR}/01-setup-routes.sh" \
+  --count "$ROUTE_COUNT" \
+  --namespace "$NAMESPACE"
+bash "${SCRIPT_DIR}/03-run-benchmark.sh" --image "$PATCHED_IMAGE"
+PATCHED_FILE=$(ls -t "${SCRIPT_DIR}"/benchmark-results-patched-*.txt 2>/dev/null | head -1 || \
+               ls -t benchmark-results-patched-*.txt 2>/dev/null | head -1 || echo "")
+
+# ── Phase 4: Comparison ───────────────────────────────────────────────────────
+banner "PHASE 4: Results comparison"
+if [[ -f "${BASELINE_FILE:-}" && -f "${PATCHED_FILE:-}" ]]; then
+  B_SECRETS=$(read_result "$BASELINE_FILE" secret_count)
+  B_LOAD=$(read_result    "$BASELINE_FILE" secret_load_seconds)
+  B_READY=$(read_result   "$BASELINE_FILE" pod_to_ready_seconds)
+  P_SECRETS=$(read_result "$PATCHED_FILE"  secret_count)
+  P_LOAD=$(read_result    "$PATCHED_FILE"  secret_load_seconds)
+  P_READY=$(read_result   "$PATCHED_FILE"  pod_to_ready_seconds)
+
+  printf "\n  %-32s %-16s %-16s\n" "Metric" "BASELINE (bug)" "PATCHED (fix)"
+  printf "  %-32s %-16s %-16s\n"   "--------------------------------" "----------------" "----------------"
+  printf "  %-32s %-16s %-16s\n"   "Secrets loaded"        "$B_SECRETS"  "$P_SECRETS"
+  printf "  %-32s %-16s %-16s\n"   "Secret loading span (s)"  "$B_LOAD"  "$P_LOAD"
+  printf "  %-32s %-16s %-16s\n"   "Pod → Ready wall time (s)" "$B_READY" "$P_READY"
+  echo ""
+
+  python3 -c "
+b, p = '${B_LOAD}', '${P_LOAD}'
+try:
+    ratio = float(b)/float(p)
+    print(f'  Speedup (secret loading): {ratio:.1f}x faster with the fix')
+    print(f'  {float(b):.0f}s  →  {float(p):.0f}s')
+except: pass
+" 2>/dev/null || true
+
+  echo ""
+  echo "  Baseline file : $BASELINE_FILE"
+  echo "  Patched file  : $PATCHED_FILE"
+else
+  echo "  (result files not found; check benchmark-results-*.txt)"
+fi
+
+# ── Phase 5: Cleanup ─────────────────────────────────────────────────────────
+banner "PHASE 5: Cleanup"
+read -rp "  Clean up test namespace and restore original router? [y/N] " CONFIRM
+if [[ "${CONFIRM,,}" == "y" ]]; then
+  bash "${SCRIPT_DIR}/04-cleanup.sh" --namespace "$NAMESPACE"
+else
+  echo "  Skipped. Run: ./04-cleanup.sh --namespace $NAMESPACE"
+fi
diff --git a/hack/benchmark-external-certs/01-setup-routes.sh b/hack/benchmark-external-certs/01-setup-routes.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# 01-setup-routes.sh
+# Idempotent: always wipes and recreates the namespace, then creates N
+# TLS secrets + routes with spec.tls.externalCertificate.
+#
+# Usage: ./01-setup-routes.sh [--count 200] [--namespace extcert-bench]
+
+set -euo pipefail
+
+COUNT=200
+NAMESPACE=extcert-bench
+APP_DOMAIN=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --count)     COUNT="$2";     shift 2 ;;
+    --namespace) NAMESPACE="$2"; shift 2 ;;
+    --domain)    APP_DOMAIN="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1"; exit 1 ;;
+  esac
+done
+
+# ── Wipe existing namespace (idempotent) ──────────────────────────────────────
+echo "==> Wiping namespace $NAMESPACE (if exists)"
+oc delete ns "$NAMESPACE" --ignore-not-found --wait=true 2>/dev/null || true
+
+echo "==> Creating namespace $NAMESPACE"
+oc create ns "$NAMESPACE"
+
+# ── Auto-detect cluster app domain ───────────────────────────────────────────
+if [[ -z "$APP_DOMAIN" ]]; then
+  APP_DOMAIN=$(oc get ingresses.config.openshift.io cluster \
+    -o jsonpath='{.spec.domain}' 2>/dev/null || echo "apps.example.com")
+  echo "==> Detected app domain: $APP_DOMAIN"
+fi
+
+# ── RBAC: custom Role granting the router SA access to secrets ───────────────
+# NOTE: system:router clusterrole does NOT include secrets — must use custom Role.
+echo "==> Creating RBAC for router secret access"
+oc apply -n "$NAMESPACE" -f - <<RBAC_EOF
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: router-secret-reader
+  namespace: ${NAMESPACE}
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: router-secret-reader
+  namespace: ${NAMESPACE}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: router-secret-reader
+subjects:
+- kind: ServiceAccount
+  name: router
+  namespace: openshift-ingress
+RBAC_EOF
+
+# ── Generate a self-signed cert (reused for all routes) ──────────────────────
+echo "==> Generating test TLS certificate"
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+openssl req -x509 -newkey rsa:2048 \
+  -keyout "$TMPDIR/tls.key" -out "$TMPDIR/tls.crt" \
+  -days 365 -nodes \
+  -subj "/CN=bench.${APP_DOMAIN}" \
+  -addext "subjectAltName=DNS:bench.${APP_DOMAIN}" \
+  2>/dev/null
+
+# ── Create secrets, services, and routes ─────────────────────────────────────
+echo "==> Creating $COUNT secrets and routes in namespace $NAMESPACE"
+CREATED=0
+for i in $(seq 1 "$COUNT"); do
+  NAME="extcert-bench-${i}"
+
+  oc create secret tls "$NAME" \
+    --cert="$TMPDIR/tls.crt" \
+    --key="$TMPDIR/tls.key" \
+    -n "$NAMESPACE" 2>/dev/null
+
+  oc create service clusterip "$NAME" --tcp=8080:8080 \
+    -n "$NAMESPACE" 2>/dev/null
+
+  oc apply -n "$NAMESPACE" -f - 2>/dev/null <<ROUTE_EOF
+apiVersion: route.openshift.io/v1
+kind: Route
+metadata:
+  name: ${NAME}
+spec:
+  host: ${NAME}.${APP_DOMAIN}
+  to:
+    kind: Service
+    name: ${NAME}
+    weight: 100
+  tls:
+    termination: edge
+    externalCertificate:
+      name: ${NAME}
+    insecureEdgeTerminationPolicy: Redirect
+ROUTE_EOF
+
+  CREATED=$((CREATED + 1))
+  if ((CREATED % 10 == 0)); then
+    echo "  ... created $CREATED/$COUNT routes"
+  fi
+done
+
+echo ""
+echo "==> Done: $COUNT external-cert routes in namespace $NAMESPACE"
+echo "    Secrets: $(oc get secrets -n $NAMESPACE --no-headers | wc -l | tr -d ' ')"
+echo "    Routes:  $(oc get routes  -n $NAMESPACE --no-headers | wc -l | tr -d ' ')"
diff --git a/hack/benchmark-external-certs/02-build-patched.sh b/hack/benchmark-external-certs/02-build-patched.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# 02-build-patched.sh
+# Builds the router binary with the patched library-go (OCPBUGS-77056 fix)
+# and pushes to quay.io/btofel/router:patched.
+#
+# The BASELINE is the existing cluster router — no build needed for that.
+#
+# Usage: ./02-build-patched.sh [--registry quay.io/btofel/router]
+
+set -euo pipefail
+
+REGISTRY="${ROUTER_IMAGE_REGISTRY:-quay.io/btofel/router}"
+IMAGE="${REGISTRY}:patched"
+ROUTER_DIR="/Users/btofel/workspace/router"
+LIBRARYGO_DIR="/Users/btofel/workspace/library-go"
+CONTAINER_TOOL="${CONTAINER_TOOL:-podman}"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --registry) REGISTRY="$2"; IMAGE="${REGISTRY}:patched"; shift 2 ;;
+    *) echo "Unknown arg: $1"; exit 1 ;;
+  esac
+done
+
+echo "==> Building patched router image: $IMAGE"
+echo "    Router:    $ROUTER_DIR"
+echo "    Library-go: $LIBRARYGO_DIR (fixed branch: $(git -C $LIBRARYGO_DIR branch --show-current))"
+
+# Work in a temp copy to avoid dirtying the real workspace
+WORKDIR=$(mktemp -d)
+trap "rm -rf $WORKDIR" EXIT
+cp -a "$ROUTER_DIR/." "$WORKDIR/"
+cd "$WORKDIR"
+
+# ── Apply local library-go replace directive ──────────────────────────────────
+echo "==> Applying library-go replace directive in go.mod"
+LIBGO_MOD=$(grep '^module ' "$LIBRARYGO_DIR/go.mod" | awk '{print $2}')
+if grep -q "replace.*library-go" go.mod 2>/dev/null; then
+  # Update existing replace line (macOS-compatible sed)
+  sed -i '' "s|replace ${LIBGO_MOD} =>.*|replace ${LIBGO_MOD} => ${LIBRARYGO_DIR}|" go.mod
+else
+  printf '\nreplace %s => %s\n' "$LIBGO_MOD" "$LIBRARYGO_DIR" >> go.mod
+fi
+
+echo "==> Running go mod tidy & vendor..."
+go mod tidy 2>&1 | tail -5 || true
+go mod vendor 2>&1 | tail -5
+
+# Verify the fix is present in vendor
+if grep -q "secret cache not synced yet" vendor/github.com/openshift/library-go/pkg/secret/secret_monitor.go; then
+  echo "==> Verified: patched secret_monitor.go in vendor"
+else
+  echo "ERROR: patched library-go not found in vendor" >&2; exit 1
+fi
+
+# ── Build binary ──────────────────────────────────────────────────────────────
+echo "==> Compiling openshift-router (linux/amd64)..."
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+  go build -o openshift-router -ldflags '-w -s' ./cmd/openshift-router
+echo "==> Binary: $(ls -lh openshift-router | awk '{print $5}')"
+
+# ── Build container image ─────────────────────────────────────────────────────
+BASE_IMAGE=$(oc get deployment router-default -n openshift-ingress -o jsonpath='{.spec.template.spec.containers[0].image}')
+echo "==> Building container from $BASE_IMAGE"
+IMGCTX=$(mktemp -d)
+cp openshift-router "$IMGCTX/"
+cat > "$IMGCTX/Dockerfile" <<DEOF
+FROM ${BASE_IMAGE}
+COPY openshift-router /usr/bin/openshift-router
+DEOF
+$CONTAINER_TOOL build -t "$IMAGE" "$IMGCTX"
+rm -rf "$IMGCTX"
+
+# ── Push ──────────────────────────────────────────────────────────────────────
+echo "==> Pushing $IMAGE"
+$CONTAINER_TOOL push "$IMAGE"
+echo "==> Done: $IMAGE"