refactor: migrate e2e prometheus from custom chart to kube-prometheus-stack

Replace the hand-rolled prometheus-operator install script and custom
Helm chart (helm/prometheus/) with the official kube-prometheus-stack
community chart (v86.2.2), installed from OCI registry.

- Disable all unused components (grafana, alertmanager, exporters,
  default rules, admission webhooks, operator TLS)
- Configure Prometheus instance, NetworkPolicies, and kubelet
  ServiceMonitor via chart values
- Add operator-controller and catalogd ServiceMonitors as
  additionalServiceMonitors using bearerTokenFile (projected SA token)
  instead of the legacy prometheus-metrics-token Secret
- Split PrometheusRules into controller-panic-alerts and
  controller-resource-alerts so the experimental override only
  replaces the resource-usage group
- Inline the install logic into the Makefile prometheus target
- Remove conftest prometheus-networkpolicies.rego policy (NetworkPolicy
  now managed by the chart)
- Remove unused kustomize bingo tooling

Co-Authored-By: Claude <noreply@anthropic.com>

Author: pedjak

.bingo/Variables.mk

@@ -77,12 +77,6 @@ $(KUBE_SCORE): $(BINGO_DIR)/kube-score.mod
 	@echo "(re)installing $(GOBIN)/kube-score-v1.20.0"
 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=kube-score.mod -o=$(GOBIN)/kube-score-v1.20.0 "github.com/zegl/kube-score/cmd/kube-score"
 
-KUSTOMIZE := $(GOBIN)/kustomize-v5.7.1
-$(KUSTOMIZE): $(BINGO_DIR)/kustomize.mod
-	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
-	@echo "(re)installing $(GOBIN)/kustomize-v5.7.1"
-	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=kustomize.mod -o=$(GOBIN)/kustomize-v5.7.1 "sigs.k8s.io/kustomize/kustomize/v5"
-
 OPERATOR_SDK := $(GOBIN)/operator-sdk-v1.41.1
 $(OPERATOR_SDK): $(BINGO_DIR)/operator-sdk.mod
 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.

.bingo/kustomize.mod

@@ -28,8 +28,6 @@ KIND="${GOBIN}/kind-v0.32.0"
 
 KUBE_SCORE="${GOBIN}/kube-score-v1.20.0"
 
-KUSTOMIZE="${GOBIN}/kustomize-v5.7.1"
-
 OPERATOR_SDK="${GOBIN}/operator-sdk-v1.41.1"
 
 OPM="${GOBIN}/opm-v1.60.0"

.bingo/kustomize.sum

@@ -165,7 +165,8 @@ make generate
 │   │   ├── base/                    # Base manifests & CRDs
 │   │   ├── templates/               # Helm templates
 │   │   └── values.yaml              # Default values
-│   └── prometheus/                  # Prometheus monitoring
+├── testdata/
+│   └── prometheus/                  # kube-prometheus-stack values for e2e monitoring
 ├── test/                            # Test suites
 │   ├── e2e/                         # End-to-end tests (see test/e2e/README.md)
 │   ├── extension-developer-e2e/     # Extension developer tests

.bingo/variables.env

@@ -55,7 +55,7 @@ ifeq ($(shell [[ $$HOME == "" || $$HOME == "/" ]] && [[ $$XDG_DATA_HOME == "" ]]
 	SETUP_ENVTEST_BIN_DIR_OVERRIDE += --bin-dir /tmp/envtest-binaries
 endif
 
-# bingo manages consistent tooling versions for things like kind, kustomize, etc.
+# bingo manages consistent tooling versions for things like kind, etc.
 include .bingo/Variables.mk
 
 ifeq ($(origin KIND_CLUSTER_NAME), undefined)
@@ -124,8 +124,7 @@ lint: lint-custom $(GOLANGCI_LINT) #HELP Run golangci linter.
 .PHONY: lint-helm
 lint-helm: $(HELM) $(CONFTEST) #HELP Run helm linter
 	helm lint helm/olmv1
-	helm lint helm/prometheus
-	(set -euo pipefail; helm template olmv1 helm/olmv1; helm template prometheus helm/prometheus) | $(CONFTEST) test --policy hack/conftest/policy/ --combine -n main -n prometheus -
+	(set -euo pipefail; helm template olmv1 helm/olmv1) | $(CONFTEST) test --policy hack/conftest/policy/ --combine -n main -
 
 .PHONY: lint-deployed-resources
 lint-deployed-resources: $(KUBE_SCORE) #EXHELP Lint deployed resources.
@@ -338,18 +337,28 @@ test-experimental-e2e: GO_BUILD_EXTRA_FLAGS := -cover
 test-experimental-e2e: COVERAGE_NAME := experimental-e2e
 test-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
 test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false
-test-experimental-e2e: PROMETHEUS_VALUES := helm/prom_experimental.yaml
+test-experimental-e2e: PROMETHEUS_VALUES := testdata/prometheus/values-experimental.yaml
 test-experimental-e2e: E2E_TIMEOUT ?= 25m
 test-experimental-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
 
+CATALOGD_CERT_SECRET = catalogd-service-cert-$(VERSION)
+
 .PHONY: prometheus
 prometheus: PROMETHEUS_NAMESPACE := olmv1-system
-prometheus: PROMETHEUS_VERSION := v0.83.0
-prometheus: $(KUSTOMIZE) #EXHELP Deploy Prometheus into specified namespace
+prometheus: PROMETHEUS_CHART_VERSION := 86.2.2
+prometheus: $(HELM) #EXHELP Deploy Prometheus into specified namespace
 ifeq ($(strip $(E2E_SUMMARY_OUTPUT)),)
 	@echo "E2E_SUMMARY_OUTPUT unset; skipping prometheus deployment"
 else
-	./hack/test/install-prometheus.sh $(PROMETHEUS_NAMESPACE) $(PROMETHEUS_VERSION) $(VERSION) $(PROMETHEUS_VALUES)
+	$(HELM) upgrade --install prometheus oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack \
+	  --namespace $(PROMETHEUS_NAMESPACE) --create-namespace \
+	  --version $(PROMETHEUS_CHART_VERSION) \
+	  -f testdata/prometheus/values.yaml \
+	  $(if $(PROMETHEUS_VALUES),-f $(PROMETHEUS_VALUES)) \
+	  --set-string 'prometheus.additionalServiceMonitors[1].endpoints[0].tlsConfig.ca.secret.name=$(CATALOGD_CERT_SECRET)' \
+	  --set-string 'prometheus.additionalServiceMonitors[1].endpoints[0].tlsConfig.cert.secret.name=$(CATALOGD_CERT_SECRET)' \
+	  --set-string 'prometheus.additionalServiceMonitors[1].endpoints[0].tlsConfig.keySecret.name=$(CATALOGD_CERT_SECRET)' \
+	  --wait --timeout 5m
 endif
 
 .PHONY: test-extension-developer-e2e

AGENTS.md

@@ -20,51 +20,21 @@ Validates core OLM NetworkPolicy requirements:
   - Ingress on port 8443 (Prometheus metrics scraping)
   - General egress enabled (for pulling bundle images, connecting to catalogd, and Kubernetes API)
 
-### prometheus-networkpolicies.rego
-
-Package: `prometheus`
-
-Validates Prometheus NetworkPolicy requirements:
-
-- Ensures a NetworkPolicy exists that allows both ingress and egress traffic for prometheus pods
-
 ## Usage
 
 These policies are automatically run as part of:
 
-- `make lint-helm` - Validates both helm/olmv1 and helm/prometheus charts (runs `main` and `prometheus` packages)
-- `make manifests` - Generates and validates core OLM manifests using only `main` package policies 
-   (Prometheus policies are intentionally skipped here, even if manifests include Prometheus resources; 
-   they are validated via `make lint-helm`)
+- `make lint-helm` - Validates the helm/olmv1 chart (runs `main` package)
+- `make manifests` - Generates and validates core OLM manifests using `main` package policies
 
 ### Running manually
 
 ```bash
-# Run all policies (main + prometheus namespaces)
-(helm template olmv1 helm/olmv1; helm template prometheus helm/prometheus) | conftest test --policy hack/conftest/policy/ --combine -n main -n prometheus -
-
-# Run only OLM policies
 helm template olmv1 helm/olmv1 | conftest test --policy hack/conftest/policy/ --combine -n main -
-
-# Run only prometheus policies
-helm template prometheus helm/prometheus | conftest test --policy hack/conftest/policy/ --combine -n prometheus -
-```
-
-### Excluding policies
-
-Use the `-n` (namespace) flag to selectively run policies:
-
-```bash
-# Skip prometheus policies
-conftest test --policy hack/conftest/policy/ --combine -n main <input>
-
-# Skip OLM policies
-conftest test --policy hack/conftest/policy/ --combine -n prometheus <input>
 ```
 
 ## Adding New Policies
 
 1. Add new rules to an existing `.rego` file or create a new one
 2. Use `package main` for policies that should run by default on all manifests
-3. Use a custom package name (e.g., `package prometheus`) for optional policies
-4. Update the Makefile targets if new namespaces need to be included
+3. Update the Makefile targets if new namespaces need to be included