update run-with-prmetheus to support local dev

Author: mjudeikis

Makefile

@@ -144,6 +144,7 @@ golangci-lint-version:
 
 .PHONY: prometheus
 prometheus: $(PROMETHEUS)
+	@echo $(PROMETHEUS)
 
 $(PROMETHEUS):
 	@hack/uget.sh \

hack/run-with-prometheus.sh

@@ -14,13 +14,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Usage: ./hack/run-with-prometheus.sh <command> [args...]
+#
+# Starts a local Prometheus (http://localhost:9090) using .prometheus-config.yaml,
+# runs the given command, then tears Prometheus down and exits with the command's
+# exit code. Without arguments the script runs nothing, exits 0, and immediately
+# kills Prometheus — you must pass a command.
+#
+# Examples:
+#   # Single-node kcp (production binary). The dev watcher below auto-populates
+#   # .prometheus-config.yaml with a scrape config pointing at localhost:6443.
+#   ./hack/run-with-prometheus.sh go run ./cmd/kcp start
+#   ./hack/run-with-prometheus.sh ./bin/kcp start
+#
+#   # Sharded setup via the dev test-server — the server itself registers scrape
+#   # configs (one job per shard plus kcp-front-proxy) through the Go testing
+#   # framework, so nothing extra is needed here. Build once with `make build-e2e`.
+#   ./hack/run-with-prometheus.sh ./bin/sharded-test-server --work-dir-path=.
+#   ./hack/run-with-prometheus.sh ./bin/sharded-test-server --work-dir-path=. --number-of-shards=3
+#
+# When a local cmd/kcp dev server is detected (.kcp/apiserver.crt and
+# .kcp/admin.kubeconfig appear in the repo root), the script auto-writes a scrape
+# config for it and reloads Prometheus. sharded-test-server writes .kcp/serving-ca.crt
+# (no apiserver.crt) so the watcher stays silent and lets the server's own
+# ScrapeMetrics calls populate .prometheus-config.yaml. E2e runs use tempdirs and
+# follow the same pattern — watcher idle, Go framework owns the config.
+#
+# If ARTIFACT_DIR is set, the tsdb is archived to
+# ${ARTIFACT_DIR}/metrics/prometheus.tar after shutdown.
+
 set -o nounset
 set -o pipefail
 set -o errexit
 
 cd "$(dirname $0)/.."
 
-PROMETHEUS="$(UGET_PRINT_PATH=relative make --no-print-directory prometheus)"
+PROMETHEUS="$(UGET_PRINT_PATH=absolute make --no-print-directory prometheus)"
+
+if [ ! -x "${PROMETHEUS}" ]; then
+  echo "prometheus binary not found at ${PROMETHEUS}" >&2
+  exit 1
+fi
 
 touch .prometheus-config.yaml
 "$PROMETHEUS" \
@@ -36,6 +70,43 @@ while ! curl -s http://localhost:9090/-/ready; do
 done
 echo 'Prometheus is ready!'
 
+# Dev-only: when running a local `kcp start` (not in Prow CI), watch for kcp's
+# cert+kubeconfig to appear and auto-populate a scrape config. In Prow, e2e tests
+# populate .prometheus-config.yaml via the Go testing framework
+# (staging/.../testing/server/metrics.go) — this watcher would race and clobber
+# those per-server entries, so it stays off. Gating on PROW_JOB_ID rather than
+# ARTIFACT_DIR because users set ARTIFACT_DIR locally to collect tsdb archives.
+WATCHER_PID=""
+if [ -z "${PROW_JOB_ID:-}" ]; then
+  (
+    for _ in $(seq 1 120); do
+      if [ -f .kcp/apiserver.crt ] && [ -f .kcp/admin.kubeconfig ]; then
+        # Use the shard-admin token (system:master) — kcp-admin is workspace-scoped
+        # and gets 401 on the raw /metrics endpoint.
+        TOKEN=$(awk '/name: shard-admin/{found=1} found && /token:/{print $2; exit}' .kcp/admin.kubeconfig)
+        if [ -n "${TOKEN}" ]; then
+          cat > .prometheus-config.yaml <<EOF
+scrape_configs:
+  - job_name: kcp
+    scrape_interval: 5s
+    scheme: https
+    bearer_token: ${TOKEN}
+    tls_config:
+      ca_file: $(pwd)/.kcp/apiserver.crt
+    static_configs:
+      - targets: ["localhost:6443"]
+EOF
+          curl -sX POST http://localhost:9090/-/reload >/dev/null
+          echo 'Wrote kcp scrape config and reloaded Prometheus.'
+          exit 0
+        fi
+      fi
+      sleep 1
+    done
+  ) &
+  WATCHER_PID=$!
+fi
+
 set -o xtrace
 set +o errexit
 "${@}"
@@ -44,6 +115,7 @@ set +o xtrace
 set -o errexit
 echo "Command terminated with ${EXIT_CODE}"
 
+[ -n "${WATCHER_PID}" ] && kill -TERM ${WATCHER_PID} 2>/dev/null || true
 kill -TERM ${PROM_PID}
 
 if [ -n "${ARTIFACT_DIR:=}" ]; then