Simplify experiment metrics run_id label (#646)

* Clarify purpose of `label.prometheus.io/run_id` label

* Replace explicit `run_id` label with metric relabeling

* Simplify experiments dashboard queries

Author: timebertt

hack/config/monitoring/default/patch_kubestatemetrics.yaml

@@ -3,9 +3,11 @@
   path: /spec/template/spec/containers/0/args/-
   value:
     --namespaces=cert-manager,default,experiment,external-dns,ingress-nginx,kube-node-lease,kube-public,kube-system,kyverno,monitoring,parca,sharding-system,webhosting-system
-# add run_id label to kube_pod_labels to select metrics by experiment run ID
-# flag doesn't support wildcard patterns
+# label.prometheus.io/run_id is injected by experiment into observed pods (sharder and webhosting-operator).
+# Add the run_id label to kube_pod_labels. This allows joining cadvisor metrics with kube_pod_labels for selecting
+# metrics by experiment run ID.
 - op: add
   path: /spec/template/spec/containers/0/args/-
   value:
+    # this flag doesn't support wildcard patterns
     --metric-labels-allowlist=pods=[label.prometheus.io/run_id]

webhosting-operator/config/experiment/base/servicemonitor.yaml

@@ -13,6 +13,8 @@ spec:
     relabelings:
     - targetLabel: job
       replacement: experiment
+    - targetLabel: run_id
+      sourceLabels: [__meta_kubernetes_pod_uid]
   selector:
     matchLabels:
       app.kubernetes.io/name: experiment