Fixes configuration for bundled collectors and e2e test for ClusterObservability

Follow-ups to the initial ClusterObservability framework so the
generated agent and cluster collectors come up cleanly on stock
distributions, plus a smoke e2e that exercises the pipelines end-to-end.

Author: jinja2

.chloggen/clusterobs-fixes.yaml

@@ -0,0 +1,17 @@
+change_type: bug_fix
+
+component: cluster-observability
+
+note: Fix dependencies (image, env, volumes) required by the bundled agent and cluster collector configs
+
+issues: [3821, 3818]
+
+subtext: |
+  Follow-ups to the initial ClusterObservability framework so the generated
+  agent and cluster collectors come up cleanly on stock distributions:
+    - Use the contrib distribution of the configured collector image.
+    - Inject `K8S_NODE_NAME` on the agent pod, referenced by the bundled
+      `kubeletstats` receiver's endpoint.
+    - Report the cluster collector's readiness from its StatefulSet workload
+      so `ClusterObservability` no longer stays in `Pending` when the cluster
+      collector is fully healthy.

.github/workflows/e2e-reusable.yaml

@@ -56,6 +56,11 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     needs: [ build-test-images ]
+    # Allow opt-in matrix entries (those with `experimental: true` in their
+    # `include` block) to fail without marking the workflow run red. Combined
+    # with fail-fast: false this means experimental groups never block other
+    # operator e2e jobs or the merge queue.
+    continue-on-error: ${{ matrix.experimental == true }}
     strategy:
       fail-fast: false
       matrix:
@@ -69,6 +74,7 @@ jobs:
         - e2e
         - e2e-automatic-rbac
         - e2e-autoscale
+        - e2e-clusterobservability
         - e2e-instrumentation-default
         - e2e-instrumentation
         - e2e-no-crds
@@ -87,6 +93,11 @@ jobs:
         - e2e-crd-validations
         - e2e-ta-standalone
         include:
+        - group: e2e-clusterobservability
+          # ClusterObservability is behind a feature gate and still maturing;
+          # mark experimental so failures don't block other operator e2e jobs.
+          experimental: true
+          setup: "prepare-e2e-clusterobservability"
         - group: e2e-instrumentation-default
           setup: "add-instrumentation-params prepare-e2e"
         - group: e2e-instrumentation

Makefile

@@ -401,6 +401,14 @@ e2e: chainsaw
 e2e-sidecar: chainsaw
 	$(CHAINSAW) test --test-dir ./tests/e2e-sidecar --report-name e2e-sidecar $(CHAINSAW_SELECTOR)
 
+# End-to-end tests for ClusterObservability. The CRD is not bundled, so install
+# it directly from config/crd/bases/. --parallel 1 because only the oldest
+# active ClusterObservability CR reconciles.
+.PHONY: e2e-clusterobservability
+e2e-clusterobservability: chainsaw
+	kubectl apply -f config/crd/bases/opentelemetry.io_clusterobservabilities.yaml
+	$(CHAINSAW) test --test-dir ./tests/e2e-clusterobservability --report-name e2e-clusterobservability --parallel 1
+
 # end-to-end-test for testing automatic RBAC creation
 .PHONY: e2e-automatic-rbac
 e2e-automatic-rbac: chainsaw
@@ -521,6 +529,16 @@ prepare-e2e: chainsaw set-image-controller add-image-targetallocator add-image-o
 prepare-e2e-no-crds: chainsaw set-image-controller add-image-targetallocator add-image-opampbridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy-no-crds
 	@mkdir -p ./.testresults/e2e
 
+# ClusterObservability needs the kubelet to serve a cluster-CA-signed cert so
+# the bundled kubeletstats receiver can verify it via the SA-token CA bundle.
+# Override KIND_CONFIG to a kubelet-config that enables serverTLSBootstrap; the
+# kubelet-serving CSR is then approved as part of start-kind.
+.PHONY: prepare-e2e-clusterobservability
+prepare-e2e-clusterobservability: KIND_CONFIG = kind-clusterobservability-$(KUBE_VERSION).yaml
+prepare-e2e-clusterobservability: add-rbac-permissions-to-operator
+	@$(MAKE) add-operator-arg OPERATOR_ARG=--feature-gates=+operator.clusterobservability
+	@$(MAKE) prepare-e2e
+
 # Run operator-sdk scorecard tests for bundles
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
@@ -619,8 +637,15 @@ container-instrumentation-all: container-instrumentation-java container-instrume
 start-kind: kind
 ifeq (true,$(START_KIND_CLUSTER))
 	$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG) || true
+	@$(MAKE) approve-kubelet-csrs
 endif
 
+# Approve any kubelet-serving CSRs created via serverTLSBootstrap. No-op for
+# kind configs that do not enable bootstrap.
+.PHONY: approve-kubelet-csrs
+approve-kubelet-csrs:
+	@kubectl get csr -o jsonpath='{range .items[?(@.spec.signerName=="kubernetes.io/kubelet-serving")]}{.metadata.name}{" "}{end}' | xargs -r kubectl certificate approve
+
 # Stop kind cluster
 .PHONY: stop-kind
 stop-kind: kind