Fixes configuration for bundled collectors and e2e test for ClusterObservability

Follow-ups to the initial ClusterObservability framework so the
generated agent and cluster collectors come up cleanly on stock
distributions, plus a smoke e2e that exercises the pipelines end-to-end.

Author: jinja2

.chloggen/clusterobs-fixes.yaml

@@ -0,0 +1,29 @@
+change_type: bug_fix
+
+component: cluster-observability
+
+note: Fix dependencies (image, env, volumes) required by the bundled agent and cluster collector configs
+
+issues: [3821, 3818]
+
+subtext: |
+  Follow-ups to the initial ClusterObservability framework so the generated
+  agent and cluster collectors come up cleanly on stock distributions:
+    - Generate the agent and cluster collectors with the k8s distribution of the
+      collector image (same version as the operator's collector image), overridable
+      via the `--clusterobservability-collector-image` flag.
+    - Inject `K8S_NODE_NAME` on the agent pod, referenced by the bundled
+      `kubeletstats` receiver's endpoint.
+    - Report the cluster collector's readiness from its StatefulSet workload
+      so `ClusterObservability` no longer stays in `Pending` when the cluster
+      collector is fully healthy.
+    - Mount the whole host root at `/hostfs` so the `hostmetrics` receiver 
+      can resolve symlinks and stat host mount points.
+      Exclude virtual/container-runtime filesystems from its `filesystem` scraper
+      to avoid `/sysroot` and `/boot` scrape errors on OpenShift.
+    - Run the agent as root with the `spc_t` SELinux type (supplied by the
+      generated SCC) on OpenShift so the `filelog` receiver can read the
+      root-owned container logs under `/var/log/pods`.
+    - Point the generated `Instrumentation` at the agent's OTLP HTTP port (4318),
+      since most auto-instrumentation SDKs default `OTEL_EXPORTER_OTLP_PROTOCOL`
+      to `http/protobuf`.

.github/workflows/e2e-reusable.yaml

@@ -58,6 +58,9 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     needs: [ build-test-images ]
+    # Allow opt-in matrix entries (those with `experimental: true` in their
+    # `include` block) to fail without marking the workflow run red.
+    continue-on-error: ${{ matrix.experimental == true }}
     strategy:
       fail-fast: false
       matrix:
@@ -71,6 +74,7 @@ jobs:
         - e2e
         - e2e-automatic-rbac
         - e2e-autoscale
+        - e2e-clusterobservability
         - e2e-instrumentation-default
         - e2e-instrumentation
         - e2e-no-crds
@@ -89,6 +93,10 @@ jobs:
         - e2e-crd-validations
         - e2e-ta-standalone
         include:
+        - group: e2e-clusterobservability
+          # ClusterObservability is behind a feature gate and still maturing
+          experimental: true
+          setup: "prepare-e2e-clusterobservability"
         - group: e2e-instrumentation-default
           setup: "add-instrumentation-params prepare-e2e"
         - group: e2e-instrumentation

Makefile

@@ -401,6 +401,14 @@ e2e: chainsaw
 e2e-sidecar: chainsaw
 	$(CHAINSAW) test --test-dir ./tests/e2e-sidecar --report-name e2e-sidecar $(CHAINSAW_SELECTOR)
 
+# End-to-end tests for ClusterObservability. The CRD is not bundled, so install
+# it directly from config/crd/bases/. --parallel 1 because only the oldest
+# active ClusterObservability CR reconciles.
+.PHONY: e2e-clusterobservability
+e2e-clusterobservability: chainsaw
+	kubectl apply -f config/crd/bases/opentelemetry.io_clusterobservabilities.yaml
+	$(CHAINSAW) test --test-dir ./tests/e2e-clusterobservability --report-name e2e-clusterobservability --parallel 1
+
 # end-to-end-test for testing automatic RBAC creation
 .PHONY: e2e-automatic-rbac
 e2e-automatic-rbac: chainsaw
@@ -541,6 +549,15 @@ prepare-e2e: chainsaw set-image-controller add-image-targetallocator add-image-o
 prepare-e2e-no-crds: chainsaw set-image-controller add-image-targetallocator add-image-opampbridge start-kind cert-manager install-metrics-server install-targetallocator-prometheus-crds load-image-all deploy-no-crds
 	@mkdir -p ./.testresults/e2e
 
+# ClusterObservability needs the kubelet to serve a cluster-CA-signed cert so the
+# bundled kubeletstats receiver can verify it via the SA-token CA bundle. Override
+# KIND_CONFIG to a kubelet-config that enables serverTLSBootstrap.
+.PHONY: prepare-e2e-clusterobservability
+prepare-e2e-clusterobservability: KIND_CONFIG = kind-clusterobservability-$(KUBE_VERSION).yaml
+prepare-e2e-clusterobservability: add-rbac-permissions-to-operator
+	@$(MAKE) add-operator-arg OPERATOR_ARG=--feature-gates=+operator.clusterobservability
+	@$(MAKE) prepare-e2e
+
 # Run operator-sdk scorecard tests for bundles
 .PHONY: scorecard-tests
 scorecard-tests: operator-sdk
@@ -645,8 +662,15 @@ ifeq (true,$(START_KIND_CLUSTER))
 	else \
 		$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG); \
 	fi
+	@$(MAKE) approve-kubelet-csrs
 endif
 
+# Approve any kubelet-serving CSRs created via serverTLSBootstrap. No-op for
+# kind configs that do not enable bootstrap.
+.PHONY: approve-kubelet-csrs
+approve-kubelet-csrs:
+	@kubectl get csr -o jsonpath='{range .items[?(@.spec.signerName=="kubernetes.io/kubelet-serving")]}{.metadata.name}{" "}{end}' | xargs -r kubectl certificate approve
+
 # Stop kind cluster
 .PHONY: stop-kind
 stop-kind: kind

internal/config/cli.go

@@ -39,6 +39,7 @@ func CreateCLIParser(cfg Config) *pflag.FlagSet {
 	f.Bool("create-sm-operator-metrics", cfg.CreateServiceMonitorOperatorMetrics, "Create a ServiceMonitor for the operator metrics")
 	f.Bool("ignore-missing-collector-crds", cfg.IgnoreMissingCollectorCRDs, "Ignore missing OpenTelemetryCollector CRDs presence in the cluster")
 	f.String("collector-image", cfg.CollectorImage, "The default OpenTelemetry collector image. This image is used when no image is specified in the CustomResource.")
+	f.String("clusterobservability-collector-image", cfg.ClusterObservabilityCollectorImage, "The OpenTelemetry collector image used for collectors generated by the ClusterObservability reconciler. Defaults to the k8s distribution.")
 	f.String("target-allocator-image", cfg.TargetAllocatorImage, "The default OpenTelemetry target allocator image. This image is used when no image is specified in the CustomResource.")
 	f.String("operator-opamp-bridge-image", cfg.OperatorOpAMPBridgeImage, "The default OpenTelemetry Operator OpAMP Bridge image. This image is used when no image is specified in the CustomResource.")
 	f.String("auto-instrumentation-java-image", cfg.AutoInstrumentationJavaImage, "The default OpenTelemetry Java instrumentation image. This image is used when no image is specified in the CustomResource.")
@@ -96,6 +97,8 @@ func ApplyCLI(cfg *Config) error {
 				cfg.IgnoreMissingCollectorCRDs, _ = f.GetBool("ignore-missing-collector-crds")
 			case "collector-image":
 				cfg.CollectorImage, _ = f.GetString("collector-image")
+			case "clusterobservability-collector-image":
+				cfg.ClusterObservabilityCollectorImage, _ = f.GetString("clusterobservability-collector-image")
 			case "target-allocator-image":
 				cfg.TargetAllocatorImage, _ = f.GetString("target-allocator-image")
 			case "operator-opamp-bridge-image":

internal/config/config.go

@@ -47,6 +47,11 @@ type Config struct {
 	AutoInstrumentationPythonImage string `yaml:"auto-instrumentation-python-image"`
 	// CollectorImage represents the flag to override the OpenTelemetry Collector container image.
 	CollectorImage string `yaml:"collector-image"`
+	// ClusterObservabilityCollectorImage is the collector image used for the agent and
+	// cluster collectors generated by the ClusterObservability reconciler. It defaults to
+	// the k8s distribution at the same version as CollectorImage, since the bundled configs
+	// only use components shipped in that distribution.
+	ClusterObservabilityCollectorImage string `yaml:"clusterobservability-collector-image"`
 	// CollectorConfigMapEntry represents the configuration file name for the collector. Immutable.
 	CollectorConfigMapEntry string `yaml:"collector-configmap-entry"`
 	// CreateRBACPermissions is true when the operator can create RBAC permissions for SAs running a collector instance. Immutable.
@@ -172,6 +177,7 @@ func New() Config {
 	v := version.Get()
 	return Config{
 		CollectorImage:                      fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:%s", v.OpenTelemetryCollector),
+		ClusterObservabilityCollectorImage:  fmt.Sprintf("ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s:%s", v.OpenTelemetryCollector),
 		CollectorConfigMapEntry:             defaultCollectorConfigMapEntry,
 		EnableMultiInstrumentation:          true,
 		EnableApacheHttpdInstrumentation:    true,

internal/config/config_test.go

@@ -29,6 +29,7 @@ func TestToStringMap(t *testing.T) {
 		"auto-instrumentation-python-image":       "",
 		"cert-manager-availability":               "0",
 		"collector-availability":                  "0",
+		"clusterobservability-collector-image":    "",
 		"collector-configmap-entry":               "",
 		"collector-image":                         "myexample:1.0",
 		"create-rbac-permissions":                 "0",

internal/config/testdata/config.yaml

@@ -2,6 +2,7 @@ targetallocator-image: ghcr.io/open-telemetry/opentelemetry-operator/target-allo
 operatoropampbridge-image: ghcr.io/open-telemetry/opentelemetry-operator/operator-opamp-bridge:0.0.0
 auto-instrumentation-python-image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:0.0.0
 collector-image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:0.0.0
+clusterobservability-collector-image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s:0.0.0
 collector-configmap-entry: collector.yaml
 create-rbac-permissions: 0
 enable-multi-instrumentation: true