Merge pull request #2259 from tkatila/e2e-add-ocp-operator-support

e2e: rewrite operator tests with OCP support

Author: mythi

DEVEL.md

@@ -10,6 +10,7 @@ Table of Contents
    * [Work with Intel Device Plugins Operator Modifications](#work-with-intel-device-plugins-operator-modifications)
    * [Publish a New Version of the Intel Device Plugins Operator to operatorhub.io](#publish-a-new-version-of-the-intel-device-plugins-operator-to-operatorhubio)
    * [Run E2E Tests](#run-e2e-tests)
+   * [Run Operator E2E Tests](#run-operator-e2e-tests)
    * [Run Controller Tests with a Local Control Plane](#run-controller-tests-with-a-local-control-plane)
 * [How to Develop Simple Device Plugins](#how-to-develop-simple-device-plugins)
     * [Logging](#logging)
@@ -268,6 +269,99 @@ without a pre-configured Kubernetes cluster. Just make sure you have
 make test-with-kind
 ```
 
+### Run Operator E2E Tests
+
+The operator E2E tests (`test/e2e/operator/`) verify that the Intel Device Plugins Operator
+correctly deploys each plugin DaemonSet and exposes device resources as allocatable on nodes.
+The tests cover QAT (cy, dc), SGX, DSA (idxd, vfio), IAA, and GPU plugins.
+
+The following environment variables must be set before running:
+
+| Variable | Required | Description |
+|---|---|---|
+| `PROJECT_NAMESPACE`| always | Kubernetes namespace where the operator is deployed |
+| `IMAGE_PATH`       | always | Image path under the registry (e.g. `myproject`) |
+| `PLUGIN_VERSION`   | always | Plugin image tag (e.g. `0.35.0`) |
+| `IMAGE_REGISTRY`   | K8s only | Container registry address (e.g. `registry.example.com:5000`); omit on OCP|
+
+#### Vanilla Kubernetes
+
+**Prerequisites:**
+
+- `cert-manager` must already be installed in the cluster (the `cert-manager` namespace must exist).
+  The tests will fail early if it is absent. See the [cert-manager install guide](https://cert-manager.io/docs/installation/).
+- Node Feature Discovery (NFD) is deployed automatically by the test suite if no pods are found in the `node-feature-discovery` namespace.
+- The `PROJECT_NAMESPACE` namespace is created automatically if it does not yet exist.
+- The operator is deployed from `deployments/operator/default/kustomization.yaml`.
+
+```bash
+export PLUGIN_VERSION=0.35.0
+export PROJECT_NAMESPACE=inteldeviceplugin-operator
+export IMAGE_PATH=myproject
+export IMAGE_REGISTRY=registry.example.com:5000
+KUBECONFIG=/path/to/kubeconfig make e2e-operator
+```
+
+Running operator tests with existing container images (0.35.0):
+```bash
+export PLUGIN_VERSION=0.35.0
+export PROJECT_NAMESPACE=inteldeviceplugin-operator
+export IMAGE_PATH=intel
+export IMAGE_REGISTRY=docker.io
+KUBECONFIG=/path/to/kubeconfig make e2e-operator
+```
+
+#### OpenShift (OCP)
+
+**Prerequisites:**
+
+- The OCP service-CA operator is used for TLS; `cert-manager` is **not** required and must not
+  interfere.
+- NFD is expected to already be running, either via the OpenShift NFD Operator
+  (`openshift-nfd` namespace) or standard upstream NFD (`node-feature-discovery` namespace).
+  The test suite deploys NFD automatically only if neither namespace has running pods.
+- The `PROJECT_NAMESPACE` namespace **must exist before running**; it is not created automatically
+  on OCP.  For OCP internal registry access, `PROJECT_NAMESPACE` and `IMAGE_PATH` typically refer
+  to the same OpenShift project.
+- Do **not** set `IMAGE_REGISTRY`; the tests default to the OCP internal registry
+  (`image-registry.openshift-image-registry.svc:5000`).
+- The operator is deployed from `deployments/operator/overlays/ocp/kustomization.yaml`.
+- Plugin images must be mirrored to the OCP internal registry before running:
+
+```bash
+# The TAG has to be same or greater than the current release semver
+TAG=0.35.1 make set-version dockerfiles mirror-images-ocp
+```
+
+Then run the tests:
+
+```bash
+export PROJECT_NAMESPACE=inteldeviceplugin-operator
+export IMAGE_PATH=inteldeviceplugin-operator
+export PLUGIN_VERSION=0.35.1
+KUBECONFIG=/path/to/kubeconfig make e2e-operator
+```
+
+#### Filtering Operator Tests
+
+The `make e2e-operator` target uses the label filter `operator && !(gpu || iaa)` by default.
+Use `--ginkgo.label-filter` directly when you need a different subset:
+
+```bash
+# Run only the QAT operator tests
+KUBECONFIG=/path/to/kubeconfig go test -v ./test/e2e/... \
+  -ginkgo.v --ginkgo.label-filter "operator && qat" \
+  -delete-namespace-on-failure=false
+
+# Run all operator tests including GPU and IAA
+KUBECONFIG=/path/to/kubeconfig go test -v ./test/e2e/... \
+  -ginkgo.v --ginkgo.label-filter "operator" \
+  -delete-namespace-on-failure=false
+```
+
+Available per-plugin labels: `qat`, `sgx`, `dsa`, `iaa`, `gpu`.
+Focus labels within operator tests: `cy`, `dc` (QAT), `idxd`, `vfio` (DSA), `i915` (GPU).
+
 ### Run Controller Tests with a Local Control Plane
 
 The controller-runtime library provides a package for integration testing by

Makefile

@@ -134,6 +134,11 @@ bundle-build:
 clean:
 	@for cmd in $(cmds) ; do pwd=$(shell pwd) ; cd cmd/$$cmd ; $(GO) clean ; cd $$pwd ; done
 
+.PHONY: mirror-images-ocp
+mirror-images-ocp:
+	@bash scripts/build-images-for-ocp.sh
+	@bash scripts/mirror-images-to-ocp.sh inteldeviceplugin-operator
+
 ORG?=intel
 REG?=$(ORG)/
 TAG?=devel
@@ -161,14 +166,26 @@ e2e-dsa:
 e2e-iaa:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events --ginkgo.label-filter "iaa && !operator" $(E2E_DRYRUN) -delete-namespace-on-failure=false
 
-# This is a CI specific target to run all tests that are possible in the SPR host
+# This is a CI specific target to run all tests that are possible in the SPR host.
+# Plugin tests run first, operator tests run second to avoid resource conflicts.
 e2e-spr:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events --ginkgo.label-filter "(qat && !compress-perf && dpdk && (cy || dc) || sgx) && !operator " $(E2E_DRYRUN) -delete-namespace-on-failure=false -e2e-verify-service-account=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events \
+		--ginkgo.label-filter "(qat && !compress-perf && !nft && !operator) || (sgx && !operator)" $(E2E_DRYRUN) \
+		-delete-namespace-on-failure=false -e2e-verify-service-account=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events \
+		--ginkgo.label-filter "operator && (qat || sgx)" $(E2E_DRYRUN) \
+		-delete-namespace-on-failure=false -e2e-verify-service-account=false
 
 # Target to run a subset of tests depending on the given filter arg. By default, runs all tests.
 e2e:
 	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events --ginkgo.label-filter "$(E2E_FILTER)" $(E2E_DRYRUN) -delete-namespace-on-failure=false
 
+e2e-operator:
+	@echo "NOTE: This depends on 'mirror-images-ocp' target but does not depend on it directly to allow running tests without building and mirroring every time."
+	@echo "For OCP: Make sure PROJECT_NAMESPACE, IMAGE_PATH and PLUGIN_VERSION env variables are set, and IMAGE_REGISTRY _not_ set."
+	@echo "For K8s: Make sure PROJECT_NAMESPACE, IMAGE_PATH, PLUGIN_VERSION and IMAGE_REGISTRY env variables are set"
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events --ginkgo.label-filter "operator && !(gpu || iaa)" $(E2E_DRYRUN) -delete-namespace-on-failure=false
+
 pre-pull:
 ifeq ($(TAG),devel)
 	@$(BUILDER) pull golang:1.25-trixie

deployments/operator/overlays/ocp/kustomization.yaml

@@ -0,0 +1,47 @@
+# OCP-specific operator overlay.
+# Replaces cert-manager TLS with the OCP service-CA operator and grants the
+# privileged SCC to the default service account in the operator namespace so
+# that device-plugin DaemonSets can access host devices.
+namePrefix: intel-deviceplugins-
+
+resources:
+- ../../crd
+- ../../rbac
+- ../../manager
+- ../../webhook
+# SCC privileged mode doesn't seem to be needed, so it's left out for now.
+# If we need it in the future, we can add it back in.
+#- scc-binding.yaml
+
+patches:
+- path: manager_metrics_patch.yaml
+  target:
+    kind: Deployment
+- path: manager_webhook_patch.yaml
+  target:
+    kind: Deployment
+    name: controller-manager
+- path: webhook-service-patch.yaml
+  target:
+    kind: Service
+    name: webhook-service
+- path: webhook-mutating-ca-patch.yaml
+  target:
+    kind: MutatingWebhookConfiguration
+- path: webhook-validating-ca-patch.yaml
+  target:
+    kind: ValidatingWebhookConfiguration
+# remove namespace object as we don't want to create (and delete) it in OCP overlay
+# In OCP, we must keep the namespace so the images within the ImageStream stay intact.
+- patch: |-
+    $patch: delete
+    apiVersion: v1
+    kind: Namespace
+    metadata:
+      labels:
+        control-plane: controller-manager
+        manager: intel-deviceplugin-operator
+        pod-security.kubernetes.io/enforce: privileged
+        pod-security.kubernetes.io/audit: privileged
+        pod-security.kubernetes.io/warn: privileged
+      name: system

deployments/operator/overlays/ocp/manager_metrics_patch.yaml

@@ -0,0 +1,7 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-bind-address=:8443"
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-secure"

deployments/operator/overlays/ocp/manager_webhook_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: webhook-server-cert

deployments/operator/overlays/ocp/scc-binding.yaml

@@ -0,0 +1,15 @@
+# Grant the privileged SCC to the default service account in the operator
+# namespace so that device-plugin DaemonSet pods can access host devices and
+# run with the required capabilities.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: privileged-scc
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: inteldeviceplugins-system
+roleRef:
+  kind: ClusterRole
+  name: system:openshift:scc:privileged
+  apiGroup: rbac.authorization.k8s.io

deployments/operator/overlays/ocp/webhook-mutating-ca-patch.yaml

@@ -0,0 +1,9 @@
+# Instruct the OCP service-CA operator to inject the cluster CA bundle into
+# the MutatingWebhookConfiguration so that the API server can validate the
+# webhook server's TLS certificate.
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+  annotations:
+    service.beta.openshift.io/inject-cabundle: "true"

deployments/operator/overlays/ocp/webhook-service-patch.yaml

@@ -0,0 +1,9 @@
+# Instruct the OCP service-CA operator to issue a signed serving certificate
+# for the webhook service and store it in the named secret.  The secret name
+# must match the volume mount defined in manager_webhook_patch.yaml.
+apiVersion: v1
+kind: Service
+metadata:
+  name: webhook-service
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: webhook-server-cert

deployments/operator/overlays/ocp/webhook-validating-ca-patch.yaml

@@ -0,0 +1,9 @@
+# Instruct the OCP service-CA operator to inject the cluster CA bundle into
+# the ValidatingWebhookConfiguration so that the API server can validate the
+# webhook server's TLS certificate.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+  annotations:
+    service.beta.openshift.io/inject-cabundle: "true"

scripts/build-images-for-ocp.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Copyright 2026 Intel Corporation. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build-images-for-ocp.sh – builds all container images required for the OCP
+# e2e tests from the current project source tree.
+#
+# Usage:
+#   ./scripts/build-images-for-ocp.sh
+#
+# Environment variables (all optional, match Makefile defaults):
+#   BUILDER   – container builder binary: docker (default), podman, or buildah
+#   TAG       – image tag (default: value from Makefile, currently 0.35.1)
+#   UBI       – set to 1 to build UBI-based images instead of distroless
+#
+# After a successful run the images are available locally as:
+#   intel/<name>:<TAG>
+# Pass these to mirror-images-to-ocp.sh to push them to an OCP cluster.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+# To load OCP_IMAGES
+. "${SCRIPT_DIR}/common-ocp.sh"
+
+log() { echo "[build-images-for-ocp] $*"; }
+
+cd "${REPO_ROOT}"
+
+# Ensure Dockerfiles are up to date with the .Dockerfile.in templates before
+# building.  This is a no-op if nothing has changed.
+log "Regenerating Dockerfiles from templates..."
+make dockerfiles
+
+log "Building images: ${OCP_IMAGES[*]}"
+for img in "${OCP_IMAGES[@]}"; do
+    log "  building ${img}..."
+    make "${img}" ${BUILDER:+BUILDER="${BUILDER}"} ${TAG:+TAG="${TAG}"} ${UBI:+UBI="${UBI}"}
+done
+
+# Print a summary so the caller can verify and export OCP_IMAGE_REGISTRY later.
+TAG="${TAG:-$(grep '^TAG' Makefile | head -1 | awk -F'[?=]' '{print $NF}' | tr -d ' ')}"
+log ""
+log "Build complete.  Images available locally:"
+for img in "${OCP_IMAGES[@]}"; do
+    log "  intel/${img}:${TAG}"
+done
+log ""
+log "Next step: push to your OCP cluster:"
+log "  ./scripts/mirror-images-to-ocp.sh <namespace>"