Add OpenShift Tests Extension (OTE) for baremetal E2E tests

Migrate baremetal E2E tests from openshift-tests-private to
cluster-baremetal-operator using the OpenShift Tests Extension (OTE)
framework. This enables the tests to run as part of the release payload.

Key changes:

1. Test Extension as Separate Go Module
   - Created cmd/cluster-baremetal-tests-ext/ with dedicated go.mod/vendor
   - Isolates OTE dependencies from main operator module
   - Includes 6 baremetal deployment sanity tests migrated from
     openshift-tests-private

2. OTE Framework Integration
   - Registered suite: cluster-baremetal/all
   - Parent suite: openshift/conformance/parallel
   - Platform filter: baremetal
   - Uses OpenShift Ginkgo fork with 31 Kubernetes replace directives

3. Build and Packaging
   - Dockerfile: Added RUN make build-tests and COPY for test binary
   - Makefile: Added build-tests target with GONOSUMDB and
     GO_COMPLIANCE_POLICY settings
   - Binary compressed with gzip for container image

4. Test Path Structure
   - Moved tests to test/e2e/openshift/baremetal/ to avoid conflicts
     with upstream BMO/IrSO tests

5. Security Fixes
   - Fixed CVE-2026-29181: Upgraded OpenTelemetry from v1.37.0 to v1.41.0
     in test extension module
   - Fixed gosec G703 path traversal warnings in hack/readme_inputs/main.go
   - Replaced shell command execution with native Go file reading in
     clusterOperatorHealthcheck function

6. CI Compatibility Fixes
   - Regenerated manifests to fix generate-check checksum mismatch
   - Fixed vendor case-sensitivity: renamed Microsoft → microsoft
     (224 files) for Linux CI compatibility
   - Pinned kustomize to v4.5.4 to fix gnostic API incompatibility
   - Cleaned up unused golangci-lint and kustomize dependencies (~200
     transitive deps) from main go.mod

7. Dependency Management
   - Main module go.mod reverted to pre-OTE state (OpenTelemetry v1.35.0)
   - Test extension maintains separate dependencies with security fixes
   - Vendor directories synchronized with go.mod/go.sum

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: jadhaj

Dockerfile

@@ -4,9 +4,11 @@ FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS
 WORKDIR /go/src/github.com/openshift/cluster-baremetal-operator
 COPY . .
 RUN make build
+RUN make build-tests
 
 FROM registry.ci.openshift.org/ocp/4.22:base-rhel9
 COPY --from=builder /go/src/github.com/openshift/cluster-baremetal-operator/bin/cluster-baremetal-operator /usr/bin/cluster-baremetal-operator
+COPY --from=builder /go/src/github.com/openshift/cluster-baremetal-operator/bin/cluster-baremetal-tests-ext.gz /usr/bin/cluster-baremetal-tests-ext.gz
 COPY --from=builder /go/src/github.com/openshift/cluster-baremetal-operator/manifests /manifests
 LABEL io.openshift.release.operator=true
 ENTRYPOINT ["/usr/bin/cluster-baremetal-operator"]

Makefile

@@ -10,10 +10,10 @@ CONTAINER_TOOL ?= docker
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 
-CONTROLLER_GEN ?= go run vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go
+CONTROLLER_GEN ?= GOFLAGS=-mod=mod go run sigs.k8s.io/controller-tools/cmd/controller-gen
 CRD_OPTIONS="crd:crdVersions=v1"
-GOLANGCI_LINT ?= GOLANGCI_LINT_CACHE=$(GOLANGCI_LINT_CACHE) go run vendor/github.com/golangci/golangci-lint/v2/cmd/golangci-lint/main.go
-KUSTOMIZE ?= go run sigs.k8s.io/kustomize/kustomize/v4
+GOLANGCI_LINT ?= GOLANGCI_LINT_CACHE=$(GOLANGCI_LINT_CACHE) GOFLAGS=-mod=mod go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint
+KUSTOMIZE ?= GOFLAGS=-mod=mod go run sigs.k8s.io/kustomize/kustomize/v4@v4.5.4
 MANIFEST_PROFILE ?= default
 TMP_DIR := $(shell mktemp -d -t manifests-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX)
 
@@ -25,8 +25,9 @@ VERBOSE ?= ""
 all: generate lint build
 
 # Run unit tests
+# Uses GOFLAGS=-mod=mod to ignore vendor/ directory
 unit:
-	go test $(VERBOSE) ./... -coverprofile cover.out
+	GOFLAGS=-mod=mod go test $(VERBOSE) ./... -coverprofile cover.out
 
 # Run tests
 test: generate lint manifests unit
@@ -35,6 +36,21 @@ test: generate lint manifests unit
 build:
 	go build -o bin/cluster-baremetal-operator main.go
 
+# Build OTE test extension binary
+# Test extension is a separate Go module in cmd/cluster-baremetal-tests-ext/ with its own go.mod and vendor/
+# This keeps OTE dependencies isolated from the main operator module
+# GONOSUMDB bypasses checksum verification for origin module - required because origin uses pseudo-versions not in public checksum database
+# CGO_ENABLED=0 produces static binary for container compatibility
+# GO_COMPLIANCE_POLICY="exempt_all" is OpenShift-specific variable that exempts test binaries from internal compliance checks
+#   - Used consistently across OpenShift test extensions (see openshift-tests, oc-tests-ext, etc.)
+#   - Blanket exemption is standard for test-only binaries that don't ship to customers
+#   - Test extensions run in CI environments only, not in production clusters
+build-tests:
+	cd cmd/cluster-baremetal-tests-ext && \
+	GONOSUMDB="github.com/openshift/origin" CGO_ENABLED=0 GO_COMPLIANCE_POLICY="exempt_all" \
+	go build -mod=vendor -o ../../bin/cluster-baremetal-tests-ext .
+	gzip -f bin/cluster-baremetal-tests-ext
+
 # Run against the configured Kubernetes cluster in ~/.kube/config
 run: generate manifests
 	go run ./main.go -images-json $(IMAGES_JSON)
@@ -100,6 +116,8 @@ manifests: generate
 fmt:
 
 # Run go lint against code
+# Uses GOFLAGS=-mod=mod to ignore vendor/ directory which may be out of sync with go.mod
+# This allows OTE dependencies in go.mod without requiring vendor/ sync
 .PHONY: lint
 lint:
 	$(GOLANGCI_LINT) run
@@ -109,9 +127,10 @@ lint:
 vet: lint
 
 # Generate code
+# Uses GOFLAGS=-mod=mod to ignore vendor/ directory
 .PHONY: generate
 generate:
-	go generate -x ./...
+	GOFLAGS=-mod=mod go generate -x ./...
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=cluster-baremetal-operator webhook paths=./... output:crd:artifacts:config=config/crd/bases
 	sed -i '/^    controller-gen.kubebuilder.io\/version: (devel)/d' config/crd/bases/*
 	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."

cmd/cluster-baremetal-tests-ext/go.mod

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	extcmd "github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+
+	// Import test packages to register Ginkgo specs
+	_ "github.com/openshift/cluster-baremetal-tests-ext/test/e2e/openshift/baremetal"
+)
+
+func main() {
+	// Extension registry
+	registry := e.NewRegistry()
+
+	// Create extension
+	ext := e.NewExtension(
+		"openshift",           // product
+		"payload",             // type
+		"cluster-baremetal",   // component name
+	)
+
+	// Add suites to the extension
+	ext.AddSuite(e.Suite{
+		Name:    "cluster-baremetal/all",
+		Parents: []string{"openshift/conformance/parallel"},
+	})
+
+	// Build test specs from Ginkgo tests automatically
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "couldn't build extension test specs from ginkgo: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Apply environment selectors - baremetal platform only
+	// All tests should only run on baremetal platform
+	specs.Select(et.NameContains("")).
+		Include(et.PlatformEquals("baremetal"))
+
+	// Add specs to extension
+	ext.AddSpecs(specs)
+	registry.Register(ext)
+
+	// Create root command
+	rootCmd := &cobra.Command{
+		Use:   "cluster-baremetal-tests-ext",
+		Short: "Cluster BareMetal Operator Test Extension",
+		Long:  "OpenShift Tests Extension for Cluster BareMetal Operator E2E Tests",
+	}
+
+	// Register OTE subcommands (info, list, run-test, run-suite, etc.)
+	rootCmd.AddCommand(extcmd.DefaultExtensionCommands(registry)...)
+
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

cmd/cluster-baremetal-tests-ext/go.sum

@@ -0,0 +1,172 @@
+package baremetal
+
+import (
+	"fmt"
+	"strings"
+
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+	compat_otp "github.com/openshift/origin/test/extended/util/compat_otp"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = g.Describe("[sig-baremetal] INSTALLER IPI for INSTALLER_GENERAL job on BareMetal", func() {
+	defer g.GinkgoRecover()
+	var (
+		oc           = compat_otp.NewCLI("baremetal-deployment-sanity", compat_otp.KubeConfigPath())
+		iaasPlatform string
+	)
+	g.BeforeEach(func() {
+		compat_otp.SkipForSNOCluster(oc)
+		iaasPlatform = compat_otp.CheckPlatform(oc)
+		if !(iaasPlatform == "baremetal") {
+			e2e.Logf("Cluster is: %s", iaasPlatform)
+			g.Skip("For Non-baremetal cluster , this is not supported!")
+		}
+	})
+	// author: jhajyahy@redhat.com
+	// port=yes - 99.7% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-29146-Verify that all clusteroperators are Available", func() {
+		g.By("Running oc get clusteroperators")
+		res, err := checkOperatorsRunning(oc)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(res).To(o.BeTrue())
+	})
+
+	// author: jhajyahy@redhat.com
+	// port=yes - 100.0% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-29719-Verify that all nodes are up and running", func() {
+		g.By("Running oc get nodes")
+		res, err := checkNodesRunning(oc)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(res).To(o.BeTrue())
+
+	})
+
+	// author: jhajyahy@redhat.com
+	// port=yes - 99.7% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-32361-Verify that deployment exists and is not empty", func() {
+		g.By("Create new namespace")
+		oc.SetupProject()
+		ns32361 := oc.Namespace()
+
+		g.By("Create deployment")
+		deployCreationErr := oc.Run("create").Args("deployment", "deploy32361", "-n", ns32361, "--image", "quay.io/openshifttest/hello-openshift@sha256:4200f438cf2e9446f6bcff9d67ceea1f69ed07a2f83363b7fb52529f7ddd8a83").Execute()
+		o.Expect(deployCreationErr).NotTo(o.HaveOccurred())
+
+		g.By("Check deployment status is available")
+		waitForDeployStatus(oc, "deploy32361", ns32361, "True")
+		status, err := oc.AsAdmin().Run("get").Args("deployment", "-n", ns32361, "deploy32361", "-o=jsonpath={.status.conditions[?(@.type=='Available')].status}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		e2e.Logf("\nDeployment %s Status is %s\n", "deploy32361", status)
+		o.Expect(status).To(o.Equal("True"))
+
+		g.By("Check pod is in Running state")
+		podName := getPodName(oc, ns32361)
+		podStatus := getPodStatus(oc, ns32361, podName)
+		o.Expect(podStatus).To(o.Equal("Running"))
+	})
+
+	// author: jhajyahy@redhat.com
+	// port=yes - 99.9% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-34195-Verify all pods replicas are running on workers only", func() {
+		g.By("Create new namespace")
+		oc.SetupProject()
+		ns34195 := oc.Namespace()
+
+		g.By("Determine appropriate replica count based on cluster topology")
+		// Query control plane topology to handle SNO, compact, and standard clusters
+		topologyOutput, err := oc.AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.controlPlaneTopology}").Output()
+		replicasNum := 3 // safe default for standard clusters
+
+		if err == nil && topologyOutput != "" {
+			switch topologyOutput {
+			case "SingleReplica":
+				// SNO cluster - single node, limited capacity
+				replicasNum = 1
+			case "HighlyAvailable":
+				// Standard cluster - use schedulable nodes count
+				schedulableNodes, nodeErr := oc.AsAdmin().Run("get").Args("nodes", "-o=jsonpath={.items[?(@.spec.taints[*].effect!='NoSchedule')].metadata.name}").Output()
+				if nodeErr == nil && schedulableNodes != "" {
+					nodeList := strings.Fields(schedulableNodes)
+					if len(nodeList) > 0 {
+						replicasNum = len(nodeList) + 1
+					}
+				} else {
+					// Fallback to worker nodes count if schedulable query fails
+					workerNodes, workerErr := compat_otp.GetClusterNodesBy(oc, "worker")
+					if workerErr == nil && len(workerNodes) > 0 {
+						replicasNum = len(workerNodes) + 1
+					}
+				}
+			default:
+				// External or unknown topology - use safe default
+				e2e.Logf("Unknown topology %s, using default replica count: %d", topologyOutput, replicasNum)
+			}
+		} else {
+			e2e.Logf("Unable to query control plane topology, using default replica count: %d", replicasNum)
+		}
+
+		g.By(fmt.Sprintf("Create deployment with %d replicas", replicasNum))
+		deployCreationErr := oc.Run("create").Args("deployment", "deploy34195", "-n", ns34195, fmt.Sprintf("--replicas=%d", replicasNum), "--image", "quay.io/openshifttest/hello-openshift@sha256:4200f438cf2e9446f6bcff9d67ceea1f69ed07a2f83363b7fb52529f7ddd8a83").Execute()
+		o.Expect(deployCreationErr).NotTo(o.HaveOccurred())
+		waitForDeployStatus(oc, "deploy34195", ns34195, "True")
+
+		g.By("Check deployed pods number is as expected")
+		pods, err := oc.AsAdmin().Run("get").Args("pods", "-n", ns34195, "--field-selector=status.phase=Running", "-o=jsonpath={.items[*].metadata.name}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		podList := strings.Fields(pods)
+		o.Expect(len(podList)).To(o.Equal(replicasNum))
+
+		g.By("Check pods are deployed on worker nodes only")
+		for _, pod := range podList {
+			podNodeName, err := compat_otp.GetPodNodeName(oc, ns34195, pod)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			res := compat_otp.IsWorkerNode(oc, podNodeName)
+			if !res {
+				e2e.Logf("\nPod %s was deployed on non worker node  %s\n", pod, podNodeName)
+			}
+			o.Expect(res).To(o.BeTrue())
+		}
+	})
+
+	// author: jhajyahy@redhat.com
+	// port=yes - 99.7% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-39126-Verify maximum CPU usage limit hasn't reached on each of the nodes", func() {
+		g.By("Running oc get nodes")
+		cpuExceededNodes := []string{}
+		sampling_time, err := getClusterUptime(oc)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		nodeNames, nodeErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-o=jsonpath={.items[*].metadata.name}").Output()
+		o.Expect(nodeErr).NotTo(o.HaveOccurred(), "Failed to execute oc get nodes")
+		nodes := strings.Fields(nodeNames)
+		for _, node := range nodes {
+			cpuUsage := getNodeCpuUsage(oc, node, sampling_time)
+			if cpuUsage > maxCpuUsageAllowed {
+				cpuExceededNodes = append(cpuExceededNodes, node)
+				e2e.Logf("\ncpu usage of exceeded node: %s is %.2f%%", node, cpuUsage)
+			}
+		}
+		o.Expect(cpuExceededNodes).Should(o.BeEmpty(), "These nodes exceed max CPU usage allowed: %s", cpuExceededNodes)
+	})
+
+	// author: jhajyahy@redhat.com
+	// port=yes - 99.6% pass rate (724 runs last 60 days)
+	g.It("Author:jhajyahy-Medium-39125-Verify that every node memory is sufficient", func() {
+		g.By("Running oc get nodes")
+		outOfMemoryNodes := []string{}
+		nodeNames, nodeErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-o=jsonpath={.items[*].metadata.name}").Output()
+		o.Expect(nodeErr).NotTo(o.HaveOccurred(), "Failed to execute oc get nodes")
+		nodes := strings.Fields(nodeNames)
+		for _, node := range nodes {
+			availMem := getNodeavailMem(oc, node)
+			e2e.Logf("\nAvailable mem of Node %s is %d", node, availMem)
+			if availMem < minRequiredMemoryInBytes {
+				outOfMemoryNodes = append(outOfMemoryNodes, node)
+				e2e.Logf("\nNode %s does not meet minimum required memory %d Bytes ", node, minRequiredMemoryInBytes)
+
+			}
+		}
+		o.Expect(outOfMemoryNodes).Should(o.BeEmpty(), "These nodes does not meet minimum required memory: %s", outOfMemoryNodes)
+	})
+})