Merge pull request #597 from pmeida/ote-bootstrap

OCPCLOUD-3009: Bootstrap OTE framework

Author: openshift-merge-bot[bot]

AGENTS.md

@@ -182,3 +182,8 @@ FContext("context name", func() { /* tests */ })
 - Each controller has a `suite_test.go` that bootstraps an `envtest.Environment`
 - See "Running Tests" above for why `make unit` is required
 
+## OpenShift Tests Extension (OTE)
+
+The `openshift-tests-extension/` directory contains a separate Go module that builds the OTE binary (`cluster-capi-operator-tests-ext`). This binary is embedded in the operator image and used by `openshift-tests` to discover and run e2e tests as part of the OpenShift test infrastructure.
+
+See [`openshift-tests-extension/README.md`](openshift-tests-extension/README.md) for the full developer guide covering: how to add tests to OTE, suite/label routing, feature gate requirements, blocking vs informing lifecycle, per-test timeouts, local runs, and CI verification.

Dockerfile.rhel

@@ -1,14 +1,16 @@
 FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS builder
 WORKDIR /go/src/github.com/openshift/cluster-capi-operator
 COPY . .
-RUN make clean build
+RUN make clean build && \
+    gzip -n bin/cluster-capi-operator-tests-ext
 
 FROM registry.ci.openshift.org/ocp/4.22:base-rhel9
 COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/capi-operator .
 COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/capi-installer .
 COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/capi-controllers .
 COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/machine-api-migration .
 COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/crd-compatibility-checker .
+COPY --from=builder /go/src/github.com/openshift/cluster-capi-operator/bin/cluster-capi-operator-tests-ext.gz /usr/bin/cluster-capi-operator-tests-ext.gz
 
 COPY ./manifests /manifests
 COPY ./capi-operator-manifests /capi-operator-manifests

Makefile

@@ -30,7 +30,7 @@ verify: fmt lint verify-ocp-manifests ## Run formatting and linting checks
 
 test: verify unit ## Run verification and unit tests
 
-build: bin/capi-operator bin/capi-installer bin/capi-controllers bin/machine-api-migration bin/crd-compatibility-checker manifests-gen ## Build all binaries
+build: bin/capi-operator bin/capi-installer bin/capi-controllers bin/machine-api-migration bin/crd-compatibility-checker bin/cluster-capi-operator-tests-ext manifests-gen ## Build all binaries
 
 clean:
 	rm -rf bin/*
@@ -61,6 +61,11 @@ ocp-manifests: manifests-gen ## Generate admission policy profiles for image emb
 bin/%: FORCE | bin/
 	go build -o "$@" "./cmd/$*"
 
+bin/cluster-capi-operator-tests-ext: FORCE | bin/ ## Build tests extension binary
+	go build -mod=vendor \
+		-o bin/cluster-capi-operator-tests-ext \
+		./openshift-tests-extension/cmd/
+
 .PHONY: localtestenv
 localtestenv: .localtestenv
 

e2e/machine_migration_vap_tests.go e2e/machine_migration_vap_test.goe2e/machine_migration_vap_tests.go renamed to e2e/machine_migration_vap_test.go

@@ -5,6 +5,7 @@ use (
 	./e2e
 	./hack/tools
 	./manifests-gen
+	./openshift-tests-extension
 )
 
 replace (

go.work

@@ -8,7 +8,7 @@ go work use -r .
 
 # Pass 1: tidy all modules
 echo "Running go mod tidy for all modules (pass 1)..."
-for module in . e2e manifests-gen hack/tools; do
+for module in . e2e manifests-gen hack/tools openshift-tests-extension; do
   if [ -f "$module/go.mod" ]; then
     echo "Tidying $module"
     (cd "$module" && go mod tidy)
@@ -21,7 +21,7 @@ go work sync
 
 # Pass 2: re-tidy after sync may have bumped versions
 echo "Running go mod tidy for all modules (pass 2)..."
-for module in . e2e manifests-gen hack/tools; do
+for module in . e2e manifests-gen hack/tools openshift-tests-extension; do
   if [ -f "$module/go.mod" ]; then
     echo "Tidying $module"
     (cd "$module" && go mod tidy)
@@ -30,7 +30,7 @@ done
 
 # Verify all modules
 echo "Verifying all modules..."
-for module in . e2e manifests-gen hack/tools; do
+for module in . e2e manifests-gen hack/tools openshift-tests-extension; do
   if [ -f "$module/go.mod" ]; then
     echo "Verifying $module"
     (cd "$module" && go mod verify)

hack/vendor.sh

@@ -0,0 +1,125 @@
+# OpenShift Tests Extension (OTE)
+
+This directory contains a separate Go module that builds the OTE binary
+(`cluster-capi-operator-tests-ext`). The binary is embedded gzipped in the
+operator image at `/usr/bin/cluster-capi-operator-tests-ext.gz` and used by
+`openshift-tests` to discover and run e2e tests as part of the OpenShift test
+infrastructure.
+
+## Adding a test to OTE
+
+Whether a test file is visible to OTE depends entirely on its file suffix:
+
+| File | `make e2e` | OTE (`list tests`) |
+|---|---|---|
+| `e2e/my_test.go` | ✅ | ❌ |
+| `e2e/my.go` | ✅ | ✅ |
+| `e2e/my.go` + `//go:build !e2e` | ❌ | ✅ |
+
+**To make a test discoverable by OTE:** write it in a regular `.go` file (not
+`_test.go`). No changes to the extension binary are needed — ginkgo
+auto-discovery picks it up on the next build.
+
+**To keep a test out of OTE:** use the standard `_test.go` suffix. It still
+runs via `make e2e`.
+
+**To make a test run via OTE only (not `make e2e`):** use a regular `.go` file
+with `//go:build !e2e`. `make e2e` passes `--tags=e2e` internally which causes
+`!e2e` files to be excluded from compilation. The OTE binary is built without
+that tag, so the file is included.
+
+## Test labels and suite routing
+
+Labels on a test control which OTE suite it lands in, and therefore which
+nightly conformance job picks it up via the `Parents` field:
+
+| Label on test | OTE suite | Parent suite |
+|---|---|---|
+| none (default) | `capio/parallel` | `openshift/conformance/parallel` |
+| `Label("Serial")` | `capio/serial` | `openshift/conformance/serial` |
+| `Label("Disruptive")` | `capio/disruptive` | `openshift/disruptive-longrunning` |
+
+A test is included in a periodic run when **both** of these are true:
+1. The cluster has the feature set that enables the test's feature gate (e.g. `TechPreviewNoUpgrade`)
+2. The job runs the matching parent suite (e.g. `openshift/conformance/serial`)
+
+Example — a serial test that lands in `capio/serial → openshift/conformance/serial`:
+
+```go
+// e2e/aws_machineset.go  (regular .go, not _test.go — visible to OTE)
+var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:ClusterAPIMachineManagementAWS] Cluster API AWS", func() {
+    It("should scale a MachineSet", Label("Serial"), func() {
+        // test body
+    })
+})
+```
+
+Example — a parallel test (no label) that lands in `capio/parallel → openshift/conformance/parallel`:
+
+```go
+var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:ClusterAPIMachineManagementAWS] Cluster API AWS", func() {
+    It("should have a running cluster", func() {
+        // test body
+    })
+})
+```
+
+## Feature gate labeling
+
+Every e2e test must carry the appropriate `[OCPFeatureGate:FeatureName]` tag in
+its `Describe` block name. This tells `openshift-tests` to skip the test
+automatically on clusters where the feature gate is not enabled:
+
+```go
+var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:ClusterAPIMachineManagementAWS] Cluster API AWS", func() {
+    It("should run a machine", func() { ... })
+})
+```
+
+Without this tag, a test runs on every cluster regardless of feature gate
+status, which will cause failures on clusters where the feature is not active.
+
+## Blocking vs informing lifecycle
+
+By default every test is `blocking` — a failure causes the suite exit code to
+be non-zero. To mark a test as `informing` (failure recorded but non-blocking):
+
+```go
+import g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+
+It("should be stable but not yet required", g.Informing(), func() { ... })
+```
+
+Informing failures appear in the JSON output and Sippy dashboards but do not
+gate merges or promotions. Use this temporarily to gather stability data before
+promoting a test to blocking. Tests must not remain informing indefinitely.
+
+**Important:** `g.Informing()` is only honored when the test runs through the
+OTE binary (`run-suite`, `run-test`). When running via `make e2e`, ginkgo has
+no concept of informing lifecycle — a failing informing test will still cause
+the `make e2e` run to exit with a non-zero code.
+
+
+## Running locally
+
+`list tests` and `list suites` work without a cluster. `run-suite` and
+`run-test` require a reachable OCP cluster because `InitCommonVariables()`
+fetches the `cluster` Infrastructure object.
+
+```bash
+# Build the binary
+make bin/cluster-capi-operator-tests-ext
+
+# List all discovered tests (no cluster needed)
+./bin/cluster-capi-operator-tests-ext list tests
+
+# Run a specific suite against a cluster
+./bin/cluster-capi-operator-tests-ext run-suite capio/serial
+
+# Run a single test by exact name
+./bin/cluster-capi-operator-tests-ext run-test "[sig-cluster-lifecycle]... test name"
+```
+
+The `KUBECONFIG` env var is read directly by `config.GetConfig()` — it must
+point to a reachable OCP cluster, not just whatever `kubectl` has as
+current-context.

openshift-tests-extension/README.md

@@ -0,0 +1,85 @@
+/*
+Copyright 2025 Red Hat, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+	"github.com/spf13/cobra"
+
+	e2e "github.com/openshift/cluster-capi-operator/e2e"
+)
+
+func main() {
+	extensionRegistry := e.NewRegistry()
+
+	ext := e.NewExtension("openshift", "payload", "cluster-capi-operator")
+
+	defaultTimeout := 30 * time.Minute
+	disruptiveTimeout := 90 * time.Minute
+
+	ext.AddSuite(e.Suite{
+		Name:        "capio/parallel",
+		Qualifiers:  []string{`!labels.exists(l, l == "Serial") && !labels.exists(l, l == "Disruptive")`},
+		Parents:     []string{"openshift/conformance/parallel"},
+		TestTimeout: &defaultTimeout,
+	})
+
+	ext.AddSuite(e.Suite{
+		Name:        "capio/serial",
+		Qualifiers:  []string{`labels.exists(l, l == "Serial") && !labels.exists(l, l == "Disruptive")`},
+		Parents:     []string{"openshift/conformance/serial"},
+		Parallelism: 1,
+		TestTimeout: &defaultTimeout,
+	})
+
+	ext.AddSuite(e.Suite{
+		Name:             "capio/disruptive",
+		Qualifiers:       []string{`labels.exists(l, l == "Disruptive")`},
+		Parents:          []string{"openshift/disruptive-longrunning"},
+		Parallelism:      1,
+		ClusterStability: e.ClusterStabilityDisruptive,
+		TestTimeout:      &disruptiveTimeout,
+	})
+
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %v", err))
+	}
+
+	specs.AddBeforeAll(func() {
+		e2e.InitCommonVariables()
+	})
+
+	ext.AddSpecs(specs)
+
+	extensionRegistry.Register(ext)
+
+	root := &cobra.Command{
+		Long: "Cluster CAPI Operator tests extension for OpenShift",
+	}
+	root.AddCommand(cmd.DefaultExtensionCommands(extensionRegistry)...)
+
+	if err := root.Execute(); err != nil {
+		os.Exit(1)
+	}
+}