Merge pull request #704 from camilamacedo86/ote-improve-output

OPRUN-4587: Make OTE local output easier to read

Author: openshift-merge-bot[bot]

openshift/tests-extension/Makefile

@@ -181,6 +181,11 @@ build: #HELP Build the extended tests binary
 	@mkdir -p $(TOOLS_BIN_DIR)
 	GO_COMPLIANCE_POLICY="exempt_all" go build -ldflags "$(LDFLAGS)" -mod=vendor -o $(TOOLS_BIN_DIR)/olmv1-tests-ext ./cmd/...
 
+.PHONY: build-local-dev
+build-local-dev: #HELP Build the extended tests binary with local dev commands (for local development only)
+	@mkdir -p $(TOOLS_BIN_DIR)
+	GO_COMPLIANCE_POLICY="exempt_all" go build -tags dev -ldflags "$(LDFLAGS)" -mod=vendor -o $(TOOLS_BIN_DIR)/olmv1-tests-ext ./cmd/...
+
 .PHONY: update-metadata
 update-metadata: #HELP Build and run 'update-metadata' to generate test metadata
 	$(TOOLS_BIN_DIR)/olmv1-tests-ext update --component openshift:payload:olmv1
@@ -245,3 +250,21 @@ verify-metadata: update-metadata
 .PHONY: verify-images-json #HELP Verify that 'images' command outputs valid JSON
 verify-images-json:
 	@./hack/verify-images-json.sh $(TOOLS_BIN_DIR)/olmv1-tests-ext
+
+#SECTION Local Testing (Human-Readable Output)
+
+.PHONY: test-local
+test-local: build-local-dev #HELP Run tests locally with clean, human-readable output (usage: make test-local SUITE=olmv1/all)
+	@if [ -z "$(SUITE)" ]; then \
+		echo "ERROR: Please specify SUITE. Example: make test-local SUITE=olmv1/all"; \
+		exit 1; \
+	fi
+	@$(TOOLS_BIN_DIR)/olmv1-tests-ext run-suite-dev "$(SUITE)"
+
+.PHONY: test-local-single
+test-local-single: build-local-dev #HELP Run a single test with clean output (usage: make test-local-single TEST="test name")
+	@if [ -z "$(TEST)" ]; then \
+		echo "ERROR: Please specify TEST. Example: make test-local-single TEST=\"[sig-olmv1] OLMv1 should pass\""; \
+		exit 1; \
+	fi
+	@$(TOOLS_BIN_DIR)/olmv1-tests-ext run-test-dev -n "$(TEST)"

openshift/tests-extension/README.md

@@ -33,6 +33,10 @@ opts out with a skip label like `[Skipped:Disconnected]`.
 | Release jobs | [amd64.ocp.releases.ci.openshift.org](https://amd64.ocp.releases.ci.openshift.org/) | Click any build to see all validation jobs run against it |
 | Component Readiness | [Sippy](https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main) | Test results feed here. Failures trigger a red alert and a Slack notification to the team |
 | OpenShift CI docs | [docs.ci.openshift.org](https://docs.ci.openshift.org/) | General documentation on how OpenShift CI works |
+| OTE Framework | [github.com/openshift-eng/openshift-tests-extension](https://github.com/openshift-eng/openshift-tests-extension) | OpenShift Tests Extension framework - wraps Ginkgo and exposes test commands |
+| OTE Enhancement | [OTE Enhancement Proposal](https://github.com/openshift/enhancements/blob/master/enhancements/testing/openshift-tests-extension.md) | Official design doc for the OpenShift Tests Extension framework |
+| Ginkgo v2 docs | [onsi.github.io/ginkgo](https://onsi.github.io/ginkgo/) | Official Ginkgo BDD testing framework documentation |
+| Ginkgo CLI reference | [Ginkgo CLI flags](https://onsi.github.io/ginkgo/#the-ginkgo-cli) | Complete reference for Ginkgo command-line flags and options |
 | Help with alerts | `#forum-ocp-testplatform` on Slack | Managed by the TRT team |
 | Help with OTE | `#wg-openshift-tests-extension` on Slack | Questions about the OpenShift Tests Extension framework |
 
@@ -106,55 +110,145 @@ Example ([source](https://github.com/openshift/release/blob/main/ci-operator/con
 
 ## How to Run the Tests Locally
 
-| Command                                         | Description                                                              |
-|-------------------------------------------------|--------------------------------------------------------------------------|
-| `make build`                                    | Builds the OLMv1 test binary.                                            |
-| `./bin/olmv1-tests-ext info`                    | Shows info about the test binary and registered test suites.             |
-| `./bin/olmv1-tests-ext list`                    | Lists all available test cases.                                          |
-| `./bin/olmv1-tests-ext run-suite olmv1/all`     | Runs the full OLMv1 test suite.                                          |
-| `./bin/olmv1-tests-ext run-test -n <test-name>` | Runs one specific test. Replace <test-name> with the test's full name.   |
+You must run OTE tests (`./bin/olmv1-tests-ext`) against an OCP Cluster with TechPreview Features enabled.
 
+### Setup: Get an OpenShift Cluster
 
-## How to Run the Tests Locally
+Use Cluster Bot to create an OpenShift cluster with OLMv1 installed:
 
-The tests can be run locally using the `olmv1-tests-ext` binary against an OpenShift cluster.
-These tests are specifically designed for OpenShift and require OpenShift-specific APIs and features.
+```shell
+launch 4.20 gcp,techpreview
+```
 
-Use the environment variable `KUBECONFIG` to point to your cluster configuration file such as:
+Set `KUBECONFIG`:
 
 ```shell
-KUBECONFIG=path/to/kubeconfig ./bin/olmv1-tests-ext run-test -n <test-name>
+mv ~/Downloads/cluster-bot-2025-08-06-082741.kubeconfig ~/.kube/cluster-bot.kubeconfig
+export KUBECONFIG=~/.kube/cluster-bot.kubeconfig
 ```
 
-To run tests that include tech preview features, 
-you need an OpenShift cluster with OLMv1 installed and those features enabled.
+### Two Ways to Run Tests
 
-### Local Test using OLMv1 on OpenShift
+#### 1. **Developer-Friendly Output** (For local development)
 
-1. Use the `Cluster Bot` to create an OpenShift cluster with OLMv1 installed.
+Use the local dev commands (`run-suite-dev`, `run-test-dev`) that provide clean, human-readable output:
 
-**Example:**
+**Implementation:** Local dev commands in `localdevoutput/` are excluded from production builds using Go build tags. Only included with `make build-local-dev`. See [localdevoutput/README.md](localdevoutput/README.md).
 
-```shell
-launch 4.20 gcp,techpreview
+| Command | Description |
+|---------|-------------|
+| `make build-local-dev` | Builds the test binary with local dev commands |
+| `make test-local SUITE=olmv1/all` | Runs a test suite with clean, color-coded output |
+| `make test-local-single TEST="test name"` | Runs a single test with clean output |
+| `make list-test-names` | Lists all available test names |
+
+**Example**
+
+```bash
+export KUBECONFIG=~/.kube/cluster-bot.kubeconfig
+make build-local-dev
+make test-local SUITE=olmv1/all
 ```
 
-2. Set the `KUBECONFIG` environment variable to point to your OpenShift cluster configuration file.
+**Output:** Clean, color-coded summary with live progress:
+```text
+[46/46] ▶ Running: [sig-olmv1][OCPFeatureGate:NewOLMWebhookProviderOpenshiftServiceCA] OLMv1 operator with webhooks should have a working validating webhook
+  ✓ PASSED [194.1 seconds] (Total: ✓45 ✗0)
 
-**Example:**
 
-```shell
-mv ~/Downloads/cluster-bot-2025-08-06-082741.kubeconfig ~/.kube/cluster-bot.kubeconfig
-export KUBECONFIG=~/.kube/cluster-bot.kubeconfig
+════════════════════════════════════════════════════════
+  Final Summary
+════════════════════════════════════════════════════════
+✓ Passed:  45
+✗ Failed:  0
+⊘ Skipped: 1
+
+✓ ALL TESTS PASSED!
 ```
 
-3. Run the tests using the `olmv1-tests-ext` binary.
+#### 2. **Raw OTE Framework Output** (For CI/CD integration)
+
+Run the binary directly for structured JSON reports:
+
+| Command | Description |
+|---------|-------------|
+| `./bin/olmv1-tests-ext info` | Shows info about the test binary and registered test suites |
+| `./bin/olmv1-tests-ext list` | Lists all available test cases |
+| `./bin/olmv1-tests-ext run-suite olmv1/all` | Runs the full OLMv1 test suite with JSON output |
+| `./bin/olmv1-tests-ext run-test -n <test-name>` | Runs one specific test with JSON output |
 
 **Example:**
-```shell
+
+```bash
+export KUBECONFIG=~/.kube/cluster-bot.kubeconfig
+make build
 ./bin/olmv1-tests-ext run-suite olmv1/all
 ```
 
+**Output:** Structured JSON report (as used by Component Readiness and other integrated solutions):
+```text
+Running Suite:  - /Users/camilam/go/src/github/operator-framework-operator-controller/openshift/tests-extension
+===============================================================================================================
+Random Seed: 1753508546 - will randomize all specs
+
+Will run 1 of 1 specs
+------------------------------
+[sig-olmv1] OLMv1 should pass a trivial sanity check
+/Users/camilam/go/src/github/operator-framework-operator-controller/openshift/tests-extension/test/olmv1.go:26
+• [0.000 seconds]
+------------------------------
+
+Ran 1 of 1 Specs in 0.000 seconds
+SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped
+[
+  {
+    "name": "[sig-olmv1] OLMv1 should pass a trivial sanity check",
+    "lifecycle": "blocking",
+    "duration": 0,
+    "startTime": "2025-07-26 05:42:26.553852 UTC",
+    "endTime": "2025-07-26 05:42:26.580263 UTC",
+    "result": "passed",
+    "output": ""
+  }
+]
+```
+
+This is the same output format used by:
+- **Component Readiness** ([Sippy](https://sippy.dptools.openshift.org/sippy-ng/component_readiness/main))
+- **OpenShift CI/CD** pipeline
+- **Release validation jobs**
+- Any automated test processing tools
+
+**When to use which:**
+- Use **clean output** (`make test-local` or `make test-local-single`) for local development, debugging, and quick visual feedback
+- Use **raw output** (direct binary execution with `./bin/olmv1-tests-ext`) when you need JSON reports, CI/CD integration, or programmatic processing
+
+### Discovering Available Flags
+
+The OTE framework wraps Ginkgo and exposes its own set of commands and flags. To see what's available:
+
+```bash
+# See all available commands
+./bin/olmv1-tests-ext --help
+
+# See flags for running test suites
+./bin/olmv1-tests-ext run-suite --help
+
+# See flags for running individual tests
+./bin/olmv1-tests-ext run-test --help
+```
+
+**Available OTE-specific flags:**
+- `--component string` - Specify the component to enable (default "default")
+- `--max-concurrency int` - Maximum number of tests to run in parallel (default 10)
+- `--output string` - Output mode (default "json")
+- `--junit-path string` - Write results to JUnit XML (for `run-suite`)
+- `--names stringArray` - Specify test name, can be used multiple times (for `run-test`)
+
+**Note:** The OTE framework does not expose all Ginkgo CLI flags.
+It provides a simplified interface focused on running tests in OpenShift environments.
+For full Ginkgo flag reference, see the [Ginkgo CLI documentation](https://onsi.github.io/ginkgo/#the-ginkgo-cli).
+
 ## Development Workflow
 
 - Add or update tests in: `openshift/tests-extension/tests/`
@@ -283,14 +377,16 @@ that the metadata is up to date:
 
 ## Makefile Commands
 
-| Target                   | Description                                                                  |
-|--------------------------|------------------------------------------------------------------------------|
-| `make build`             | Builds the test binary.                                                      |
-| `make update-metadata`   | Updates the metadata JSON file.                                              |
-| `make build-update`      | Runs build + update-metadata + cleans codeLocations.                         |
-| `make verify`            | Runs formatting, vet, and linter.                                            |
-| `make list-test-names`   | Shows all test names in the binary.                                          |
-| `make clean-metadata`    | Removes machine-specific codeLocations from the JSON metadata. [More info](https://issues.redhat.com/browse/TRT-2186) |
+| Target                           | Description                                                                  |
+|----------------------------------|------------------------------------------------------------------------------|
+| `make build`                     | Builds the test binary.                                                      |
+| `make test-local SUITE=<suite>`  | Runs a test suite with clean, human-readable output for local development. |
+| `make test-local-single TEST="<name>"` | Runs a single test with clean, human-readable output. |
+| `make list-test-names`           | Shows all test names in the binary.                                          |
+| `make update-metadata`           | Updates the metadata JSON file.                                              |
+| `make build-update`              | Runs build + update-metadata + cleans codeLocations.                         |
+| `make verify`                    | Runs formatting, vet, and linter.                                            |
+| `make clean-metadata`            | Removes machine-specific codeLocations from the JSON metadata. [More info](https://issues.redhat.com/browse/TRT-2186) |
 
 **Note:** Metadata is stored in: `.openshift-tests-extension/openshift_payload_olmv1.json`
 

openshift/tests-extension/cmd/main.go

@@ -14,12 +14,13 @@ import (
 	"os"
 	"strings"
 
-	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	otecmd "github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
 	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
 	"github.com/spf13/cobra"
 
+	localdevcmd "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/localdevoutput/cmd"
 	"github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/pkg/env"
 	_ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test"
 	_ "github.com/openshift/operator-framework-operator-controller/openshift/tests-extension/test/qe/specs"
@@ -300,33 +301,38 @@ func main() {
 	}
 
 	// Get all default commands from the extension framework
-	allCommands := cmd.DefaultExtensionCommands(registry)
+	allCommands := otecmd.DefaultExtensionCommands(registry)
 
-	// Add KUBECONFIG check to run-suite and run-test commands only.
+	// Add local dev commands for local development (only included when built with -tags dev)
+	allCommands = append(allCommands, localdevcmd.RegisterLocalDevCommands(registry)...)
+
+	// Add KUBECONFIG check to run-suite, run-test, and dev variants.
 	// Other commands (list, info, images, update, completion, help) don't need KUBECONFIG.
 	for _, command := range allCommands {
-		// Identify run-suite and run-test commands by their Use field
-		if command.Use == "run-suite NAME" || command.Use == "run-test [-n NAME...] [NAME]" {
-			// Save the original RunE function
-			originalRunE := command.RunE
-
-			// Wrap it with KUBECONFIG check
-			command.RunE = func(cmd *cobra.Command, args []string) error {
-				// Check KUBECONFIG before running the test
+		switch command.Name() {
+		case "run-suite", "run-test", "run-suite-dev", "run-test-dev":
+			localCmd := command
+			originalRunE := localCmd.RunE
+
+			localCmd.RunE = func(cmd *cobra.Command, args []string) error {
 				if err := exutil.CheckKubeconfigSet(); err != nil {
 					return err
 				}
-				// Call the original RunE function
-				return originalRunE(cmd, args)
+				if originalRunE != nil {
+					return originalRunE(cmd, args)
+				}
+				if localCmd.Run != nil {
+					localCmd.Run(cmd, args)
+					return nil
+				}
+				return fmt.Errorf("command %s has no Run or RunE function", localCmd.Name())
 			}
 		}
 	}
 
 	root.AddCommand(allCommands...)
 
-	if err := func() error {
-		return root.Execute()
-	}(); err != nil {
+	if err := root.Execute(); err != nil {
 		os.Exit(1)
 	}
 }

openshift/tests-extension/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/openshift/origin v1.5.0-alpha.3.0.20251010041851-79ff1dbbe815
 	github.com/operator-framework/operator-controller v1.8.1-0.20260319123036-8ccea5a0cf67
 	github.com/pborman/uuid v1.2.1
+	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.10.2
 	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/pretty v1.2.1
@@ -73,7 +74,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect

openshift/tests-extension/localdevoutput/README.md

@@ -0,0 +1,54 @@
+# Local Dev Output Commands
+
+Developer-friendly test commands with clean, human-readable output for local execution.
+
+## Structure
+
+```text
+localdevoutput/
+├── cmd/                          # Cobra commands for local development
+│   ├── register.go               # Returns nil (production build)
+│   ├── register_local_dev.go     # Registers local dev commands (with -tags dev)
+│   ├── run_suite_local_dev.go    # run-suite-dev command
+│   └── run_test_local_dev.go     # run-test-dev command
+└── pkg/output/                   # Output formatting
+    ├── formatter.go              # ANSI colors, progress, summaries
+    └── writer.go                 # OTE ResultWriter implementation
+```
+
+## Purpose
+
+These commands are **for local execution only**. They provide human-readable output when testing against OCP clusters locally.
+
+## Build Separation
+
+- **Production build**: `make build` → Excludes local dev commands (ships to OCP payload)
+- **Local dev build**: `make build-local-dev` → Includes local dev commands (for developers)
+
+Build tags (`//go:build dev`) ensure dev commands are only compiled when explicitly requested with `-tags dev`.
+
+### Output Flow
+
+```text
+Test Run → ResultWriter.Write(result) → Formatter → Colored Terminal Output
+```
+
+The `CleanResultWriter` implements OTE's `ResultWriter` interface to intercept test results and format them with colors and progress indicators.
+
+## Usage
+
+```bash
+# Run test suite with clean output
+make test-local SUITE=olmv1/all
+
+# Run single test
+make test-local-single TEST="[sig-olmv1] test name"
+
+# Direct binary usage
+./bin/olmv1-tests-ext run-suite-dev olmv1/all
+./bin/olmv1-tests-ext run-test-dev -n "test name"
+```
+
+## Why Separate Directory?
+
+Isolated in `localdevoutput/` to keep local development tools separate from the core test framework that ships to production.

openshift/tests-extension/localdevoutput/cmd/register.go

@@ -0,0 +1,12 @@
+//go:build !dev
+
+package cmd
+
+import (
+	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	"github.com/spf13/cobra"
+)
+
+func RegisterLocalDevCommands(registry *extension.Registry) []*cobra.Command {
+	return nil
+}