Add E2E test for MaxParallelUpgrades enforcement during userData changes

Enhances the existing testUserDataTamper test to verify that
MaxParallelUpgrades=1 is enforced when userData template changes trigger
machine deletions (pub-key-hash mismatch deletion path).

The test now:
1. Deploys parallel-upgrades-checker monitoring job before userData change
2. Verifies the checker did not fail (which would indicate >1 simultaneous upgrades)
3. Cleans up the checker job after the test

This tests the specific scenario where the bug was bypassing
MaxParallelUpgrades: userData template changes causing simultaneous
deletion of machines across MachineSets.

The implementation leverages existing infrastructure:
- hack/e2e/resources/parallel-upgrade-checker-job.yaml (monitoring job)
- testUserDataTamper (already triggers userData changes)
- waitForNewMachineNodes (already waits for machine recreation)

Helper functions added:
- deployParallelUpgradesChecker() - Deploys the monitoring job
- verifyParallelUpgradesChecker() - Checks for violations
- cleanupParallelUpgradesChecker() - Removes the job

The test will skip if fewer than 2 Machine nodes are present, as
the parallel behavior cannot be verified with a single node.

Signed-off-by: Ranjith Rajaram <ranjith@redhat.com>

Author: Ranjith Rajaram

test/e2e/main_test.go

@@ -4,14 +4,18 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
+	"os/exec"
+	"strings"
 	"testing"
 	"time"
 
 	config "github.com/openshift/api/config/v1"
 	imageClient "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1"
 	"github.com/openshift/library-go/pkg/image/imageutil"
 	core "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/openshift/windows-machine-config-operator/pkg/retry"
@@ -184,3 +188,76 @@ func TestMain(m *testing.M) {
 
 	os.Exit(m.Run())
 }
+
+// deployParallelUpgradesChecker deploys the monitoring job that checks for MaxParallelUpgrades violations
+func (tc *testContext) deployParallelUpgradesChecker() error {
+	// Read the job YAML template
+	jobYAML, err := os.ReadFile("hack/e2e/resources/parallel-upgrade-checker-job.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to read parallel upgrades checker job YAML: %w", err)
+	}
+
+	// Get the tools image from the OpenShift imagestream
+	toolsImage, err := getOpenShiftToolsImage(tc.client.Images)
+	if err != nil {
+		// Fall back to a default tools image if imagestream is not available
+		log.Printf("Warning: could not get tools image from imagestream: %v. Using default.", err)
+		toolsImage = "registry.ci.openshift.org/ocp/4.17:tools"
+	}
+
+	// Replace placeholders
+	jobYAMLStr := string(jobYAML)
+	jobYAMLStr = strings.Replace(jobYAMLStr, "REPLACE_WITH_OPENSHIFT_TOOLS_IMAGE", toolsImage, 1)
+	jobYAMLStr = strings.Replace(jobYAMLStr, "namespace: wmco-test",
+		fmt.Sprintf("namespace: %s", tc.workloadNamespace), 1)
+
+	// Create the job using kubectl apply
+	cmd := exec.Command("kubectl", "apply", "-f", "-")
+	cmd.Stdin = strings.NewReader(jobYAMLStr)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to create parallel upgrades checker job: %w\nOutput: %s", err, string(output))
+	}
+
+	log.Printf("Deployed parallel upgrades checker job in namespace %s", tc.workloadNamespace)
+	return nil
+}
+
+// verifyParallelUpgradesChecker checks if the parallel upgrades checker job detected any violations
+func (tc *testContext) verifyParallelUpgradesChecker() error {
+	// Check for failed pods
+	failedPods, err := tc.client.K8s.CoreV1().Pods(tc.workloadNamespace).List(context.TODO(), meta.ListOptions{
+		LabelSelector: "job-name=parallel-upgrades-checker",
+		FieldSelector: "status.phase=Failed",
+	})
+	if err != nil {
+		return fmt.Errorf("error checking parallel upgrades checker status: %w", err)
+	}
+
+	// Gather logs for debugging
+	_, err = tc.gatherPodLogs("job-name=parallel-upgrades-checker", false)
+	if err != nil {
+		log.Printf("warning: unable to gather parallel upgrades checker logs: %v", err)
+	}
+
+	// If any pod failed, the check detected a violation
+	if len(failedPods.Items) > 0 {
+		return fmt.Errorf("parallel upgrades check failed: MaxParallelUpgrades was violated (%d pods failed)",
+			len(failedPods.Items))
+	}
+
+	log.Printf("Parallel upgrades checker passed: MaxParallelUpgrades=1 was enforced")
+	return nil
+}
+
+// cleanupParallelUpgradesChecker removes the parallel upgrades checker job
+func (tc *testContext) cleanupParallelUpgradesChecker() {
+	propagationPolicy := meta.DeletePropagationBackground
+	err := tc.client.K8s.BatchV1().Jobs(tc.workloadNamespace).Delete(context.TODO(),
+		"parallel-upgrades-checker", meta.DeleteOptions{
+			PropagationPolicy: &propagationPolicy,
+		})
+	if err != nil && !apierrors.IsNotFound(err) {
+		log.Printf("warning: failed to cleanup parallel upgrades checker job: %v", err)
+	}
+}

test/e2e/secrets_test.go

@@ -78,11 +78,23 @@ func (tc *testContext) testUserData(t *testing.T) {
 }
 
 // testUserDataTamper tests if userdata reverts to previous value if updated
+// AND verifies MaxParallelUpgrades=1 during userData-triggered deletions
 func (tc *testContext) testUserDataTamper(t *testing.T) {
 	validUserDataSecret, err := tc.client.K8s.CoreV1().Secrets(clusterinfo.MachineAPINamespace).Get(context.TODO(),
 		secrets.UserDataSecret, meta.GetOptions{})
 	require.NoError(t, err, "could not find Windows userData secret in required namespace")
 
+	// Requires at least 2 Machine nodes to test parallel behavior
+	require.NoError(t, tc.loadExistingNodes(), "error loading existing nodes")
+	if len(gc.machineNodes) < 2 {
+		t.Skip("Test requires at least 2 Machine nodes to verify MaxParallelUpgrades enforcement")
+	}
+
+	// Deploy parallel upgrades checker before triggering userData change
+	err = tc.deployParallelUpgradesChecker()
+	require.NoError(t, err, "could not deploy parallel upgrades checker")
+	defer tc.cleanupParallelUpgradesChecker()
+
 	updatedSecret := validUserDataSecret.DeepCopy()
 	updatedSecret.Data["userData"] = []byte("invalid data")
 	_, err = tc.client.K8s.CoreV1().Secrets(clusterinfo.MachineAPINamespace).Update(context.TODO(), updatedSecret,
@@ -93,6 +105,10 @@ func (tc *testContext) testUserDataTamper(t *testing.T) {
 	// until the Machine is back up.
 	assert.NoError(t, tc.waitForNewMachineNodes(), "error waiting for Machine nodes to be reconfigured")
 	assert.NoError(t, tc.waitForValidUserData(validUserDataSecret), "error waiting for valid userdata")
+
+	// Verify that MaxParallelUpgrades=1 was enforced throughout the process
+	err = tc.verifyParallelUpgradesChecker()
+	assert.NoError(t, err, "MaxParallelUpgrades was violated during userData change")
 }
 
 // waitForNewMachineNodes returns an error if waitForConfiguredWindowsNodes returns the same Machine backed nodes