MCO-2211: MCO-2210: MCO-2234 MCO-Migrate MCO tests from openshift-tests-private

test/extended-priv/const.go

@@ -51,6 +51,8 @@ const (
 	VspherePlatform = "vsphere"
 	// BaremetalPlatform value used to identify BareMetal infrastructure
 	BaremetalPlatform = "baremetal"
+	// NonePlatform value used to identify a None Platform value
+	NonePlatform = "none"
 
 	// ExpirationDockerfileLabel Expiration label in Dockerfile
 	ExpirationDockerfileLabel = `LABEL maintainer="mco-qe-team" quay.expires-after=24h`

test/extended-priv/controller.go

@@ -191,3 +191,36 @@ func checkMCCPanic(oc *exutil.CLI) {
 
 	logger.Infof("OK!\n")
 }
+
+// GetLogsAsList returns the MCO controller logs as a list strings. One string per line
+func (mcc *Controller) GetLogsAsList() ([]string, error) {
+	logs, err := mcc.GetLogs()
+	if err != nil {
+		return nil, err
+	}
+
+	return strings.Split(logs, "\n"), nil
+}
+
+// GetFilteredLogsAsList returns the filtered logs as a lit of strings, one string per line.
+func (mcc *Controller) GetFilteredLogsAsList(regex string) ([]string, error) {
+	logs, err := mcc.GetLogsAsList()
+	if err != nil {
+		return nil, err
+	}
+
+	filteredLogs := []string{}
+	for _, line := range logs {
+		match, err := regexp.MatchString(regex, line)
+		if err != nil {
+			logger.Errorf("Error filtering log lines. Error: %s", err)
+			return nil, err
+		}
+
+		if match {
+			filteredLogs = append(filteredLogs, line)
+		}
+	}
+
+	return filteredLogs, nil
+}

test/extended-priv/machineconfig.go

@@ -17,6 +17,11 @@ type MachineConfigList struct {
 	ResourceList
 }
 
+// NewMachineConfigList construct a new node list struct to handle all existing nodes
+func NewMachineConfigList(oc *exutil.CLI) *MachineConfigList {
+	return &MachineConfigList{*NewResourceList(oc, "mc")}
+}
+
 // MachineConfig struct is used to handle MachineConfig resources in OCP
 type MachineConfig struct {
 	Resource
@@ -149,3 +154,36 @@ func DisableSkew(machineConfiguration *MachineConfiguration) {
 	o.Eventually(machineConfiguration.IsGenerationUpToDate, "2m", "10s").Should(o.BeTrue(), "MachineConfiguration observedGeneration did not catch up to generation")
 	logger.Infof("Skew functionality has been disabled")
 }
+
+// GetRenderedMachineConfigForMaster returns a list with all the MCs  whose name starts with "render-master"
+func (mcl *MachineConfigList) GetRenderedMachineConfigForMaster() ([]*MachineConfig, error) {
+	mcl.SetItemsFilter(`?(@.metadata.ownerReferences[0].name=="master")`)
+	allMCs, err := mcl.GetAll()
+	if err != nil {
+		return nil, err
+	}
+
+	returnMCs := []*MachineConfig{}
+
+	for _, mc := range allMCs {
+		if strings.HasPrefix(mc.GetName(), "rendered-master") {
+			returnMCs = append(returnMCs, &MachineConfig{Resource: *mc})
+		}
+	}
+
+	return returnMCs, nil
+}
+
+func (mcl *MachineConfigList) GetRenderedMachineConfigForMasterOrFail() []*MachineConfig {
+	renderedMcMasterList, err := mcl.GetRenderedMachineConfigForMaster()
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting the list of the machineconfigs that were created by a MCP ")
+	return renderedMcMasterList
+
+}
+
+// GetMachineConfigCreatedByMCPs returns a list of the machineconfigs that were created by a MCP
+func (mcl *MachineConfigList) GetMCPRenderedMachineConfigsOrFail() []*MachineConfig {
+	renderedMcList, err := mcl.GetRenderedMachineConfigForMaster()
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting the list of the machineconfigs that were created by a MCP ")
+	return renderedMcList
+}

test/extended-priv/machineconfigpool.go

@@ -71,6 +71,20 @@ func (mcp *MachineConfigPool) pause(enable bool) {
 	o.Expect(err).NotTo(o.HaveOccurred())
 }
 
+// SetMaxUnavailable sets the value for maxUnavailable
+func (mcp *MachineConfigPool) SetMaxUnavailable(maxUnavailable int) {
+	logger.Infof("patch mcp %v, change spec.maxUnavailable to %d", mcp.name, maxUnavailable)
+	err := mcp.Patch("merge", fmt.Sprintf(`{"spec":{"maxUnavailable": %d}}`, maxUnavailable))
+	o.Expect(err).NotTo(o.HaveOccurred())
+}
+
+// RemoveMaxUnavailable removes spec.maxUnavailable attribute from the pool config
+func (mcp *MachineConfigPool) RemoveMaxUnavailable() {
+	logger.Infof("patch mcp %v, removing spec.maxUnavailable", mcp.name)
+	err := mcp.Patch("json", `[{ "op": "remove", "path": "/spec/maxUnavailable" }]`)
+	o.Expect(err).NotTo(o.HaveOccurred())
+}
+
 // IsPaused return true is mcp is paused
 func (mcp *MachineConfigPool) IsPaused() bool {
 	return IsTrue(mcp.GetOrFail(`{.spec.paused}`))
@@ -1492,6 +1506,23 @@ func FilterExtensions(extensions map[string][]string, hasARM64, fips bool, osIma
 	return filteredExtensions, extensionNames, packages
 }
 
+// GetAllApplicableExtensionsToMCPOrFail returns all the extensions that are supported for the given MCP, and all the packages that will install those extensions
+func GetAllApplicableExtensionsToMCPOrFail(mcp *MachineConfigPool) (extensions, packages []string) {
+	fips := isFIPSEnabledInClusterConfig(mcp.GetOC().AsAdmin())
+
+	armNodes, err := mcp.GetNodesByArchitecture(architecture.ARM64)
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting the list of ARM nodes in %s", mcp)
+
+	osImageStream, err := GetEffectiveOsImageStream(mcp)
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting effective osImageStream from MCP %s", mcp.GetName())
+
+	_, extensions, packages = FilterExtensions(AllExtenstions, len(armNodes) > 0, fips, osImageStream)
+
+	logger.Infof("All extensions that can be applied to %s: %s", mcp, extensions)
+	logger.Infof("All packages that will be installed with those extensions: %s", packages)
+	return extensions, packages
+}
+
 func (mcp *MachineConfigPool) GetNodesWithoutArchitecture(arch architecture.Architecture, archs ...architecture.Architecture) ([]*Node, error) {
 	archsList := arch.String()
 	for _, itemArch := range archs {
@@ -1548,3 +1579,97 @@ func GetPoolWithArchDifferentFromOrFail(oc *exutil.CLI, arch architecture.Archit
 	e2e.Failf("Something went wrong. There is no suitable pool to execute the test case. There is no pool with nodes using  an architecture different from %s", arch)
 	return nil
 }
+
+// GetSortedUpdatedNodes returns a list of nodes in the order that they are being updated by the MCO
+// If maxUnavailable>0, then the function will fail if more that maxUpdatingNodes are being updated at the same time
+func (mcp *MachineConfigPool) GetSortedUpdatedNodes(maxUnavailable int) []*Node {
+	timeToWait := mcp.estimateWaitDuration()
+	logger.Infof("Waiting %s in pool %s for all nodes to start updating.", timeToWait, mcp.name)
+
+	poolNodes, errget := mcp.GetNodes()
+	o.Expect(errget).NotTo(o.HaveOccurred(), fmt.Sprintf("Cannot get nodes in pool %s", mcp.GetName()))
+
+	pendingNodes := poolNodes
+	updatedNodes := []*Node{}
+	immediate := false
+	err := wait.PollUntilContextTimeout(context.TODO(), 20*time.Second, timeToWait, immediate, func(_ context.Context) (bool, error) {
+		// If there are degraded machines, stop polling, directly fail
+		degradedstdout, degradederr := mcp.getDegradedMachineCount()
+		if degradederr != nil {
+			logger.Errorf("the err:%v, and try next round", degradederr)
+			return false, nil
+		}
+
+		if degradedstdout != 0 {
+			logger.Errorf("Degraded MC:\n%s", mcp.PrettyString())
+			exutil.AssertWaitPollNoErr(fmt.Errorf("Degraded machines"), fmt.Sprintf("mcp %s has degraded %d machines", mcp.name, degradedstdout))
+		}
+
+		// Check that there aren't more thatn maxUpdatingNodes updating at the same time
+		if maxUnavailable > 0 {
+			totalUpdating := 0
+			for _, node := range poolNodes {
+				isUpdating, err := node.IsUpdating()
+				if err != nil {
+					logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err)
+					return false, err
+				}
+				if isUpdating {
+					totalUpdating++
+				}
+			}
+			if totalUpdating > maxUnavailable {
+				// print nodes for debug
+				mcp.oc.Run("get").Args("nodes").Execute()
+				exutil.AssertWaitPollNoErr(fmt.Errorf("maxUnavailable Not Honored. Pool %s, error: %d nodes were updating at the same time. Only %d nodes should be updating at the same time", mcp.GetName(), totalUpdating, maxUnavailable), "")
+			}
+		}
+
+		remainingNodes := []*Node{}
+		for _, node := range pendingNodes {
+			isUpdating, err := node.IsUpdating()
+			if err != nil {
+				logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err)
+				return false, err
+			}
+			if isUpdating {
+				logger.Infof("Node %s is UPDATING", node.GetName())
+				updatedNodes = append(updatedNodes, node)
+			} else {
+				remainingNodes = append(remainingNodes, node)
+			}
+		}
+
+		if len(remainingNodes) == 0 {
+			logger.Infof("All nodes have started to be updated on mcp %s", mcp.name)
+			return true, nil
+
+		}
+		logger.Infof(" %d remaining nodes", len(remainingNodes))
+		pendingNodes = remainingNodes
+		return false, nil
+	})
+
+	exutil.AssertWaitPollNoErr(err, fmt.Sprintf("Could not get the list of updated nodes on mcp %s", mcp.name))
+	return updatedNodes
+}
+
+// IsOCL returns true if the pool is using On Cluster Layering functionality
+func (mcp MachineConfigPool) IsOCL() (bool, error) {
+	isOCLEnabled, err := IsFeaturegateEnabled(mcp.GetOC(), "OnClusterBuild")
+	if err != nil {
+		return false, err
+	}
+	if !isOCLEnabled {
+		logger.Infof("IS pool %s OCL: false", mcp.GetName())
+		return false, nil
+	}
+
+	mosc, err := mcp.GetMOSC()
+	if err != nil {
+		return false, err
+	}
+	isOCL := mosc != nil
+	logger.Infof("IS pool %s OCL: %t", mcp.GetName(), isOCL)
+	return isOCL, err
+}