From 753896d305acbe2e89c138ecb51613bd796e4fa9 Mon Sep 17 00:00:00 2001 From: Harel Meir Date: Tue, 9 Jun 2026 10:24:05 +0300 Subject: [PATCH] [4.19] eus upgrade alignment Signed-off-by: Harel Meir --- conftest.py | 15 +- docs/INSTALL_AND_UPGRADE.md | 21 +- tests/conftest.py | 20 +- .../product_upgrade/conftest.py | 96 +++++---- .../product_upgrade/test_eus_upgrade.py | 10 +- .../product_upgrade/utils.py | 195 ++++++++++++++---- tests/virt/upgrade/conftest.py | 63 +++--- tests/virt/upgrade/test_upgrade_virt.py | 2 +- tests/virt/upgrade/utils.py | 45 ++-- tox.ini | 2 +- utilities/operator.py | 7 +- 11 files changed, 289 insertions(+), 187 deletions(-) diff --git a/conftest.py b/conftest.py index e3921659a4..dbc6b170a8 100644 --- a/conftest.py +++ b/conftest.py @@ -22,6 +22,7 @@ from _pytest.runner import CallInfo from kubernetes.dynamic.exceptions import ConflictError from ocp_resources.resource import get_client +from packaging.version import Version from pyhelper_utils.shell import run_command from pytest import Item from pytest_testconfig import config as py_config @@ -151,7 +152,6 @@ def pytest_addoption(parser): "--eus-ocp-images", help="Comma-separated OCP images to use for EUS-to-EUS upgrade.", ) - install_upgrade_group.addoption("--eus-cnv-target-version", help="target CNV version for eus upgrade") install_upgrade_group.addoption( "--upgrade-skip-default-sc-setup", help="Skip the fixture that changes the default sc in upgrade lane", @@ -328,18 +328,21 @@ def pytest_cmdline_main(config): if upgrade_option == "ocp" and not config.getoption("ocp_image"): raise ValueError("Running with --upgrade ocp: Missing --ocp-image") - if upgrade_option == "cnv": + if upgrade_option in ("cnv", "eus"): if not config.getoption("cnv_version"): raise ValueError("Missing --cnv-version") if not config.getoption("cnv_image"): - if config.getoption("cnv_source") != "production": + if upgrade_option == "eus" or config.getoption("cnv_source") != "production": raise ValueError("Missing --cnv-image") - if upgrade_option == "eus": + if upgrade_option == "eus" and not config.option.collectonly: + cnv_version = config.getoption("cnv_version") + if Version(version=cnv_version).minor % 2: + raise ValueError(f"EUS target version {cnv_version} must have an even minor version") eus_ocp_images = config.getoption("eus_ocp_images") if not (eus_ocp_images and len(eus_ocp_images.split(",")) == 2): raise ValueError( - f"Two OCP images are needed to perform EUS-to-EUS upgrade with --eus-ocp-images." + f"Two OCP images are needed for EUS-to-EUS upgrade with --eus-ocp-images." f" Provided images: {eus_ocp_images}" ) @@ -834,7 +837,7 @@ def is_skip_must_gather(node: Node) -> bool: def get_inspect_command_namespace_string(node: Node, test_name: str) -> str: namespace_str = "" - components = [key for key in NAMESPACE_COLLECTION.keys() if f"tests/{key}/" in test_name] + components = [key for key in NAMESPACE_COLLECTION if f"tests/{key}/" in test_name] if not components: LOGGER.warning(f"{test_name} does not require special data collection on failure") else: diff --git a/docs/INSTALL_AND_UPGRADE.md b/docs/INSTALL_AND_UPGRADE.md index 28ed1cf759..d7d8c0250a 100644 --- a/docs/INSTALL_AND_UPGRADE.md +++ b/docs/INSTALL_AND_UPGRADE.md @@ -96,12 +96,27 @@ To upgrade to cnv 4.Y.z, using the cnv image that has been shipped, following co ``` #### EUS upgrade -You must provide --eus-ocp-images via cli, which is two comma separated ocp images for EUS upgrade. -The default target cnv version will be 4.Y+2.0. Optionally, --eus-csv-target-version can be provided for 4.Y+2.z version. +EUS-to-EUS upgrades are only viable between even-numbered minor versions (e.g., 4.20 -> 4.22). + +Parameters: + +| Parameter Name | Requirement | Default Value | Possible Value | +|:------------------|:---------------:|:-------------:|:----------------------------:| +| `--cnv-version` | **Required** | - | 4.Y.z | +| `--cnv-image` | **Required** | - | -image path- | +| `--cnv-channel` | **Optional** | stable | stable, candidate, nightly | +| `--eus-ocp-images` | **Required** | - | comma-separated OCP images | + Command to run entire upgrade test suite for EUS upgrade, including pre and post upgrade validation: ```bash ---upgrade eus --eus-ocp-images , --eus-cnv-target-version <4.Y+2.z|None> +--upgrade eus --cnv-version --cnv-image --eus-ocp-images , +``` + +Command to run only EUS upgrade test, without any pre/post validation: + +```bash +-m eus_upgrade --upgrade eus --cnv-version --cnv-image --eus-ocp-images , ``` #### Custom upgrade lanes diff --git a/tests/conftest.py b/tests/conftest.py index d47694c796..f74bc345f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,7 +64,7 @@ from ocp_resources.virtual_machine_instancetype import VirtualMachineInstancetype from ocp_resources.virtual_machine_preference import VirtualMachinePreference from ocp_utilities.monitoring import Prometheus -from packaging.version import Version, parse +from packaging.version import parse from pytest_testconfig import config as py_config from timeout_sampler import TimeoutSampler @@ -1895,28 +1895,14 @@ def hco_target_csv_name(cnv_target_version): return get_hco_csv_name_by_version(cnv_target_version=cnv_target_version) if cnv_target_version else None -@pytest.fixture(scope="session") -def eus_hco_target_csv_name(eus_target_cnv_version): - if eus_target_cnv_version is None: - LOGGER.warning("Cannot determine EUS HCO target CSV name: EUS target version is None (non-EUS version)") - return None - return get_hco_csv_name_by_version(cnv_target_version=eus_target_cnv_version) - - @pytest.fixture(scope="session") def cnv_target_version(pytestconfig): return pytestconfig.option.cnv_version @pytest.fixture(scope="session") -def eus_target_cnv_version(pytestconfig, cnv_current_version): - cnv_current_version = Version(version=cnv_current_version) - minor = cnv_current_version.minor - # EUS-to-EUS upgrades are only viable between even-numbered minor versions, return None if non-eus version - if minor % 2: - LOGGER.warning(f"EUS upgrade can not be performed from non-eus version: {cnv_current_version}") - return None - return pytestconfig.option.eus_cnv_target_version or f"{cnv_current_version.major}.{minor + 2}.0" +def cnv_channel(pytestconfig): + return pytestconfig.option.cnv_channel @pytest.fixture() diff --git a/tests/install_upgrade_operators/product_upgrade/conftest.py b/tests/install_upgrade_operators/product_upgrade/conftest.py index 6e5fe5abd7..db89d6bb68 100644 --- a/tests/install_upgrade_operators/product_upgrade/conftest.py +++ b/tests/install_upgrade_operators/product_upgrade/conftest.py @@ -15,13 +15,12 @@ ) from tests.install_upgrade_operators.product_upgrade.utils import ( approve_cnv_upgrade_install_plan, + build_eus_upgrade_path_dict, extract_ocp_version_from_ocp_image, get_alerts_fired_during_upgrade, get_all_firing_cnv_alerts, - get_iib_images_of_cnv_versions, get_nodes_labels, get_nodes_taints, - get_shortest_upgrade_path, perform_cnv_upgrade, run_ocp_upgrade_command, set_workload_update_methods_hco, @@ -42,6 +41,7 @@ from utilities.constants import ( HCO_CATALOG_SOURCE, TIMEOUT_10MIN, + TIMEOUT_180MIN, NamespacesNames, ) from utilities.data_collector import ( @@ -50,7 +50,6 @@ from utilities.infra import ( exit_pytest_execution, generate_openshift_pull_secret_file, - get_csv_by_name, get_prometheus_k8s_token, get_related_images_name_and_version, get_subscription, @@ -324,24 +323,24 @@ def fired_alerts_during_upgrade( @pytest.fixture(scope="session") -def eus_cnv_upgrade_path(eus_target_cnv_version): - if eus_target_cnv_version is None: +def eus_cnv_upgrade_path( + cnv_target_version, + cnv_current_version, + cnv_channel, + cnv_image_url, +): + if Version(version=cnv_current_version).minor % 2: exit_pytest_execution( - message="EUS upgrade can not be performed from non-eus version", return_code=EUS_ERROR_CODE + message=f"EUS upgrade can not be performed from non-eus version: {cnv_current_version}", + return_code=EUS_ERROR_CODE, + filename="eus_upgrade_failure.txt", ) - # Get the shortest path to the target (EUS) version - upgrade_path_to_target_version = get_shortest_upgrade_path(target_version=eus_target_cnv_version) - # Get the shortest path to the intermediate (non-EUS) version - upgrade_path_to_intermediate_version = get_shortest_upgrade_path( - target_version=upgrade_path_to_target_version["startVersion"] + return build_eus_upgrade_path_dict( + current_cnv_version=cnv_current_version, + target_cnv_version=cnv_target_version, + target_channel=cnv_channel, + target_cnv_image_url=cnv_image_url, ) - # Return a dictionary with the versions and images for the EUS-to-EUS upgrade - upgrade_path = { - "non-eus": get_iib_images_of_cnv_versions(versions=upgrade_path_to_intermediate_version["versions"]), - EUS: get_iib_images_of_cnv_versions(versions=upgrade_path_to_target_version["versions"], errata_status="false"), - } - LOGGER.info(f"Upgrade path for EUS-to-EUS upgrade: {upgrade_path}") - return upgrade_path @pytest.fixture(scope="session") @@ -373,6 +372,7 @@ def eus_unpaused_worker_mcp( machine_config_pools_list=worker_machine_config_pools, initial_mcp_conditions=worker_machine_config_pools_conditions, nodes=workers, + timeout=TIMEOUT_180MIN, ) @@ -490,7 +490,7 @@ def triggered_non_eus_to_target_eus_ocp_upgrade(eus_ocp_image_urls): @pytest.fixture() def source_eus_to_non_eus_ocp_upgraded( admin_client, - masters, + control_plane_nodes, master_machine_config_pools, ocp_version_eus_to_non_eus_from_image_url, triggered_source_eus_to_non_eus_ocp_upgrade, @@ -500,14 +500,14 @@ def source_eus_to_non_eus_ocp_upgraded( machine_config_pools_list=master_machine_config_pools, target_ocp_version=ocp_version_eus_to_non_eus_from_image_url, initial_mcp_conditions=get_machine_config_pools_conditions(machine_config_pools=master_machine_config_pools), - nodes=masters, + nodes=control_plane_nodes, ) @pytest.fixture() def non_eus_to_target_eus_ocp_upgraded( admin_client, - masters, + control_plane_nodes, master_machine_config_pools, ocp_version_non_eus_to_eus_from_image_url, triggered_non_eus_to_target_eus_ocp_upgrade, @@ -517,7 +517,7 @@ def non_eus_to_target_eus_ocp_upgraded( machine_config_pools_list=master_machine_config_pools, target_ocp_version=ocp_version_non_eus_to_eus_from_image_url, initial_mcp_conditions=get_machine_config_pools_conditions(machine_config_pools=master_machine_config_pools), - nodes=masters, + nodes=control_plane_nodes, ) @@ -526,17 +526,25 @@ def source_eus_to_non_eus_cnv_upgraded( admin_client, hco_namespace, eus_cnv_upgrade_path, + cnv_subscription_scope_session, + cnv_registry_source, hyperconverged_resource_scope_function, - updated_cnv_subscription_source, ): - for version, cnv_image in sorted(eus_cnv_upgrade_path["non-eus"].items()): + for version, build_info in sorted( + eus_cnv_upgrade_path["non-eus"].items(), + key=lambda item: Version(version=item[0]), + ): + cnv_image = build_info["cnv_image_url"] LOGGER.info(f"Cnv upgrade to version {version} using image: {cnv_image}") perform_cnv_upgrade( admin_client=admin_client, cnv_image_url=cnv_image, cr_name=hyperconverged_resource_scope_function.name, hco_namespace=hco_namespace, - cnv_target_version=version.lstrip("v"), + cnv_target_version=version, + subscription=cnv_subscription_scope_session, + subscription_source=cnv_registry_source["cnv_subscription_source"], + subscription_channel=build_info["channel"], ) LOGGER.info("Successfully performed cnv upgrades from source EUS to non-EUS version.") @@ -546,27 +554,27 @@ def non_eus_to_target_eus_cnv_upgraded( admin_client, hco_namespace, eus_cnv_upgrade_path, + cnv_subscription_scope_session, + cnv_registry_source, hyperconverged_resource_scope_function, - updated_cnv_subscription_source, ): - version, cnv_image = next(iter(eus_cnv_upgrade_path[EUS].items())) - LOGGER.info(f"Cnv upgrade to version {version} using image: {cnv_image}") - perform_cnv_upgrade( - admin_client=admin_client, - cnv_image_url=cnv_image, - cr_name=hyperconverged_resource_scope_function.name, - hco_namespace=hco_namespace, - cnv_target_version=version.lstrip("v"), - ) - - -@pytest.fixture() -def eus_created_target_hco_csv(admin_client, hco_namespace, eus_hco_target_csv_name): - return get_csv_by_name( - csv_name=eus_hco_target_csv_name, - admin_client=admin_client, - namespace=hco_namespace.name, - ) + for version, build_info in sorted( + eus_cnv_upgrade_path[EUS].items(), + key=lambda item: Version(version=item[0]), + ): + cnv_image = build_info["cnv_image_url"] + LOGGER.info(f"Cnv upgrade to version {version} using image: {cnv_image}") + perform_cnv_upgrade( + admin_client=admin_client, + cnv_image_url=cnv_image, + cr_name=hyperconverged_resource_scope_function.name, + hco_namespace=hco_namespace, + cnv_target_version=version, + subscription=cnv_subscription_scope_session, + subscription_source=cnv_registry_source["cnv_subscription_source"], + subscription_channel=build_info["channel"], + ) + LOGGER.info("Successfully performed cnv upgrades from non-EUS to target EUS version.") @pytest.fixture() diff --git a/tests/install_upgrade_operators/product_upgrade/test_eus_upgrade.py b/tests/install_upgrade_operators/product_upgrade/test_eus_upgrade.py index b276538452..b063aa2ff0 100644 --- a/tests/install_upgrade_operators/product_upgrade/test_eus_upgrade.py +++ b/tests/install_upgrade_operators/product_upgrade/test_eus_upgrade.py @@ -8,6 +8,11 @@ LOGGER = logging.getLogger(__name__) +pytestmark = pytest.mark.usefixtures( + "nodes_taints_before_upgrade", + "nodes_labels_before_upgrade", +) + @pytest.mark.product_upgrade_test @pytest.mark.upgrade @@ -20,7 +25,6 @@ def test_eus_upgrade_process( self, admin_client, hco_namespace, - eus_target_cnv_version, eus_cnv_upgrade_path, eus_paused_worker_mcp, eus_paused_workload_update, @@ -29,7 +33,7 @@ def test_eus_upgrade_process( upgraded_odf, non_eus_to_target_eus_ocp_upgraded, non_eus_to_target_eus_cnv_upgraded, - eus_created_target_hco_csv, + created_target_hco_csv, eus_unpaused_workload_update, eus_unpaused_worker_mcp, ): @@ -37,6 +41,6 @@ def test_eus_upgrade_process( verify_upgrade_cnv( dyn_client=admin_client, hco_namespace=hco_namespace, - expected_images=get_related_images_name_and_version(csv=eus_created_target_hco_csv).values(), + expected_images=get_related_images_name_and_version(csv=created_target_hco_csv).values(), ) LOGGER.info("EUS post upgrade validation completed.") diff --git a/tests/install_upgrade_operators/product_upgrade/utils.py b/tests/install_upgrade_operators/product_upgrade/utils.py index 208ae27725..f5f64db755 100644 --- a/tests/install_upgrade_operators/product_upgrade/utils.py +++ b/tests/install_upgrade_operators/product_upgrade/utils.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from ocp_resources.subscription import Subscription from ocp_utilities.monitoring import Prometheus from deepdiff import DeepDiff @@ -27,9 +28,9 @@ from tests.install_upgrade_operators.constants import WORKLOAD_UPDATE_STRATEGY_KEY_NAME, WORKLOADUPDATEMETHODS from tests.install_upgrade_operators.utils import wait_for_install_plan +from tests.upgrade_params import EUS from utilities.constants import ( BASE_EXCEPTIONS_DICT, - BREW_REGISTERY_SOURCE, FIRING_STATE, HCO_CATALOG_SOURCE, IMAGE_CRON_STR, @@ -51,6 +52,7 @@ get_deployments, get_pod_by_name_prefix, get_pods, + stable_channel_released_to_prod, wait_for_consistent_resource_conditions, wait_for_version_explorer_response, ) @@ -58,6 +60,7 @@ approve_install_plan, get_hco_csv_name_by_version, update_image_in_catalog_source, + update_subscription_source, wait_for_mcp_update_completion, ) @@ -456,6 +459,7 @@ def verify_upgrade_ocp( machine_config_pools_list=machine_config_pools_list, initial_mcp_conditions=initial_mcp_conditions, nodes=nodes, + timeout=TIMEOUT_180MIN, ) wait_for_cluster_version_stable_conditions( @@ -550,58 +554,162 @@ def get_alerts_fired_during_upgrade( return alerts_by_name -def get_upgrade_path(target_version: str) -> dict[str, list[dict[str, str | list[str]]]]: +def get_upgrade_path(target_version: str, channel: str = "stable") -> dict[str, Any]: + """Query the version explorer for upgrade paths to a target CNV version on a given channel.""" return wait_for_version_explorer_response( - api_end_point="GetUpgradePath", query_string=f"targetVersion={target_version}" + api_end_point="GetUpgradePath", + query_string=f"targetVersion={target_version}&channel={channel}", ) -def get_shortest_upgrade_path(target_version: str) -> dict[str, str | list[str]]: - """ - Get the shortest upgrade path to a given CNV target version(latest z stream) +def find_path_with_start_version(paths: list[dict[str, Any]], start_version: str) -> dict[str, Any] | None: + """Find an upgrade path entry matching start_version.""" + for path in paths: + if str(path["startVersion"]).lstrip("v") == start_version: + return path + return None + + +def get_cnv_image_url_for_version(version: str, channel: str) -> str: + """Return the CNV image URL of the first successful build for a version on a given channel.""" + builds = get_build_info_by_version(version=version, channel=channel)["successful_builds"] + assert builds, f"No successful builds for {version} on {channel} channel" + return builds[0]["iib"] + + +def get_stable_released_builds(minor_version: str) -> list[dict[str, Any]]: + """Return stable released builds for a minor version, sorted newest-first.""" + builds = wait_for_version_explorer_response( + api_end_point="GetReleasedBuilds", + query_string=f"minor_version={minor_version}&stage=false", + )["builds"] + stable_builds = sorted( + [build for build in builds if stable_channel_released_to_prod(channels=build["channels"])], + key=lambda build: Version(version=build["csv_version"]), + reverse=True, + ) + assert stable_builds, f"No stable released builds for {minor_version}" + return stable_builds - Args: - target_version (str): The target version of the upgrade path. + +def get_stable_channel_iib(build: dict[str, Any]) -> str: + """Extract the IIB URL from a build's stable channel entry.""" + stable_entry = next( + (item for item in build["channels"] if item.get("channel") == "stable" and item.get("released_to_prod")), + None, + ) + if not stable_entry: + raise ValueError(f"No stable released channel entry in build: {build}") + return stable_entry["iib"] + + +def build_version_images( + versions: list[str], + target_version: str, + target_cnv_image_url: str, + channel: str = "stable", +) -> dict[str, dict[str, str]]: + """Build a {version: {"cnv_image_url": ..., "channel": ...}} dict from a versions list.""" + images: dict[str, dict[str, str]] = {} + for raw_version in versions: + version = raw_version.lstrip("v") + if version == target_version: + images[version] = {"cnv_image_url": target_cnv_image_url, "channel": channel} + else: + images[version] = { + "cnv_image_url": get_cnv_image_url_for_version(version=version, channel=channel), + "channel": channel, + } + return images + + +def find_intermediate_cnv_versions( + current_version: str, + target_version: str, + target_channel: str = "stable", +) -> tuple[dict[str, dict[str, str]], dict[str, Any]]: + """Find the intermediate CNV versions for an EUS-to-EUS upgrade. + + Walks stable z-streams for the intermediate minor (current+1) from latest to oldest, + picks the first valid startVersion in the target's upgrade path, then resolves the + full hop path from current to intermediate. Returns: - dict: The shortest upgrade path to the target version. + Tuple of (non_eus_images, target_step). """ - upgrade_paths = get_upgrade_path(target_version=target_version)["path"] - assert upgrade_paths, f"Couldn't find upgrade path for {target_version} version" - upgrade_path = max( - upgrade_paths, - key=lambda path: Version(version="0") - if "-hotfix" in path["startVersion"] - else Version(version=str(path["startVersion"])), - ) - return upgrade_path + target_paths = get_upgrade_path(target_version=target_version, channel=target_channel)["path"] + assert target_paths, f"No upgrade paths for {target_version} on {target_channel} channel" + + current = Version(version=current_version) + intermediate_minor = f"{current.major}.{current.minor + 1}" + stable_builds = get_stable_released_builds(minor_version=intermediate_minor) + + for build in stable_builds: + intermediate_version = build["csv_version"].lstrip("v") + target_step = find_path_with_start_version(paths=target_paths, start_version=intermediate_version) + if not target_step: + continue + + LOGGER.info(f"Using {intermediate_version} as intermediate version for EUS upgrade to {target_version}") + intermediate_iib = get_stable_channel_iib(build=build) + + intermediate_paths = get_upgrade_path(target_version=intermediate_version, channel="stable")["path"] + intermediate_step = find_path_with_start_version(paths=intermediate_paths, start_version=current_version) + + if intermediate_step: + non_eus_images = build_version_images( + versions=intermediate_step["versions"], + target_version=intermediate_version, + target_cnv_image_url=intermediate_iib, + ) + else: + non_eus_images = {intermediate_version: {"cnv_image_url": intermediate_iib, "channel": "stable"}} + return non_eus_images, target_step + + raise AssertionError(f"No valid intermediate in {intermediate_minor} for upgrade to {target_version}") -def get_iib_images_of_cnv_versions(versions: list[str], errata_status: str = "true") -> dict[str, str]: - version_images = {} - for version in versions: - iib = get_successful_fbc_build_iib( - build_info=get_build_info_by_version(version=version, errata_status=errata_status)["successful_builds"] - ) - version_images[version] = f"{BREW_REGISTERY_SOURCE}/rh-osbs/iib:{iib}" - return version_images +def build_eus_upgrade_path_dict( + current_cnv_version: str, + target_cnv_version: str, + target_channel: str = "stable", + target_cnv_image_url: str | None = None, +) -> dict[str, dict[str, dict[str, str]]]: + """Build the EUS-to-EUS upgrade path with IIB images for each phase. -def get_successful_fbc_build_iib(build_info: list[dict[str, str]]) -> str: - LOGGER.info(f"Build info found: {build_info}") - for build in build_info: - if build["pipeline"] == "RHTAP FBC": - return build["iib"] - raise AssertionError("Should have a fbc build") + Args: + current_cnv_version: Currently installed CNV version (e.g., "4.20.15"). + target_cnv_version: EUS target CNV version (e.g., "4.22.0"). + target_channel: Channel for the target GetUpgradePath query (default "stable"). + target_cnv_image_url: CNV image URL for the EUS target (from --cnv-image CLI arg). + + Returns: + Dict with "non-eus" and "eus" keys, each mapping version string + to a dict with "cnv_image_url" and "channel" keys. + """ + assert target_cnv_image_url, "target_cnv_image_url is required for EUS upgrade path" + non_eus_images, target_step = find_intermediate_cnv_versions( + current_version=current_cnv_version, + target_version=target_cnv_version, + target_channel=target_channel, + ) + eus_images = build_version_images( + versions=target_step["versions"], + target_version=target_cnv_version, + target_cnv_image_url=target_cnv_image_url, + channel=target_channel, + ) + upgrade_path = {"non-eus": non_eus_images, EUS: eus_images} + LOGGER.info(f"Upgrade path for EUS-to-EUS upgrade: {upgrade_path}") + return upgrade_path -def get_build_info_by_version(version: str, errata_status: str = "true") -> dict[str, Any]: - query_string = f"version={version}" - if errata_status: - query_string = f"{query_string}&errata_status={errata_status}" +def get_build_info_by_version(version: str, channel: str = "stable") -> dict[str, Any]: + """Get successful builds for a CNV version from the version explorer, filtered by channel.""" return wait_for_version_explorer_response( api_end_point="GetSuccessfulBuildsByVersion", - query_string=query_string, + query_string=f"version={version}&channel={channel}", ) @@ -628,9 +736,22 @@ def perform_cnv_upgrade( cr_name: str, hco_namespace: Namespace, cnv_target_version: str, + subscription: Subscription | None = None, + subscription_source: str | None = None, + subscription_channel: str | None = None, ) -> None: hco_target_csv_name = get_hco_csv_name_by_version(cnv_target_version=cnv_target_version) + if subscription or subscription_source or subscription_channel: + assert subscription and subscription_source and subscription_channel, ( + "subscription, subscription_source, and subscription_channel must all be provided together" + ) + update_subscription_source( + subscription=subscription, + subscription_source=subscription_source, + subscription_channel=subscription_channel, + ) + LOGGER.info("Updating image in CatalogSource") update_image_in_catalog_source( dyn_client=admin_client, diff --git a/tests/virt/upgrade/conftest.py b/tests/virt/upgrade/conftest.py index e2e1e94a7c..76634bc687 100644 --- a/tests/virt/upgrade/conftest.py +++ b/tests/virt/upgrade/conftest.py @@ -14,8 +14,7 @@ from tests.virt.constants import VM_LABEL from tests.virt.upgrade.utils import ( - get_all_migratable_vms, - get_virt_launcher_image_from_csv, + get_virt_launcher_images_from_csv, validate_vms_pod_updated, vm_from_template, wait_for_automatic_vm_migrations, @@ -25,14 +24,10 @@ OS_FLAVOR_RHEL, TIMEOUT_30MIN, TIMEOUT_40MIN, - TIMEOUT_90MIN, Images, ) from utilities.hco import ResourceEditorValidateHCOReconcile -from utilities.infra import ( - check_pod_disruption_budget_for_completed_migrations, - get_csv_by_name, -) +from utilities.infra import get_csv_by_name from utilities.storage import ( create_dv, data_volume_template_with_source_ref_dict, @@ -155,46 +150,36 @@ def vms_for_upgrade_dict_before(vms_for_upgrade): @pytest.fixture(scope="session") -def upgrade_namespaces(upgrade_namespace_scope_session, kmp_enabled_namespace): - return [kmp_enabled_namespace, upgrade_namespace_scope_session] - +def virt_migratable_vms(admin_client, upgrade_namespace_scope_session): + migratable_vms = [] + for vm in VirtualMachine.get(client=admin_client, namespace=upgrade_namespace_scope_session.name): + if vm.ready and any( + condition.type == "LiveMigratable" and condition.status == "True" + for condition in vm.vmi.instance.status.conditions + ): + migratable_vms.append(vm) -@pytest.fixture(scope="session") -def migratable_vms(admin_client, hco_namespace, upgrade_namespaces): - migratable_vms = get_all_migratable_vms(admin_client=admin_client, namespaces=upgrade_namespaces) - LOGGER.info(f"All migratable vms: {[vm.name for vm in migratable_vms]}") return migratable_vms @pytest.fixture() def unupdated_vmi_pods_names( admin_client, - hco_namespace, - hco_target_csv_name, - eus_hco_target_csv_name, - upgrade_namespaces, - migratable_vms, - virt_launcher_from_csv_before_upgrade, + virt_migratable_vms, + virt_launcher_images_from_csv_before_upgrade, csv_after_upgrade, ): - virt_launcher_image_after_upgrade = get_virt_launcher_image_from_csv(csv=csv_after_upgrade) - - if virt_launcher_from_csv_before_upgrade == virt_launcher_image_after_upgrade: - LOGGER.warning(f"virt-launcher unchanged, skipping migration check: {virt_launcher_from_csv_before_upgrade}") - return [] - - wait_for_automatic_vm_migrations(vm_list=migratable_vms) - - for ns in upgrade_namespaces: - LOGGER.info(f"Checking PodDisruptionBudget in namespaces: {ns.name}") - check_pod_disruption_budget_for_completed_migrations( - admin_client=admin_client, namespace=ns.name, timeout=TIMEOUT_90MIN + virt_launcher_images_after_upgrade = get_virt_launcher_images_from_csv(csv=csv_after_upgrade) + if virt_launcher_images_from_csv_before_upgrade == virt_launcher_images_after_upgrade: + LOGGER.warning( + f"virt-launcher unchanged, skipping migration check: {virt_launcher_images_from_csv_before_upgrade}" ) + return [] + wait_for_automatic_vm_migrations(vm_list=virt_migratable_vms) return validate_vms_pod_updated( - admin_client=admin_client, - expected_virt_launcher_image=virt_launcher_image_after_upgrade, - vm_list=migratable_vms, + expected_virt_launcher_images=virt_launcher_images_after_upgrade, + vm_list=virt_migratable_vms, ) @@ -371,14 +356,14 @@ def parallel_live_migrations_increased(hyperconverged_resource_scope_session): @pytest.fixture(scope="session") -def virt_launcher_from_csv_before_upgrade(csv_scope_session): - return get_virt_launcher_image_from_csv(csv=csv_scope_session) +def virt_launcher_images_from_csv_before_upgrade(csv_scope_session): + return get_virt_launcher_images_from_csv(csv=csv_scope_session) @pytest.fixture() -def csv_after_upgrade(admin_client, hco_namespace, hco_target_csv_name, eus_hco_target_csv_name): +def csv_after_upgrade(admin_client, hco_namespace, hco_target_csv_name): return get_csv_by_name( admin_client=admin_client, namespace=hco_namespace.name, - csv_name=hco_target_csv_name or eus_hco_target_csv_name, + csv_name=hco_target_csv_name, ) diff --git a/tests/virt/upgrade/test_upgrade_virt.py b/tests/virt/upgrade/test_upgrade_virt.py index 6a604befc8..8ad2acc1d4 100644 --- a/tests/virt/upgrade/test_upgrade_virt.py +++ b/tests/virt/upgrade/test_upgrade_virt.py @@ -54,7 +54,7 @@ @pytest.mark.usefixtures( "base_templates", "parallel_live_migrations_increased", - "virt_launcher_from_csv_before_upgrade", + "virt_launcher_images_from_csv_before_upgrade", ) class TestUpgradeVirt: """Pre-upgrade tests""" diff --git a/tests/virt/upgrade/utils.py b/tests/virt/upgrade/utils.py index 9cbe5d9a73..76e6eb4aae 100644 --- a/tests/virt/upgrade/utils.py +++ b/tests/virt/upgrade/utils.py @@ -1,11 +1,16 @@ import logging from contextlib import contextmanager from datetime import datetime +from typing import TYPE_CHECKING import pytest from ocp_resources.datavolume import DataVolume from ocp_resources.template import Template from ocp_resources.virtual_machine import VirtualMachine + +if TYPE_CHECKING: + from ocp_resources.cluster_service_version import ClusterServiceVersion + from ocp_resources.virtual_machine_instance_migration import ( VirtualMachineInstanceMigration, ) @@ -19,9 +24,6 @@ VIRT_LAUNCHER, ) from utilities.exceptions import ResourceMissingFieldError -from utilities.infra import ( - get_pod_disruption_budget, -) from utilities.virt import ( VirtualMachineForTestsFromTemplate, get_vm_boot_time, @@ -33,10 +35,9 @@ TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -def get_virt_launcher_image_from_csv(csv): - for item in csv.instance.spec.relatedImages: - if VIRT_LAUNCHER in item["name"]: - return item["image"] +def get_virt_launcher_images_from_csv(csv: "ClusterServiceVersion") -> set[str]: + if images := {item["image"] for item in csv.instance.spec.relatedImages if VIRT_LAUNCHER in item["name"]}: + return images raise ValueError(f"Image digest for {VIRT_LAUNCHER} not found") @@ -80,30 +81,6 @@ def get_src_pvc_default_name(template): raise ResourceMissingFieldError(f"Template {template.name} does not have a parameter {DATA_SOURCE_NAME}") -def get_all_migratable_vms(admin_client, namespaces): - # Check pod disruption budget associated with given namespaces. Collect associated vm names. These vms are - # the only migratable ones - pod_disruption_budget_list = [ - pod_disruption_budget - for ns in namespaces - for pod_disruption_budget in get_pod_disruption_budget(admin_client=admin_client, namespace_name=ns.name) - ] - pod_disruption_budget_info = { - pod_disruption_budget.name: pod_disruption_budget.instance.metadata.ownerReferences[0]["name"] - for pod_disruption_budget in pod_disruption_budget_list - } - LOGGER.info(f"PodDisruptionBudgets: {pod_disruption_budget_info}") - - return [ - VirtualMachine( - client=admin_client, - namespace=pod_disruption_budget.namespace, - name=pod_disruption_budget.instance.metadata.ownerReferences[0]["name"], - ) - for pod_disruption_budget in pod_disruption_budget_list - ] - - def get_workload_update_migrations_list(namespaces): workload_migrations = {} for namespace in namespaces: @@ -164,11 +141,13 @@ def wait_for_automatic_vm_migrations(vm_list): raise -def validate_vms_pod_updated(admin_client, expected_virt_launcher_image, vm_list): +def validate_vms_pod_updated( + expected_virt_launcher_images: set[str], vm_list: list[VirtualMachine] +) -> list[dict[str, str]]: return [ {pod.name: pod.instance.spec.containers[0].image} for pod in [vm.vmi.virt_launcher_pod for vm in vm_list] - if pod.instance.spec.containers[0].image != expected_virt_launcher_image + if pod.instance.spec.containers[0].image not in expected_virt_launcher_images ] diff --git a/tox.ini b/tox.ini index 7f7874b3cb..da72cc7cc7 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ commands = uv run pytest --collect-only uv run pytest --upgrade=cnv --cnv-image=NA --cnv-version=NA --collect-only uv run pytest --upgrade=ocp --ocp-image=NA --collect-only - uv run pytest --upgrade=eus --eus-ocp-images=NA,NA --collect-only + uv run pytest --upgrade=eus --cnv-image=NA --cnv-version=NA --eus-ocp-images=NA,NA --collect-only uv run pytest --upgrade_custom=cnv --cnv-image=NA --cnv-version=NA --collect-only uv run pytest --upgrade_custom=ocp --ocp-image=NA --collect-only uv run pytest --tc-file=tests/global_config_aws.py --collect-only diff --git a/utilities/operator.py b/utilities/operator.py index eec02fc67d..3121819d3b 100644 --- a/utilities/operator.py +++ b/utilities/operator.py @@ -150,8 +150,8 @@ def consecutive_checks_for_mcp_condition(mcp_sampler, machine_config_pools_list) raise -def wait_for_mcp_update_end(machine_config_pools_list): - wait_for_mcp_updated_condition_true(machine_config_pools_list=machine_config_pools_list) +def wait_for_mcp_update_end(machine_config_pools_list, timeout=TIMEOUT_75MIN): + wait_for_mcp_updated_condition_true(machine_config_pools_list=machine_config_pools_list, timeout=timeout) wait_for_mcp_ready_machine_count(machine_config_pools_list=machine_config_pools_list) @@ -399,7 +399,7 @@ def wait_for_csv_successful_state(admin_client, namespace_name, subscription_nam raise ResourceNotFoundError(f"Subscription {subscription_name} not found in namespace: {namespace_name}") -def wait_for_mcp_update_completion(machine_config_pools_list, initial_mcp_conditions, nodes): +def wait_for_mcp_update_completion(machine_config_pools_list, initial_mcp_conditions, nodes, timeout=TIMEOUT_75MIN): initial_updating_transition_times = get_mcp_updating_transition_times(mcp_conditions=initial_mcp_conditions) wait_for_mcp_update_start( @@ -408,6 +408,7 @@ def wait_for_mcp_update_completion(machine_config_pools_list, initial_mcp_condit ) wait_for_mcp_update_end( machine_config_pools_list=machine_config_pools_list, + timeout=timeout, ) wait_for_nodes_to_have_same_kubelet_version(nodes=nodes) wait_for_all_nodes_ready(nodes=nodes)