diff --git a/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/actor.py b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/actor.py new file mode 100644 index 0000000000..59e9f27868 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/actor.py @@ -0,0 +1,39 @@ +from leapp.actors import Actor +from leapp.libraries.actor import checktargetkernelminor +from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.stdlib import api +from leapp.models import TargetUserSpaceInfo +from leapp.reporting import Report +from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag + + +class CheckTargetKernelMinor(Actor): + """ + Inhibit the upgrade when target CloudLinux repositories would deliver + a kernel from a newer minor than the rest of the CloudLinux userland. + + Detects the gradual-rollout-leak behind CLOS-3716 (ZD 268790): the CLN + package channel serves the latest-minor kernel ahead of the matching + userland during a staged minor rollout. Installing such a kernel on + the older-minor userland breaks per-kernel-minor modules like kmodlve, + so we refuse the upgrade until the rollout is complete or repos are + pinned to one minor. + + See the `checktargetkernelminor` library docstring for the detection + details. + """ + + name = 'check_target_kernel_minor' + consumes = (TargetUserSpaceInfo,) + produces = (Report,) + tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag) + + @run_on_cloudlinux + def process(self): + info = next(api.consume(TargetUserSpaceInfo), None) + if info is None: + self.log.info( + 'No TargetUserSpaceInfo available; skipping kernel-minor check.' + ) + return + checktargetkernelminor.process(installroot=info.path) diff --git a/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/libraries/checktargetkernelminor.py b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/libraries/checktargetkernelminor.py new file mode 100644 index 0000000000..993e6e73f9 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/libraries/checktargetkernelminor.py @@ -0,0 +1,176 @@ +""" +Detect a target-repository state where the available kernel's minor version +is greater than the available `cloudlinux-release`'s minor version. + +This is the gradual-rollout-leak shape behind CLOS-3716 (ZD 268790): the CLN +package channel `cloudlinux-x86_64-server-9` resolves to the latest minor +available to a system's gradual-rollout slot, and a new minor's kernel is +typically promoted ahead of the rest of the userland. During an in-flight +rollout, elevate can therefore pull a 9.7 kernel onto a 9.6 userland; the +matching kmod-lve is built per kernel minor, so the new default-boot kernel +fails to load kmodlve and breaks LVE. + +The actor consuming this library inhibits at Checks phase whenever the +target repositories expose this mismatch, so the upgrade refuses to proceed +while rollout content is in flight. This is defense in depth - it catches +the leak regardless of which CL repo (CLN channel, no-auth fallback, etc.) +ends up serving the newer-minor kernel. +""" + +import re + +from leapp import reporting +from leapp.libraries.common.config.version import get_target_major_version +from leapp.libraries.stdlib import CalledProcessError, api, run + + +# CloudLinux/RHEL dist-tag with minor version: e.g. +# 611.5.1.el9_7 -> (9, 7) +# 570.12.1.el9_6.x86_64 -> (9, 6) +# 1.0-1.el8_10 -> (8, 10) +# 1.0-1.el8_6.cloudlinux -> (8, 6) +_DIST_MINOR_RE = re.compile(r'\.el(\d+)_(\d+)(?:\.|$)') + +# cloudlinux-release versions are "." - e.g. "9.6", "8.10". +_RELEASE_MINOR_RE = re.compile(r'^(\d+)\.(\d+)(?:\.|$)') + + +def parse_kernel_minor(release_str, target_major): + """Return the minor version embedded in a kernel RPM release string, + or None if not present or not for the target major. + + The target-major filter is critical: the target userspace's repoquery + returns packages from both source (e.g. CL8) and target (e.g. CL9) repos, + and dist-tagged minors must not be conflated across majors + (e.g. `el8_10` minor 10 is meaningless when comparing against an + `el9_*` kernel). + """ + if not release_str: + return None + m = _DIST_MINOR_RE.search(release_str) + if not m: + return None + if int(m.group(1)) != int(target_major): + return None + return int(m.group(2)) + + +def parse_release_minor(version_str, target_major): + """Return the minor version from a cloudlinux-release RPM version field, + or None if not present or not for the target major. + + See `parse_kernel_minor` for why the major-filter matters. + """ + if not version_str: + return None + m = _RELEASE_MINOR_RE.match(version_str) + if not m: + return None + if int(m.group(1)) != int(target_major): + return None + return int(m.group(2)) + + +def _repoquery(installroot, pkg): + """Query the target userspace container for available versions of `pkg`. + + Returns a list of (version, release) tuples. Empty list on error or + nothing available - callers treat absence as "cannot determine" rather + than as evidence of safety. + """ + cmd = [ + 'dnf', '-q', 'repoquery', + '--installroot={}'.format(installroot), + '--available', + '--queryformat=%{version}|%{release}\n', + pkg, + ] + try: + result = run(cmd, split=False) + except (OSError, CalledProcessError) as exc: + api.current_logger().warning( + 'repoquery for %s in %s failed: %s', pkg, installroot, exc + ) + return [] + rows = [] + for line in (result.get('stdout') or '').splitlines(): + line = line.strip() + if not line or '|' not in line: + continue + version, release = line.split('|', 1) + rows.append((version, release)) + return rows + + +def _max_kernel_minor(rows, target_major): + """Highest minor across kernel rows for the target major; None if no row matches.""" + minors = [parse_kernel_minor(release, target_major) for _, release in rows] + minors = [m for m in minors if m is not None] + return max(minors) if minors else None + + +def _max_release_minor(rows, target_major): + """Highest minor across cloudlinux-release rows for the target major; None if no row matches.""" + minors = [parse_release_minor(version, target_major) for version, _ in rows] + minors = [m for m in minors if m is not None] + return max(minors) if minors else None + + +def _report_inhibitor(target_major, kernel_minor, release_minor): + reporting.create_report([ + reporting.Title( + 'CloudLinux target repositories are mid gradual-rollout: kernel newer than userland' + ), + reporting.Summary( + 'The target CloudLinux repositories currently offer a kernel from minor el{major}_{kminor}' + ' while the newest available cloudlinux-release is for minor {major}.{rminor}.' + ' CloudLinux gradual-rollouts typically promote the kernel ahead of the rest of the' + ' userland, and running the upgrade now would install a kernel newer than the matching' + ' kmod-lve and other per-kernel-minor packages. On reboot the new default-boot kernel' + ' would fail to load kmodlve, breaking LVE.'.format( + major=target_major, kminor=kernel_minor, rminor=release_minor, + ) + ), + reporting.Remediation( + hint='Re-run the upgrade once the new CloudLinux minor has finished its gradual rollout' + ' (target repositories should then offer a cloudlinux-release matching the kernel' + ' minor). Alternatively, pin the target repositories to a single, fully-released minor.' + ), + reporting.Severity(reporting.Severity.HIGH), + reporting.Groups([ + reporting.Groups.INHIBITOR, + reporting.Groups.KERNEL, + reporting.Groups.REPOSITORY, + reporting.Groups.UPGRADE_PROCESS, + ]), + ]) + + +def process(installroot, query_fn=None, target_major=None): + """Inhibit when newest available kernel's minor exceeds newest available + cloudlinux-release's minor in the target userspace's repositories. + """ + if query_fn is None: + query_fn = _repoquery + if target_major is None: + target_major = get_target_major_version() + + kernel_rows = query_fn(installroot, 'kernel-core') + release_rows = query_fn(installroot, 'cloudlinux-release') + + k_minor = _max_kernel_minor(kernel_rows, target_major) + r_minor = _max_release_minor(release_rows, target_major) + + if k_minor is None or r_minor is None: + api.current_logger().info( + 'Skipping kernel-minor check: could not determine both minors' + ' (kernel=%s, release=%s)', k_minor, r_minor, + ) + return + + api.current_logger().info( + 'Target userspace: newest kernel-core minor=el%s_%s, newest cloudlinux-release minor=%s.%s', + target_major, k_minor, target_major, r_minor, + ) + if k_minor > r_minor: + _report_inhibitor(target_major, k_minor, r_minor) diff --git a/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/tests/test_checktargetkernelminor.py b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/tests/test_checktargetkernelminor.py new file mode 100644 index 0000000000..97ebed1cc2 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/checktargetkernelminor/tests/test_checktargetkernelminor.py @@ -0,0 +1,189 @@ +"""Unit tests for the checktargetkernelminor library. + +The actor itself is a thin wrapper around ``library.process(installroot)``; +the interesting logic - parsing the minor out of RPM version/release strings +and comparing the highest available kernel minor against the highest +available cloudlinux-release minor - lives in the library, so that is what +gets exercised here. +""" + +import pytest + +from leapp.libraries.actor import checktargetkernelminor as lib + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + + +class TestParseKernelMinor: + """Extract the minor from a kernel RPM release field (`...elN_M...`), + requiring the major to match the target. Cross-major rows must not be + conflated - the target userspace's repoquery returns both source and + target packages, and only the target major's minors are meaningful. + """ + + @pytest.mark.parametrize( + ('release', 'target_major', 'expected'), + [ + # The customer's exact case (CLOS-3716), target=9: + ('611.5.1.el9_7', '9', 7), + ('570.12.1.el9_6', '9', 6), + ('570.62.1.el9_6', '9', 6), + # Vendor suffix appended after the dist tag must not break parsing: + ('1.0-1.el8_6.cloudlinux', '8', 6), + # el8_10 must parse correctly for el7->el8 path: + ('1.0-1.el8_10', '8', 10), + ('1.0-1.el8_8', '8', 8), + # Major mismatch -> ignored (this is the validation fix): + ('1.0-1.el8_10', '9', None), + ('611.5.1.el9_7', '8', None), + # No minor in the dist tag - cannot determine: + ('1.0-1.el9', '9', None), + ('1.0-1', '9', None), + ('', '9', None), + (None, '9', None), + ], + ) + def test_parse(self, release, target_major, expected): + assert lib.parse_kernel_minor(release, target_major) == expected + + +class TestParseReleaseMinor: + """Extract the minor from a cloudlinux-release RPM version field, + requiring the major to match the target. + """ + + @pytest.mark.parametrize( + ('version', 'target_major', 'expected'), + [ + ('9.6', '9', 6), + ('9.7', '9', 7), + ('8.10', '8', 10), + ('9.6.1', '9', 6), + # Major mismatch -> ignored (this is the validation fix): + ('8.10', '9', None), + ('9.6', '8', None), + # Major-only or junk - cannot determine: + ('9', '9', None), + ('', '9', None), + (None, '9', None), + ('garbage', '9', None), + ], + ) + def test_parse(self, version, target_major, expected): + assert lib.parse_release_minor(version, target_major) == expected + + +# --------------------------------------------------------------------------- +# Process: kernel-minor vs release-minor comparison +# --------------------------------------------------------------------------- + + +def _make_query(kernel_rows, release_rows): + """Inject canned repoquery results for kernel-core and cloudlinux-release.""" + + def query(installroot, pkg): + if pkg == 'kernel-core': + return kernel_rows + if pkg == 'cloudlinux-release': + return release_rows + return [] + + return query + + +@pytest.fixture +def captured_reports(monkeypatch): + """Collect every reporting.create_report() call made by the library.""" + sink = [] + monkeypatch.setattr(lib.reporting, 'create_report', lambda parts: sink.append(parts)) + return sink + + +def _report_groups(report_parts): + """Return the list of group strings from a captured create_report() call.""" + for part in report_parts: + # leapp's reporting.Groups carries `fields={'groups': [...]}` via its + # ``__init__``. Walk attributes that look like the groups list. + groups = getattr(part, 'value', None) + if isinstance(groups, list) and groups and all(isinstance(g, str) for g in groups): + return groups + return [] + + +class TestProcess: + def test_match_no_inhibit(self, captured_reports): + """Newest kernel minor equals newest release minor -> upgrade proceeds.""" + q = _make_query( + kernel_rows=[('5.14.0', '570.12.1.el9_6'), ('5.14.0', '570.62.1.el9_6')], + release_rows=[('9.6', '7.el9')], + ) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert captured_reports == [] + + def test_kernel_minor_ahead_inhibits(self, captured_reports): + """CLOS-3716: kernel el9_7 available while cloudlinux-release still at 9.6.""" + q = _make_query( + kernel_rows=[('5.14.0', '570.12.1.el9_6'), ('5.14.0', '611.5.1.el9_7')], + release_rows=[('9.6', '7.el9')], + ) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert len(captured_reports) == 1 + groups = _report_groups(captured_reports[0]) + assert 'inhibitor' in groups + assert 'kernel' in groups + + def test_release_ahead_no_inhibit(self, captured_reports): + """Release minor ahead of kernel minor - not the rollout-leak shape.""" + q = _make_query( + kernel_rows=[('5.14.0', '570.12.1.el9_6')], + release_rows=[('9.6', '7.el9'), ('9.7', '1.el9')], + ) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert captured_reports == [] + + def test_missing_kernel_data_no_inhibit(self, captured_reports): + """Empty repoquery for kernel-core -> no determination, no report.""" + q = _make_query(kernel_rows=[], release_rows=[('9.6', '7.el9')]) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert captured_reports == [] + + def test_missing_release_data_no_inhibit(self, captured_reports): + """Empty repoquery for cloudlinux-release -> no determination, no report.""" + q = _make_query(kernel_rows=[('5.14.0', '570.12.1.el9_6')], release_rows=[]) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert captured_reports == [] + + def test_el8_10_path(self, captured_reports): + """el8_10 must parse as minor 10 (>= 9) so el7->el8 path works too.""" + q = _make_query( + kernel_rows=[('4.18.0', '513.5.1.el8_9'), ('4.18.0', '600.1.1.el8_10')], + release_rows=[('8.9', '1.el8')], + ) + lib.process(installroot='/var/lib/leapp/el8userspace', query_fn=q, target_major='8') + assert len(captured_reports) == 1 + + def test_cross_major_packages_ignored(self, captured_reports): + """The target userspace's repoquery returns source-major packages too; + rows for the wrong major must be ignored (validation found this on a + live VM where `cloudlinux-release-8.10` made the unfiltered code + report a meaningless `9.10` minor). + """ + q = _make_query( + kernel_rows=[ + # Target el9 entries: + ('5.14.0', '570.12.1.el9_6'), + # Source el8 entries that must NOT be counted (no minor tag here, but + # an el8_N row must also not bleed into the el9 max): + ('4.18.0', '348.lve.el8'), + ('4.18.0', '600.1.1.el8_10'), + ], + release_rows=[ + ('8.10', '1.el8'), ('8.10', '7.el8'), # source-major - ignore + ('9.6', '7.el9'), # only this counts for target=9 + ], + ) + lib.process(installroot='/var/lib/leapp/el9userspace', query_fn=q, target_major='9') + assert captured_reports == [] # kernel minor 6 == release minor 6