build-sync: Add RHCOS tagging to QCI

joepvd · joepvd · commit 558912f4e8df · 2026-06-05T10:33:03.000+02:00
Tag RHCOS container images (rhel-coreos, machine-os-content, rhel-coreos-extensions,
rhel-coreos-10, rhel-coreos-10-extensions) to quay.io/openshift/ci during build-sync.

Uses `oc tag` with reference mode to create image references without copying bytes:
- --reference-policy='source' - Reference points to source location
- --import-mode='PreserveOriginal' - Preserve original manifest format
- --reference - Create reference without importing image

## QCI Pullspec Format

Images are tagged to QCI with:
- **Prunable tags** (for cleanup): `quay.io/openshift/ci:&lt;timestamp&gt;_prune_art__ocp_&lt;version&gt;_&lt;tag&gt;`
- **Floating tags** (latest): `quay.io/openshift/ci:art__ocp_&lt;version&gt;_&lt;tag&gt;`

Examples for OCP 4.17:
```
quay.io/openshift/ci:20260603120000_prune_art__ocp_4.17_rhel-coreos
quay.io/openshift/ci:art__ocp_4.17_rhel-coreos
quay.io/openshift/ci:art__ocp_4.17_rhel-coreos-extensions
quay.io/openshift/ci:art__ocp_4.17_machine-os-content
```

For private images (openshift-priv):
```
quay.io/openshift/ci:art__ocp_4.17_priv_rhel-coreos
```

Implements support for:
- QCI credentials via QCI_USER/QCI_PASSWORD environment variables
- Both public and private namespace mirroring (using same source pullspec)
- Proper error handling with Slack notifications
- Skips for older versions (&lt;4.12), non-stream assemblies, and custom data gitrefs
diff --git a/.worktrees/autoclose-golang-bugs b/.worktrees/autoclose-golang-bugs
@@ -0,0 +1 @@
+Subproject commit b133254bd2a983a81e80cf23311eaab4d46d68b2
diff --git a/.worktrees/provides b/.worktrees/provides
@@ -0,0 +1 @@
+Subproject commit 6ec1c9b10d0b1bf04291730936fc0796cf4f49be
diff --git a/pyartcd/pyartcd/jira_client.py b/pyartcd/pyartcd/jira_client.py
@@ -47,6 +47,9 @@ def assign_to_me(self, key):
     def close_task(self, key):
         self._client.transition_issue(key, 'Closed')
 
+    def close_with_resolution(self, key, resolution='Not a Bug'):
+        self._client.transition_issue(key, 'Closed', fields={'resolution': {'name': resolution}})
+
     def start_task(self, key):
         self._client.transition_issue(key, 'In Progress')
 
diff --git a/pyartcd/pyartcd/pipelines/build_sync.py b/pyartcd/pyartcd/pipelines/build_sync.py
@@ -3,6 +3,7 @@
 import json
 import os
 import re
+from datetime import datetime
 
 import click
 import yaml
@@ -11,12 +12,14 @@
 from artcommonlib.constants import (
     KONFLUX_DEFAULT_IMAGE_REPO,
     REGISTRY_CI_OPENSHIFT,
+    REGISTRY_QUAY_CI,
     REGISTRY_QUAY_OCP_RELEASE_DEV,
+    REGISTRY_QUAY_OPENSHIFT,
 )
 from artcommonlib.exectools import limit_concurrency
 from artcommonlib.github_auth import get_github_client_for_org
 from artcommonlib.redis import RedisError
-from artcommonlib.registry_config import RegistryConfig
+from artcommonlib.registry_config import RegistryConfig, RegistryCredential
 from artcommonlib.release_util import SoftwareLifecyclePhase
 from artcommonlib.telemetry import start_as_current_span_async
 from artcommonlib.util import split_git_url, uses_konflux_imagestream_override
@@ -163,14 +166,25 @@ async def run(self):
 
         source_files = [quay_auth_file]
 
+        # Build credentials list for QCI (quay.io/openshift/ci) if available
+        credentials = []
+        qci_user = os.environ.get('QCI_USER')
+        qci_password = os.environ.get('QCI_PASSWORD')
+        if qci_user and qci_password:
+            credentials.append(RegistryCredential(REGISTRY_QUAY_OPENSHIFT, qci_user, qci_password))
+        else:
+            self.logger.warning('QCI_USER/QCI_PASSWORD not set - RHCOS images will not be mirrored to QCI')
+
         with RegistryConfig(
             kubeconfig=os.environ.get('KUBECONFIG'),
             source_files=source_files,
             registries=[
                 REGISTRY_QUAY_OCP_RELEASE_DEV,
                 KONFLUX_DEFAULT_IMAGE_REPO,
                 REGISTRY_CI_OPENSHIFT,
+                REGISTRY_QUAY_CI,
             ],
+            credentials=credentials,
         ) as global_auth_file:
             self._registry_config = global_auth_file
 
@@ -434,6 +448,126 @@ async def _populate_ci_imagestreams(self):
         except (ChildProcessError, KeyError) as e:
             await self.slack_client.say(f'Unable to mirror CoreOS images to CI for {self.version}: {e}')
 
+    @start_as_current_span_async(TRACER, "build-sync.mirror-rhcos-to-qci")
+    async def _mirror_rhcos_to_qci(self):
+        """
+        Mirror RHCOS images to quay.io/openshift/ci (QCI) for external CI consumption.
+
+        Uses the same tag convention as images:streams mirror:
+        - Prunable: {timestamp}_prune_art__ocp_{version}_{tag}
+        - Floating: art__ocp_{version}_{tag}
+
+        This ensures CI jobs can pull RHCOS images directly from QCI without
+        requiring access to the app.ci cluster's internal imagestreams.
+        """
+        current_span = trace.get_current_span()
+        current_span.set_attribute("build-sync.version", self.version)
+        current_span.set_attribute("build-sync.assembly", self.assembly)
+
+        # Same guards as _populate_ci_imagestreams
+        major, minor = [int(n) for n in self.version.split('.')]
+        if major <= 4 and minor < 12:
+            self.logger.info('Skipping QCI mirror for version %s (< 4.12)', self.version)
+            return
+
+        if not self.assembly == 'stream' or self.doozer_data_gitref:
+            self.logger.info('Skipping QCI mirror for non-stream assembly or custom gitref')
+            return
+
+        # Check if QCI credentials are available
+        if not os.environ.get('QCI_USER') or not os.environ.get('QCI_PASSWORD'):
+            self.logger.warning('QCI_USER/QCI_PASSWORD not set - skipping QCI mirror')
+            return
+
+        try:
+            tags_to_transfer = rhcos.get_container_names(self.group_runtime)
+            prune_timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+
+            self.logger.info('Mirroring RHCOS images to QCI: %s', ', '.join(tags_to_transfer))
+
+            tasks = []
+            for tag in tags_to_transfer:
+                # Mirror to public QCI
+                tasks.append(self._mirror_single_rhcos_to_qci(tag, prune_timestamp, private=False))
+                # Mirror to private QCI (for openshift-priv CI)
+                tasks.append(self._mirror_single_rhcos_to_qci(tag, prune_timestamp, private=True))
+
+            await asyncio.gather(*tasks)
+            self.logger.info('Successfully mirrored RHCOS images to QCI')
+
+        except Exception as e:
+            self.logger.error('Failed to mirror RHCOS images to QCI: %s', e)
+            await self.slack_client.say(f'Unable to mirror CoreOS images to QCI for {self.version}: {e}')
+
+    @start_as_current_span_async(TRACER, "build-sync.mirror-single-rhcos-to-qci")
+    @limit_concurrency(10)
+    async def _mirror_single_rhcos_to_qci(self, tag: str, prune_timestamp: str, private: bool):
+        """
+        Tag a single RHCOS image to QCI using reference mode.
+
+        Creates both a prunable tag (for cleanup) and a floating tag (always points to latest).
+        Uses oc tag with --reference to create references without copying image bytes.
+
+        For private mirroring, we use the same source pullspec from the public ART imagestream
+        (consistent with how _tag_into_ci_imagestream handles private CI).
+
+        Args:
+            tag: RHCOS container name (e.g., 'rhel-coreos', 'rhel-coreos-extensions')
+            prune_timestamp: Timestamp string for prunable tag naming
+            private: Whether to add _priv suffix for private CI consumption
+        """
+        current_span = trace.get_current_span()
+        current_span.set_attribute("build-sync.tag", tag)
+        current_span.set_attribute("build-sync.private", private)
+
+        # Always get pullspec from public ART imagestream (same source for both public and private,
+        # consistent with _tag_into_ci_imagestream which uses the same pullspec for private CI)
+        namespace = 'ocp'
+        is_name = self.is_base_name
+
+        # Get the pullspec from the ART imagestream (already multi-arch)
+        cmd = (
+            f'oc --kubeconfig {os.environ["KUBECONFIG"]} -n {namespace} '
+            f'get istag/{is_name}:{tag} -o=jsonpath={{{{.tag.from.name}}}}'
+        )
+        rc, tag_pullspec, stderr = await exectools.cmd_gather_async(cmd, check=False)
+        tag_pullspec = tag_pullspec.strip()
+
+        if rc != 0 or not tag_pullspec:
+            self.logger.warning(
+                'No pullspec found for %s/%s:%s (rc=%d, stderr=%s)', namespace, is_name, tag, rc, stderr
+            )
+            return
+
+        # Build QCI tag names following the images:streams mirror convention
+        # Replace problematic characters with underscores for valid tag names
+        priv_suffix = '_priv' if private else ''
+        qci_tag_base = f"ocp_{self.version}{priv_suffix}_{tag}"
+
+        prunable_tag = f"{prune_timestamp}_prune_art__{qci_tag_base}"
+        floating_tag = f"art__{qci_tag_base}"
+
+        self.logger.info('Tagging %s/%s:%s to QCI: %s', namespace, is_name, tag, floating_tag)
+
+        if self.runtime.dry_run:
+            self.logger.info('[DRY RUN] Would tag %s -> %s, %s', tag_pullspec, prunable_tag, floating_tag)
+            return
+
+        # Tag to QCI using reference mode - creates references without copying bytes
+        # Use separate commands for prunable and floating to handle potential partial failures
+        for qci_tag in [prunable_tag, floating_tag]:
+            dest = f"{REGISTRY_QUAY_CI}:{qci_tag}"
+            cmd = (
+                f'oc --kubeconfig {os.environ["KUBECONFIG"]} '
+                f'tag {tag_pullspec} {dest} '
+                f"--reference-policy='source' --import-mode='PreserveOriginal' --reference"
+            )
+            try:
+                await exectools.cmd_assert_async(cmd, retries=3)
+            except ChildProcessError as e:
+                self.logger.error('Failed to tag %s to %s: %s', tag_pullspec, dest, e)
+                raise
+
     @start_as_current_span_async(TRACER, "build-sync.update-nightly-imagestreams")
     async def _update_nightly_imagestreams(self):
         """
@@ -491,6 +625,9 @@ async def _update_nightly_imagestreams(self):
         # Populate CI imagestreams
         await self._populate_ci_imagestreams()
 
+        # Mirror RHCOS images to QCI for external CI consumption
+        await self._mirror_rhcos_to_qci()
+
         if self.publish:
             # Run 'oc adm release new' in parallel
             tasks = []
diff --git a/pyartcd/pyartcd/pipelines/rebuild_golang_rpms.py b/pyartcd/pyartcd/pipelines/rebuild_golang_rpms.py
@@ -3,7 +3,7 @@
 import json
 import logging
 import os
-from typing import List
+from typing import Dict, List, Tuple
 
 import click
 import koji
@@ -26,6 +26,13 @@
 
 _LOGGER = logging.getLogger(__name__)
 
+# RPMs excluded from rebuild per OCP version.
+# Key: (major, minor) tuple representing the minimum version (inclusive).
+# Value: list of RPM names to exclude from that version onward.
+EXCLUDED_RPMS: Dict[Tuple[int, int], List[str]] = {
+    (4, 19): ['ignition'],
+}
+
 
 class RebuildGolangRPMsPipeline:
     def __init__(
@@ -51,15 +58,25 @@ def __init__(
         self.all = all
         self.koji_session = koji.ClientSession(BREW_HUB)
 
-    async def get_rpms_from_open_trackers(self) -> List[str]:
+    def get_excluded_rpms(self) -> set:
+        """Return the set of RPM names excluded for this OCP version."""
+        major, minor = (int(x) for x in self.ocp_version.split('.'))
+        excluded = set()
+        for (min_major, min_minor), rpms in EXCLUDED_RPMS.items():
+            if (major, minor) >= (min_major, min_minor):
+                excluded.update(rpms)
+        return excluded
+
+    async def get_rpms_from_open_trackers(self) -> Tuple[List[str], List[dict]]:
         """Fetch RPMs that need rebuilding using find-bugs:golang command.
 
         This method invokes the find-bugs:golang command to identify which RPMs
-        are affected by golang security issues and returns the list of RPMs
-        that need to be rebuilt.
+        are affected by golang security issues. RPMs in the exclusion list are
+        separated out so their trackers can be closed.
 
         Returns:
-            List[str]: List of RPM names that need rebuilding
+            Tuple of (rpms_to_rebuild, excluded_bugs) where excluded_bugs is a
+            list of dicts with 'id' and 'pscomponent' keys.
         """
         _LOGGER.info('Fetching RPMs affected by golang security issues from find-bugs:golang')
 
@@ -85,24 +102,56 @@ async def get_rpms_from_open_trackers(self) -> List[str]:
 
         if not stdout or not stdout.strip():
             _LOGGER.info('No output from find-bugs:golang command, no unfixed bugs found')
-            return []
+            return [], []
 
         json_data = json.loads(stdout)
         if not json_data:
             _LOGGER.info('No unfixed bugs found in find-bugs:golang output')
-            return []
+            return [], []
+
+        excluded_rpm_names = self.get_excluded_rpms()
 
         # Extract RPM names (pscomponent) from unfixed bugs
         rpms_to_rebuild = set()
+        excluded_bugs = []
         for bug in json_data:
             # better to panic here if pscomponent is missing since it means find-bugs:golang output format has changed in an unexpected way
             component = bug['pscomponent']
-            rpms_to_rebuild.add(component)
-            _LOGGER.info(f"Adding RPM {component} from unfixed bug {bug['id']}")
+            if component in excluded_rpm_names:
+                _LOGGER.info(f"RPM {component} is excluded for OCP {self.ocp_version}, will close tracker {bug['id']}")
+                excluded_bugs.append({'id': bug['id'], 'pscomponent': component})
+            else:
+                rpms_to_rebuild.add(component)
+                _LOGGER.info(f"Adding RPM {component} from unfixed bug {bug['id']}")
 
         rpms_list = sorted(list(rpms_to_rebuild))
         _LOGGER.info(f'Found {len(rpms_list)} RPMs to rebuild from find-bugs:golang: {rpms_list}')
-        return rpms_list
+        if excluded_bugs:
+            _LOGGER.info(f'Found {len(excluded_bugs)} excluded RPM trackers to close')
+        return rpms_list, excluded_bugs
+
+    async def close_excluded_rpm_trackers(self, excluded_bugs: List[dict]):
+        """Close tracker bugs for RPMs that are excluded from this OCP version.
+
+        Args:
+            excluded_bugs: List of dicts with 'id' and 'pscomponent' keys.
+        """
+        if not excluded_bugs:
+            return
+
+        jira_client = self.runtime.new_jira_client()
+        for bug in excluded_bugs:
+            bug_id = bug['id']
+            rpm = bug['pscomponent']
+            comment = f"{rpm} is not shipped in OCP {self.ocp_version}. Closing tracker as Not a Bug."
+
+            if self.runtime.dry_run:
+                _LOGGER.info(f"[DRY RUN] Would close {bug_id} with resolution 'Not a Bug' and comment: {comment}")
+                continue
+
+            _LOGGER.info(f"Closing tracker {bug_id} for excluded RPM {rpm}")
+            jira_client.add_comment(bug_id, comment)
+            jira_client.close_with_resolution(bug_id, resolution='Not a Bug')
 
     async def run(self):
         go_version, el_nvr_map = extract_and_validate_golang_nvrs(self.ocp_version, self.go_nvrs)
@@ -112,11 +161,21 @@ async def run(self):
         # If all=False and rpms is empty, fetch RPMs from find-bugs:golang
         if not self.all and not self.rpms:
             _LOGGER.info('all=False and rpms is empty, fetching RPMs from find-bugs:golang')
-            self.rpms = await self.get_rpms_from_open_trackers()
+            self.rpms, excluded_bugs = await self.get_rpms_from_open_trackers()
+            await self.close_excluded_rpm_trackers(excluded_bugs)
             if not self.rpms:
                 _LOGGER.info('No open rpm trackers found, exiting pipeline')
                 return
 
+        # Filter out excluded RPMs from explicit lists (--rpms or --all)
+        excluded_rpm_names = self.get_excluded_rpms()
+        if self.rpms:
+            filtered = [r for r in self.rpms if r not in excluded_rpm_names]
+            skipped = [r for r in self.rpms if r in excluded_rpm_names]
+            if skipped:
+                _LOGGER.warning(f"Excluding RPMs not shipped in OCP {self.ocp_version}: {skipped}")
+            self.rpms = filtered
+
         # For rpms - first make sure builds are tagged and available
         _LOGGER.info('Checking if golang builds are tagged and available in rpm buildroots')
         error_msg = ""
diff --git a/pyartcd/tests/pipelines/test_build_sync.py b/pyartcd/tests/pipelines/test_build_sync.py
diff --git a/pyartcd/tests/test_rebuild_golang_rpms.py b/pyartcd/tests/test_rebuild_golang_rpms.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Subproject commit b133254bd2a983a81e80cf23311eaab4d46d68b2`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Subproject commit 6ec1c9b10d0b1bf04291730936fc0796cf4f49be`