feat(ISV-6697): Release pipeline uses pending index repositories

ansible/inventory/group_vars/operator-pipeline.yml

@@ -25,6 +25,7 @@ operator_pipeline_private_cert_local_path: ../../vaults/{{ env }}/operator-pipel
 
 operator_hosted_pipeline_registry_auth_path: ../../vaults/{{ env }}/registry-auth/hosted-pipeline.json
 operator_release_pipeline_registry_auth_pull_path: ../../vaults/{{ env }}/registry-auth/release-pipeline-pull.json
+operator_release_pipeline_registry_auth_pull_pending_path: ../../vaults/{{ env }}/registry-auth/release-pipeline-pull-pending.json
 operator_release_pipeline_registry_auth_push_path: ../../vaults/{{ env }}/registry-auth/release-pipeline-push.json
 operator_release_pipeline_registry_auth_serve_path: ../../vaults/{{ env }}/registry-auth/release-pipeline-serve.json
 

ansible/roles/operator-pipeline/tasks/community-release-pipeline-trigger.yml

@@ -163,3 +163,6 @@
                     - name: ssh-dir
                       secret:
                         secretName: github-rh-operator-bundle-bot-ssh-credentials
+                    - name: iib-credentials
+                      secret:
+                        secretName: $(tt.params.quay_push_final_index_secret)

ansible/roles/operator-pipeline/tasks/operator-release-pipeline-trigger.yml

@@ -149,3 +149,6 @@
                     - name: ssh-dir
                       secret:
                         secretName: github-rh-operator-bundle-bot-ssh-credentials
+                    - name: iib-credentials
+                      secret:
+                        secretName: $(tt.params.quay_push_final_index_secret)

ansible/roles/operator-pipeline/tasks/pipeline-secrets.yml

@@ -126,6 +126,27 @@
       data:
         .dockerconfigjson: "{{ lookup('file', operator_release_pipeline_registry_auth_serve_path, rstrip=False) | b64encode }}"
 
+- name: Create Operator release pipeline pull pending registry auth secret
+  no_log: true
+  tags:
+    - secrets
+  kubernetes.core.k8s:
+    state: present
+    force: true
+    namespace: "{{ oc_namespace }}"
+    definition:
+      apiVersion: v1
+      kind: Secret
+      type: opaque
+      metadata:
+        name: release-pipeline-pull-pending-registry-auth
+        labels:
+          app: operator-pipeline
+          suffix: "{{ suffix }}"
+          env: "{{ env }}"
+      data:
+        config.json: "{{ lookup('file', operator_release_pipeline_registry_auth_pull_pending_path, rstrip=False) | b64encode }}"
+
 - name: Create Operator pipeline github bot token secret
   no_log: true
   tags:

ansible/roles/operator-pipeline/templates/openshift/pipelines/operator-release-pipeline.yml

@@ -65,6 +65,9 @@ spec:
     - name: signing_pub_secret_key
       description: The key within the Kubernetes Secret that contains the public key for verifying signatures.
       default: sig-key.pub
+    - name: registry_pull_pending_secret_name
+      description: The name of the Kubernetes Secret that contains registry credentials for pulling from pending repositories.
+      default: release-pipeline-pull-pending-registry-auth
     - name: cert_project_required
       description: >-
         A flag determines whether a cert project identifier is required
@@ -85,6 +88,10 @@ spec:
     - name: registry-serve-credentials
     - name: hosted-registry-credentials
     - name: image-data
+    - name: iib-credentials
+      description: |
+        Quay credentials for IIB to authenticate and overwrite from_index.
+        Required for IIB to push to pending repositories.
 
   tasks:
 
@@ -491,11 +498,22 @@ spec:
         - name: credentials
           workspace: hosted-registry-credentials
 
+    # Generate timestamp for consistent tagging across IIB builds and publish
+    - name: generate-build-timestamp
+      runAfter:
+        - get-supported-versions
+      taskRef:
+        name: generate-build-timestamp
+      params:
+        - name: pipeline_image
+          value: "$(params.pipeline_image)"
+
     # acquire/lease the resource to resolve the conflict of concurrent pipelineruns
     - name: acquire-lease
       retries: 8
       runAfter:
         - build-fragment-images
+        - generate-build-timestamp
       when:
         - &notUndistributed
           input: "$(tasks.get-pyxis-certification-data.results.operator_distribution)"
@@ -525,7 +543,7 @@ spec:
         - name: pipeline_image
           value: "$(params.pipeline_image)"
         - name: index_images
-          value: "$(tasks.get-supported-versions.results.public_repository_mirrors_with_version)"
+          value: "$(tasks.get-supported-versions.results.pending_repositories_with_version)"
         - name: commit_sha
           value: "$(params.git_commit)"
         - name: catalogs_with_added_or_modified_operators
@@ -542,10 +560,14 @@ spec:
           value: "$(params.kerberos_keytab_secret_name)"
         - name: kerberos_keytab_secret_key
           value: "$(params.kerberos_keytab_secret_key)"
+        - name: build_tags_suffix
+          value: "$(tasks.generate-build-timestamp.results.timestamp)"
       workspaces:
         - name: output
           workspace: results
           subPath: paths
+        - name: iib-credentials
+          workspace: iib-credentials
 
     # call IIB to add the bundle to index
     - name: add-bundle-to-index
@@ -565,7 +587,7 @@ spec:
         - name: pipeline_image
           value: "$(params.pipeline_image)"
         - name: index_images
-          value: "$(tasks.get-supported-versions.results.public_repository_mirrors_with_version)"
+          value: "$(tasks.get-supported-versions.results.pending_repositories_with_version)"
         - name: bundle_pullspec
           value: "$(tasks.copy-bundle-image-to-released-registry.results.image_pullspec)"
         - name: iib_url
@@ -578,10 +600,14 @@ spec:
           value: "$(params.kerberos_keytab_secret_key)"
         - name: upgrade-graph-mode
           value: "$(tasks.read-config.results.upgrade-graph-mode)"
+        - name: build_tags_suffix
+          value: "$(tasks.generate-build-timestamp.results.timestamp)"
       workspaces:
         - name: output
           workspace: results
           subPath: paths
+        - name: iib-credentials
+          workspace: iib-credentials
 
     - name: sign-index-image
       runAfter:
@@ -595,6 +621,8 @@ spec:
       params:
         - name: pipeline_image
           value: "$(params.pipeline_image)"
+        - name: public_repository_mirror
+          value: "$(tasks.get-supported-versions.results.public_repository_mirror)"
         - name: requester
           value: "araszka"
         - name: sig_key_id
@@ -624,6 +652,9 @@ spec:
           value: "$(params.signing_pub_secret_name)"
         - name: signing_pub_secret_key
           value: "$(params.signing_pub_secret_key)"
+
+        - name: registry_auth_secret_name
+          value: "$(params.registry_pull_pending_secret_name)"
       workspaces:
         - name: results
           workspace: results
@@ -646,6 +677,8 @@ spec:
           value: "$(tasks.get-supported-versions.results.public_repository_mirror)"
         - name: quay_push_final_index_secret
           value: "$(params.quay_push_final_index_secret)"
+        - name: build_tags_suffix
+          value: "$(tasks.generate-build-timestamp.results.timestamp)"
       workspaces:
         - name: results
           workspace: results

ansible/roles/operator-pipeline/templates/openshift/tasks/add-bundle-to-index.yml

@@ -42,6 +42,11 @@ spec:
       default: ""
       description: "A graph update mode that defines how channel graphs are updated"
 
+    - name: build_tags_suffix
+      default: ""
+      description: |
+        Timestamp suffix for build tags (used with overwrite to ensure consistent tagging with publish task).
+
   results:
     - name: status
       description: Indicates a status of adding a bundle to an index
@@ -53,6 +58,12 @@ spec:
     - name: output
     - name: credentials
       optional: true
+    - name: iib-credentials
+      description: |
+        Optional workspace containing Quay credentials for IIB to authenticate and overwrite from_index.
+        Required for release pipeline to enable IIB to push to pending repositories.
+        Expected to contain 'username' and 'password' files.
+      optional: true
 
   steps:
     - name: add-bundle-to-index
@@ -67,9 +78,9 @@ spec:
           mountPath: "/etc/kerberos"
       script: |
         #! /usr/bin/env bash
-        set -xe
+        # DO NOT USE set -x to avoid revealing credentials in logs!
+        set -e
 
-        ENV=$(params.environment)
         INDEX_IMAGES="$(params.index_images)"
 
         EXTRA_ARGS=""
@@ -85,6 +96,17 @@ spec:
           EXTRA_ARGS+=" --mode $(params.upgrade-graph-mode)"
         fi
 
+        # Add IIB overwrite token if credentials workspace is provided (for release pipeline)
+        if [[ "$(workspaces.iib-credentials.bound)" == "true" ]]; then
+          IIB_QUAY_USER=$(cat $(workspaces.iib-credentials.path)/username)
+          IIB_QUAY_TOKEN=$(cat $(workspaces.iib-credentials.path)/password)
+          export IIB_OVERWRITE_TOKEN="${IIB_QUAY_USER}:${IIB_QUAY_TOKEN}"
+
+          # Add build tags suffix for consistent tagging with publish task
+          if [[ "$(params.build_tags_suffix)" != "" ]]; then
+            EXTRA_ARGS+=" --build-tags-suffix $(params.build_tags_suffix)"
+          fi
+        fi
 
         # DO NOT use `--verbose` to avoid auth headers appearing in logs
         index \
@@ -96,4 +118,31 @@ spec:
 
 
         echo -n "success" | tee "$(results.status.path)"
+        echo
         cat index-image-paths.txt
+        echo
+
+        # WORKAROUND: Manually overwriting index images using skopeo (TODO: remove when overwrite token is fixed)
+        if [[ "$(workspaces.iib-credentials.bound)" == "true" ]]; then
+          TEMP_IMAGES=$(cat index-image-paths.txt | tr "," " ")
+          for i in $TEMP_IMAGES
+          do
+            SRC_IMAGE=$(echo $i | awk -F '+' '{print $2}')
+            DEST_IMAGE=$(echo $i | awk -F '+' '{print $1}')
+            echo "1. Version tag: copying $SRC_IMAGE to $DEST_IMAGE"
+            skopeo copy --format v2s2 --all --src-no-creds \
+              --dest-creds $IIB_QUAY_USER:$IIB_QUAY_TOKEN \
+              docker://$SRC_IMAGE \
+              docker://$DEST_IMAGE
+
+            # Also copy with permanent tag if build_tags_suffix is set
+            if [[ "$(params.build_tags_suffix)" != "" ]]; then
+              DEST_IMAGE_PERMANENT="${DEST_IMAGE}-$(params.build_tags_suffix)"
+              echo "2. Permanent tag: copying $SRC_IMAGE to $DEST_IMAGE_PERMANENT"
+              skopeo copy --format v2s2 --all --src-no-creds \
+                --dest-creds $IIB_QUAY_USER:$IIB_QUAY_TOKEN \
+                docker://$SRC_IMAGE \
+                docker://$DEST_IMAGE_PERMANENT
+            fi
+          done
+        fi