ci: build peerpod-ctrl and webhook images in nightly e2e

After the kustomize-to-helm migration, nightly e2e tests deploy
peerpod-ctrl and webhook as sub-charts but use stale `latest` images
from quay.io. Since peerpod-ctrl shares the cloud-providers Go module
with caa, version skew can mask breaking changes.

Build both images alongside caa/podvm in the nightly pipeline and wire
their references through to the helm install via PEERPOD_CTRL_IMAGE and
WEBHOOK_IMAGE environment variables.

The subchart image override logic is centralized in Helm.ConfigureSubchartImages()
to avoid duplicating code across all provider implementations.

Assisted-by: Claude
Signed-off-by: Wainer dos Santos Moschetta <wainersm@redhat.com>

Author: wainersm

.github/workflows/e2e_aws.yaml

@@ -18,6 +18,16 @@ on:
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
         required: false
         type: string
+      peerpod_ctrl_image:
+        description: The peerpod-ctrl OCI image (including tag)
+        required: false
+        default: ''
+        type: string
+      webhook_image:
+        description: The peer-pods-webhook OCI image (including tag)
+        required: false
+        default: ''
+        type: string
       cluster_type:
         description: Specify the cluster type. Accepted values are "onprem" or "eks".
         default: onprem
@@ -70,8 +80,10 @@ jobs:
         CAA_IMAGE: "${{ inputs.caa_image }}"
         CLUSTER_TYPE: "${{ inputs.cluster_type }}"
         CONTAINER_RUNTIME: "${{ inputs.container_runtime }}"
+        PEERPOD_CTRL_IMAGE: "${{ inputs.peerpod_ctrl_image }}"
         PODVM_IMAGE: "${{ inputs.podvm_image }}"
         RESOURCES_BASENAME: "ci-caa-${{ github.run_id }}-${{ github.run_attempt }}"
+        WEBHOOK_IMAGE: "${{ inputs.webhook_image }}"
     permissions:
       id-token: write # Required by aws-actions/configure-aws-credentials
       contents: read  # Required by aws-actions/configure-aws-credentials

.github/workflows/e2e_byom.yaml

@@ -13,6 +13,16 @@ on:
       caa_image:
         type: string
         required: true
+      peerpod_ctrl_image:
+        description: The peerpod-ctrl OCI image (including tag)
+        required: false
+        default: ''
+        type: string
+      webhook_image:
+        description: The peer-pods-webhook OCI image (including tag)
+        required: false
+        default: ''
+        type: string
       git_ref:
         default: 'main'
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
@@ -36,6 +46,8 @@ jobs:
     env:
       CAA_IMAGE: "${{ inputs.caa_image }}"
       BYOM_PODVM_IMAGE: "${{ inputs.podvm_image }}"
+      PEERPOD_CTRL_IMAGE: "${{ inputs.peerpod_ctrl_image }}"
+      WEBHOOK_IMAGE: "${{ inputs.webhook_image }}"
     steps:
       - name: Checkout Code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/e2e_docker.yaml

@@ -23,6 +23,16 @@ on:
         description: Name of the container runtime. Either containerd or crio.
         required: false
         type: string
+      peerpod_ctrl_image:
+        description: The peerpod-ctrl OCI image (including tag)
+        required: false
+        default: ''
+        type: string
+      webhook_image:
+        description: The peer-pods-webhook OCI image (including tag)
+        required: false
+        default: ''
+        type: string
     secrets:
       QUAY_PASSWORD:
         required: true
@@ -44,6 +54,8 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       CAA_IMAGE: "${{ inputs.caa_image }}"
+      PEERPOD_CTRL_IMAGE: "${{ inputs.peerpod_ctrl_image }}"
+      WEBHOOK_IMAGE: "${{ inputs.webhook_image }}"
     steps:
       - name: Checkout Code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

.github/workflows/e2e_libvirt.yaml

@@ -28,6 +28,16 @@ on:
         description: Name of the container runtime. Either containerd or crio.
         required: false
         type: string
+      peerpod_ctrl_image:
+        description: The peerpod-ctrl OCI image (including tag)
+        required: false
+        default: ''
+        type: string
+      webhook_image:
+        description: The peer-pods-webhook OCI image (including tag)
+        required: false
+        default: ''
+        type: string
     secrets:
       REGISTRY_CREDENTIAL_ENCODED:
         required: true
@@ -162,10 +172,12 @@ jobs:
           CLOUD_PROVIDER: libvirt
           CONTAINER_RUNTIME: ${{ inputs.container_runtime }}
           DEPLOY_KBS: "true"
+          PEERPOD_CTRL_IMAGE: "${{ inputs.peerpod_ctrl_image }}"
           TEST_TEARDOWN: "no"
           TEST_PROVISION_FILE: ${{ github.workspace }}/src/cloud-api-adaptor/libvirt.properties
           TEST_PODVM_IMAGE: ${{ env.PODVM_QCOW2 }}
           TEST_E2E_TIMEOUT: "75m"
+          WEBHOOK_IMAGE: "${{ inputs.webhook_image }}"
         run: |
           # Default: provision cluster and install CAA
           export TEST_PROVISION="yes"

.github/workflows/e2e_run_all.yaml

@@ -185,14 +185,56 @@ jobs:
     secrets:
       QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
 
+  # Build and push the peerpod-ctrl image (per-arch)
+  peerpod_ctrl_image_amd64:
+    uses: ./.github/workflows/peerpod-ctrl_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tags: ${{ inputs.caa_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: linux/amd64
+      runner: ubuntu-24.04
+    permissions:
+      contents: read
+      packages: write # Required to publish the image to ghcr
+    secrets:
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
+  peerpod_ctrl_image_s390x:
+    uses: ./.github/workflows/peerpod-ctrl_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tags: ${{ inputs.caa_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: linux/s390x
+      runner: ubuntu-24.04-s390x
+    permissions:
+      contents: read
+      packages: write # Required to publish the image to ghcr
+    secrets:
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
+  # Build and push the webhook image
+  webhook_image:
+    uses: ./.github/workflows/webhook_image.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tags: ${{ inputs.caa_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+    permissions:
+      contents: read
+      packages: write # Required to publish the image to ghcr
+    secrets:
+      QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
+
   # Run AWS e2e tests if pull request labeled 'test_e2e_aws'
   aws:
     name: aws
     if: |
       github.event_name == 'schedule' ||
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_aws')
-    needs: [podvm_ubuntu_amd64, caa_image_amd64]
+    needs: [podvm_ubuntu_amd64, caa_image_amd64, peerpod_ctrl_image_amd64, webhook_image]
     strategy:
       fail-fast: false
       matrix:
@@ -207,6 +249,8 @@ jobs:
       container_runtime: ${{ matrix.container_runtime }}
       podvm_image: ${{ needs.podvm_ubuntu_amd64.outputs.qcow2_oras_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-amd64
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       AWS_IAM_ROLE_ARN: ${{ secrets.AWS_IAM_ROLE_ARN }}
 
@@ -234,13 +278,15 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
-    needs: [podvm_mkosi_amd64, libvirt_e2e_arch_prep, caa_image_amd64]
+    needs: [podvm_mkosi_amd64, libvirt_e2e_arch_prep, caa_image_amd64, peerpod_ctrl_image_amd64, webhook_image]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: ubuntu-24.04
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev-amd64
       podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.qcow2_oras_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-amd64
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }}
 
@@ -252,13 +298,15 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
-    needs: [podvm_mkosi_s390x, libvirt_e2e_arch_prep, caa_image_s390x]
+    needs: [podvm_mkosi_s390x, libvirt_e2e_arch_prep, caa_image_s390x, peerpod_ctrl_image_s390x, webhook_image]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: s390x-large
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev-s390x
       podvm_image: ${{ needs.podvm_mkosi_s390x.outputs.qcow2_oras_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-s390x
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }}
 
@@ -270,13 +318,15 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
-    needs: [podvm_ubuntu_amd64, libvirt_e2e_arch_prep, caa_image_amd64]
+    needs: [podvm_ubuntu_amd64, libvirt_e2e_arch_prep, caa_image_amd64, peerpod_ctrl_image_amd64, webhook_image]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: ubuntu-24.04
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-amd64-dev
       podvm_image: ${{ needs.podvm_ubuntu_amd64.outputs.qcow2_oras_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-amd64
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }}
 
@@ -288,13 +338,15 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
-    needs: [podvm_ubuntu_s390x, libvirt_e2e_arch_prep, caa_image_s390x]
+    needs: [podvm_ubuntu_s390x, libvirt_e2e_arch_prep, caa_image_s390x, peerpod_ctrl_image_s390x, webhook_image]
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       runner: s390x-large
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-s390x-dev
       podvm_image: ${{ needs.podvm_ubuntu_s390x.outputs.qcow2_oras_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-s390x
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       REGISTRY_CREDENTIAL_ENCODED: ${{ secrets.REGISTRY_CREDENTIAL_ENCODED }}
 
@@ -305,7 +357,7 @@ jobs:
       github.event_name == 'schedule' ||
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_docker')
-    needs: [podvm_mkosi_amd64, caa_image_amd64]
+    needs: [podvm_mkosi_amd64, caa_image_amd64, peerpod_ctrl_image_amd64, webhook_image]
     strategy:
       fail-fast: false
       matrix:
@@ -323,6 +375,8 @@ jobs:
       container_runtime: ${{ matrix.container_runtime }}
       podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.docker_oci_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-amd64
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}
     secrets:
       QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
 
@@ -333,9 +387,11 @@ jobs:
       github.event_name == 'schedule' ||
       github.event_name == 'workflow_dispatch' ||
       contains(github.event.pull_request.labels.*.name, 'test_e2e_byom')
-    needs: [podvm_ubuntu_amd64, caa_image_amd64]
+    needs: [podvm_ubuntu_amd64, caa_image_amd64, peerpod_ctrl_image_amd64, webhook_image]
     uses: ./.github/workflows/e2e_byom.yaml
     with:
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev-amd64
       podvm_image: ${{ needs.podvm_ubuntu_amd64.outputs.byom_e2e_image }}
       git_ref: ${{ inputs.git_ref }}
+      peerpod_ctrl_image: ${{ inputs.registry }}/peerpod-ctrl:${{ inputs.caa_image_tag }}-amd64
+      webhook_image: ${{ inputs.registry }}/peer-pods-webhook:${{ inputs.caa_image_tag }}

src/cloud-api-adaptor/test/e2e/README.md

@@ -89,6 +89,13 @@ Other options are provided via environment variables if you need to further cust
 - `TEST_CAA_NAMESPACE` - This option is available, primarily for running the e2e tests on a downstream version
 of confidential containers, where the cloud-api-adaptor pod is deployed to a different namespace than the default
  `confidential-containers-system`.
+- `PEERPOD_CTRL_IMAGE` - Override the peerpod-ctrl container image used by the Helm sub-chart
+(e.g. `ghcr.io/confidential-containers/peerpod-ctrl:latest`). When set, the provisioner overrides
+the Helm values `resourceCtrl.image.repository` and `resourceCtrl.image.tag`. If unset, the sub-chart
+defaults are used.
+- `WEBHOOK_IMAGE` - Override the peer-pods-webhook container image used by the Helm sub-chart
+(e.g. `ghcr.io/confidential-containers/peer-pods-webhook:latest`). When set, the provisioner overrides
+the Helm values `webhook.image.repository` and `webhook.image.tag`. If unset, the sub-chart defaults are used.
 
 # Running end-to-end tests against pre-configured cluster
 

src/cloud-api-adaptor/test/provisioner/aws/provision_common.go

@@ -1186,6 +1186,8 @@ func NewAwsInstallChart(installDir, provider string) (pv.InstallChart, error) {
 	}, nil
 }
 
+func (a *AwsInstallChart) GetHelm() *pv.Helm { return a.Helm }
+
 func (a *AwsInstallChart) Install(ctx context.Context, cfg *envconf.Config) error {
 	return a.Helm.Install(ctx, cfg)
 }

src/cloud-api-adaptor/test/provisioner/azure/provision_common.go

@@ -392,6 +392,8 @@ func NewAzureInstallChart(installDir, provider string) (pv.InstallChart, error)
 	}, nil
 }
 
+func (a *AzureInstallChart) GetHelm() *pv.Helm { return a.Helm }
+
 func (a *AzureInstallChart) Install(ctx context.Context, cfg *envconf.Config) error {
 	if err := a.Helm.Install(ctx, cfg); err != nil {
 		return err

src/cloud-api-adaptor/test/provisioner/byom/provision_common.go

@@ -257,6 +257,8 @@ func NewByomInstallChart(installDir, provider string) (pv.InstallChart, error) {
 	}, nil
 }
 
+func (b *ByomInstallChart) GetHelm() *pv.Helm { return b.Helm }
+
 func (b *ByomInstallChart) Install(ctx context.Context, cfg *envconf.Config) error {
 	// Create SSH key secret before installing Helm chart
 	if err := b.createSSHKeySecret(ctx, cfg); err != nil {

src/cloud-api-adaptor/test/provisioner/docker/provision_common.go

@@ -220,6 +220,8 @@ func NewDockerInstallChart(installDir, provider string) (pv.InstallChart, error)
 	}, nil
 }
 
+func (d *DockerInstallChart) GetHelm() *pv.Helm { return d.Helm }
+
 func (d *DockerInstallChart) Install(ctx context.Context, cfg *envconf.Config) error {
 	return d.Helm.Install(ctx, cfg)
 }