From 1f665b54923c3a03f8d83e20273f4c2f6ad83e1e Mon Sep 17 00:00:00 2001 From: rabi Date: Wed, 3 Jun 2026 11:35:37 +0530 Subject: [PATCH] Add bootc passthrough baremetal CI job Add a bootc EDPM baremetal job variant that reuses the normal content provider while switching the deployment to PassThrough with the direct bootc image. Depends-On: https://github.com/openstack-k8s-operators/install_yamls/pull/1149 Signed-off-by: rabi --- ci/playbooks/bootc-ipa-pre-deploy.yml | 323 +++++++++++++++++++++++++ ci/scripts/patch_ipa_fips_boot_karg.py | 124 ++++++++++ zuul.d/jobs.yaml | 26 ++ zuul.d/projects.yaml | 3 + 4 files changed, 476 insertions(+) create mode 100644 ci/playbooks/bootc-ipa-pre-deploy.yml create mode 100644 ci/scripts/patch_ipa_fips_boot_karg.py diff --git a/ci/playbooks/bootc-ipa-pre-deploy.yml b/ci/playbooks/bootc-ipa-pre-deploy.yml new file mode 100644 index 00000000..da87bf45 --- /dev/null +++ b/ci/playbooks/bootc-ipa-pre-deploy.yml @@ -0,0 +1,323 @@ +--- +- name: Build and host bootc IPA artifacts + hosts: "{{ cifmw_target_hook_host | default(cifmw_target_host | default('localhost')) }}" + gather_facts: false + vars: + cifmw_bootc_ipa_artifact_dir: "{{ cifmw_basedir }}/artifacts/bootc-ipa" + cifmw_bootc_ipa_build_dir: "{{ ansible_user_dir }}/bootc-ipa-build" + cifmw_bootc_ipa_build_enabled: false + cifmw_bootc_ipa_build_output_prefix: "ipa-centos9-stable-2025.2-podman" + cifmw_bootc_ipa_release: "9-stream" + cifmw_bootc_ipa_branch: "stable/2025.2" + cifmw_bootc_ipa_flavor: "centos9" + cifmw_bootc_ipa_distro: "centos" + cifmw_bootc_ipa_server_namespace: "openstack" + cifmw_bootc_ipa_server_name: "bootc-ipa-artifacts" + cifmw_bootc_ipa_server_port: 8080 + cifmw_bootc_ipa_server_image: "registry.access.redhat.com/ubi9/ubi:latest" + cifmw_bootc_ipa_published_basename: >- + ipa-{{ cifmw_bootc_ipa_flavor }}-{{ cifmw_bootc_ipa_branch | replace('/', '-') }} + cifmw_bootc_ipa_raw_kernel_file: >- + {{ cifmw_bootc_ipa_artifact_dir }}/{{ cifmw_bootc_ipa_build_output_prefix }}.kernel + cifmw_bootc_ipa_raw_initramfs_file: >- + {{ cifmw_bootc_ipa_artifact_dir }}/{{ cifmw_bootc_ipa_build_output_prefix }}.initramfs + cifmw_bootc_ipa_kernel_file: >- + {{ cifmw_bootc_ipa_artifact_dir }}/{{ cifmw_bootc_ipa_published_basename }}.kernel + cifmw_bootc_ipa_initramfs_file: >- + {{ cifmw_bootc_ipa_artifact_dir }}/{{ cifmw_bootc_ipa_published_basename }}.initramfs + cifmw_bootc_ipa_tarball: >- + {{ cifmw_bootc_ipa_artifact_dir }}/{{ cifmw_bootc_ipa_published_basename }}.tar.gz + cifmw_bootc_ipa_source_dir: "{{ cifmw_bootc_ipa_build_dir }}/ironic-python-agent" + cifmw_bootc_ipa_fips_patch_enabled: true + cifmw_bootc_ipa_service_ip: "" + cifmw_bootc_ipa_service_url: "" + tasks: + - name: Ensure IPA artifact directory exists + ansible.builtin.file: + path: "{{ cifmw_bootc_ipa_artifact_dir }}" + state: directory + mode: "0755" + + - name: Ensure IPA build directory exists + when: cifmw_bootc_ipa_build_enabled | bool + ansible.builtin.file: + path: "{{ cifmw_bootc_ipa_build_dir }}" + state: directory + mode: "0755" + + - name: Write minimal container steps file + when: cifmw_bootc_ipa_build_enabled | bool + ansible.builtin.copy: + dest: "{{ cifmw_bootc_ipa_build_dir }}/mysteps.yaml" + content: | + steps: [] + mode: "0644" + + - name: Clone IPA source for FIPS boot karg patch + when: cifmw_bootc_ipa_build_enabled | bool and cifmw_bootc_ipa_fips_patch_enabled | bool + ansible.builtin.git: + repo: "https://opendev.org/openstack/ironic-python-agent.git" + dest: "{{ cifmw_bootc_ipa_source_dir }}" + version: "{{ cifmw_bootc_ipa_branch }}" + + - name: Patch IPA with FIPS boot karg fixup + when: cifmw_bootc_ipa_build_enabled | bool and cifmw_bootc_ipa_fips_patch_enabled | bool + ansible.builtin.command: + cmd: >- + python3 {{ playbook_dir }}/../scripts/patch_ipa_fips_boot_karg.py + {{ cifmw_bootc_ipa_source_dir }}/ironic_python_agent/extensions/standby.py + + - name: Build podman-enabled IPA artifacts + when: cifmw_bootc_ipa_build_enabled | bool + args: + chdir: "{{ cifmw_bootc_ipa_artifact_dir }}" + creates: "{{ cifmw_bootc_ipa_raw_initramfs_file }}" + executable: /bin/bash + ansible.builtin.shell: | + set -euo pipefail + python3 -m venv "{{ cifmw_bootc_ipa_build_dir }}/.venv" + . "{{ cifmw_bootc_ipa_build_dir }}/.venv/bin/activate" + python -m pip install --upgrade pip setuptools wheel + python -m pip install ironic-python-agent-builder + export DIB_ALLOW_ARBITRARY_CONTAINERS=true + export DIB_RUNNER=podman + export DIB_STEPS_FILE_PATH="{{ cifmw_bootc_ipa_build_dir }}/mysteps.yaml" + {% if cifmw_bootc_ipa_fips_patch_enabled | bool %} + export DIB_REPOLOCATION_ironic_python_agent="{{ cifmw_bootc_ipa_source_dir }}" + export DIB_REPOREF_ironic_python_agent="{{ cifmw_bootc_ipa_branch }}" + {% endif %} + ironic-python-agent-builder \ + -o "{{ cifmw_bootc_ipa_build_output_prefix }}" \ + -r "{{ cifmw_bootc_ipa_release }}" \ + -b "{{ cifmw_bootc_ipa_branch }}" \ + -e ironic-python-agent-podman \ + -v "{{ cifmw_bootc_ipa_distro }}" + + - name: Prepare IPA artifacts for hosting + when: cifmw_bootc_ipa_build_enabled | bool + block: + - name: Check for raw IPA artifacts + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ cifmw_bootc_ipa_raw_kernel_file }}" + - "{{ cifmw_bootc_ipa_raw_initramfs_file }}" + register: cifmw_bootc_ipa_raw_artifacts + + - name: Assert raw IPA artifacts exist + ansible.builtin.assert: + that: + - cifmw_bootc_ipa_raw_artifacts.results | map(attribute='stat.exists') | min + fail_msg: >- + Missing raw IPA artifacts. Expected + {{ cifmw_bootc_ipa_raw_kernel_file }} and + {{ cifmw_bootc_ipa_raw_initramfs_file }}. + + - name: Copy kernel to published filename + ansible.builtin.copy: + src: "{{ cifmw_bootc_ipa_raw_kernel_file }}" + dest: "{{ cifmw_bootc_ipa_kernel_file }}" + mode: "0644" + remote_src: true + + - name: Copy initramfs to published filename + ansible.builtin.copy: + src: "{{ cifmw_bootc_ipa_raw_initramfs_file }}" + dest: "{{ cifmw_bootc_ipa_initramfs_file }}" + mode: "0644" + remote_src: true + + - name: Package IPA tarball with downloader-compatible filenames + args: + chdir: "{{ cifmw_bootc_ipa_artifact_dir }}" + creates: "{{ cifmw_bootc_ipa_tarball }}" + executable: /bin/bash + ansible.builtin.shell: | + set -euo pipefail + tar -caf "{{ cifmw_bootc_ipa_tarball }}" \ + "{{ cifmw_bootc_ipa_published_basename }}.kernel" \ + "{{ cifmw_bootc_ipa_published_basename }}.initramfs" + + - name: Ensure IPA hosting namespace exists + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ cifmw_bootc_ipa_server_namespace }}" + + - name: Create IPA hosting deployment + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + state: present + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{ cifmw_bootc_ipa_server_name }}" + namespace: "{{ cifmw_bootc_ipa_server_namespace }}" + spec: + replicas: 1 + selector: + matchLabels: + app: "{{ cifmw_bootc_ipa_server_name }}" + template: + metadata: + labels: + app: "{{ cifmw_bootc_ipa_server_name }}" + spec: + containers: + - name: server + image: "{{ cifmw_bootc_ipa_server_image }}" + command: + - python3 + - -m + - http.server + - "{{ cifmw_bootc_ipa_server_port | string }}" + - --directory + - /srv + ports: + - containerPort: "{{ cifmw_bootc_ipa_server_port }}" + volumeMounts: + - name: content + mountPath: /srv + volumes: + - name: content + emptyDir: {} + + - name: Create IPA hosting service + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + state: present + definition: + apiVersion: v1 + kind: Service + metadata: + name: "{{ cifmw_bootc_ipa_server_name }}" + namespace: "{{ cifmw_bootc_ipa_server_namespace }}" + spec: + selector: + app: "{{ cifmw_bootc_ipa_server_name }}" + ports: + - name: http + port: "{{ cifmw_bootc_ipa_server_port }}" + targetPort: "{{ cifmw_bootc_ipa_server_port }}" + + - name: Get IPA hosting service cluster IP + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_version: v1 + kind: Service + name: "{{ cifmw_bootc_ipa_server_name }}" + namespace: "{{ cifmw_bootc_ipa_server_namespace }}" + register: cifmw_bootc_ipa_service + + - name: Set IPA hosting service URL from cluster IP + ansible.builtin.set_fact: + cifmw_bootc_ipa_service_ip: "{{ cifmw_bootc_ipa_service.resources[0].spec.clusterIP }}" + cifmw_bootc_ipa_service_url: "http://{{ cifmw_bootc_ipa_service.resources[0].spec.clusterIP }}:{{ cifmw_bootc_ipa_server_port }}" + + - name: Wait for IPA hosting pod to be ready + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc wait pod -n {{ cifmw_bootc_ipa_server_namespace }} + -l app={{ cifmw_bootc_ipa_server_name }} + --for=condition=Ready + --timeout=300s + + - name: Get IPA hosting pod name + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc get pod -n {{ cifmw_bootc_ipa_server_namespace }} + -l app={{ cifmw_bootc_ipa_server_name }} + -o jsonpath='{.items[0].metadata.name}' + register: cifmw_bootc_ipa_server_pod + changed_when: false + + - name: Copy IPA tarball into hosting pod + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc cp + "{{ cifmw_bootc_ipa_tarball }}" + "{{ cifmw_bootc_ipa_server_namespace }}/{{ cifmw_bootc_ipa_server_pod.stdout }}:/srv/{{ cifmw_bootc_ipa_tarball | basename }}" + + - name: Verify IPA tarball is served from hosting pod + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc exec -n {{ cifmw_bootc_ipa_server_namespace }} + {{ cifmw_bootc_ipa_server_pod.stdout }} -- + sh -c "curl -sf -o /dev/null http://127.0.0.1:{{ cifmw_bootc_ipa_server_port }}/{{ cifmw_bootc_ipa_tarball | basename }}" + + - name: Feed generated bootc IPA values back to main play + ansible.builtin.copy: + dest: "{{ cifmw_basedir }}/artifacts/{{ step }}_{{ hook_name }}.yml" + mode: "0644" + content: | + cifmw_bootc_ipa_service_url: "{{ cifmw_bootc_ipa_service_url }}" + cifmw_edpm_prepare_extra_vars: + BMO_IPA_BASEURI: "{{ cifmw_bootc_ipa_service_url }}" + BMO_IPA_BRANCH: "{{ cifmw_bootc_ipa_branch }}" + BMO_IPA_FLAVOR: "{{ cifmw_bootc_ipa_flavor }}" + + - name: Find ironic configmap namespace + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc get configmap ironic-bmo-configmap -A + -o jsonpath='{.items[0].metadata.namespace}' + register: cifmw_bootc_ipa_ironic_namespace + changed_when: false + failed_when: false + + - name: Patch live ironic configmap with bootc IPA source + when: cifmw_bootc_ipa_ironic_namespace.rc == 0 and cifmw_bootc_ipa_ironic_namespace.stdout | length > 0 + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + state: patched + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: ironic-bmo-configmap + namespace: "{{ cifmw_bootc_ipa_ironic_namespace.stdout }}" + data: + IPA_BASEURI: "{{ cifmw_bootc_ipa_service_url }}" + IPA_BRANCH: "{{ cifmw_bootc_ipa_branch }}" + IPA_FLAVOR: "{{ cifmw_bootc_ipa_flavor }}" + + - name: Restart ironic deployment to re-run IPA downloader + when: cifmw_bootc_ipa_ironic_namespace.rc == 0 and cifmw_bootc_ipa_ironic_namespace.stdout | length > 0 + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc rollout restart deployment/ironic + -n {{ cifmw_bootc_ipa_ironic_namespace.stdout }} + + - name: Wait for ironic rollout after configmap patch + when: cifmw_bootc_ipa_ironic_namespace.rc == 0 and cifmw_bootc_ipa_ironic_namespace.stdout | length > 0 + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + ansible.builtin.command: + cmd: >- + oc rollout status deployment/ironic + -n {{ cifmw_bootc_ipa_ironic_namespace.stdout }} + --timeout=10m diff --git a/ci/scripts/patch_ipa_fips_boot_karg.py b/ci/scripts/patch_ipa_fips_boot_karg.py new file mode 100644 index 00000000..53ae09da --- /dev/null +++ b/ci/scripts/patch_ipa_fips_boot_karg.py @@ -0,0 +1,124 @@ +"""Patch IPA standby.py to inject boot=UUID karg after bootc install for FIPS. + +When bootc install to-disk creates a non-LUKS layout, /boot lives on the root +partition with no separate boot partition. The boot= kernel argument is not +set, but dracut's FIPS module needs it to mount /boot during initramfs. +Without it the HMAC integrity check fails because ostree-prepare-root has +already transformed $NEWROOT, hiding the real /boot content. + +This script splices a small fixup method into +ironic_python_agent/extensions/standby.py and wires it into +execute_bootc_install so the BLS entry gets boot=UUID= appended +whenever fips=1 is present without a boot= argument. +""" +import sys +import textwrap + +FIXUP_METHOD = textwrap.dedent("""\ + + def _fixup_fips_boot_karg(self, device): + \"\"\"Inject boot=UUID into BLS entry when fips=1 is set without boot=. + + bootc install to-disk (non-LUKS) puts /boot on the root partition but + does not inject a boot= kernel argument. The dracut FIPS module needs + boot= to mount /boot during initramfs; without it the kernel HMAC + check fails because ostree-prepare-root has already overlaid the + sysroot. + \"\"\" + import tempfile as _tempfile + + try: + output = utils.execute( + 'lsblk', '-nlo', 'PATH,FSTYPE,UUID', device, + use_standard_locale=True) + except Exception: + LOG.warning('lsblk failed on %s, skipping FIPS boot karg fixup', + device) + return + + root_part = root_uuid = None + for line in output[0].strip().split('\\n'): + fields = line.split() + if len(fields) >= 3 and fields[1] in ('xfs', 'ext4', 'btrfs'): + root_part, root_uuid = fields[0], fields[2] + break + + if not root_part: + return + + mnt = _tempfile.mkdtemp(prefix='ipa-fips-') + try: + utils.execute('mount', root_part, mnt) + bls_dir = os.path.join(mnt, 'boot', 'loader', 'entries') + if not os.path.isdir(bls_dir): + return + for name in os.listdir(bls_dir): + if not name.endswith('.conf'): + continue + path = os.path.join(bls_dir, name) + with open(path) as fh: + text = fh.read() + if 'fips=1' in text and 'boot=' not in text: + text = text.replace( + 'fips=1', 'fips=1 boot=UUID=%s' % root_uuid) + with open(path, 'w') as fh: + fh.write(text) + LOG.info('Injected boot=UUID=%s into BLS %s for FIPS', + root_uuid, name) + except Exception as exc: + LOG.warning('FIPS boot karg fixup failed: %s', exc) + finally: + utils.execute('umount', mnt, check_exit_code=False) + try: + os.rmdir(mnt) + except OSError: + pass + +""") + +CALL_LINE = " self._fixup_fips_boot_karg(device)\n" + + +def patch(filepath): + with open(filepath) as fh: + content = fh.read() + + # --- 1. Insert the method before _download_container_and_bootc_install --- + marker = " def _download_container_and_bootc_install(" + if marker not in content: + print("ERROR: cannot find _download_container_and_bootc_install", + file=sys.stderr) + sys.exit(1) + + content = content.replace(marker, FIXUP_METHOD + marker, 1) + + # --- 2. Insert the call after _validate_partitioning in bootc path --- + # We look for the pattern: + # _validate_partitioning(device) + # followed within a few lines by the bootc-specific configdrive block. + target = " _validate_partitioning(device)\n" + idx = content.find(target) + while idx != -1: + rest = content[idx + len(target):] + # The bootc execute_bootc_install has 'Container image' nearby + if "Container image" in rest[:600]: + content = (content[:idx + len(target)] + + "\n" + CALL_LINE + "\n" + + content[idx + len(target):]) + break + idx = content.find(target, idx + 1) + else: + print("ERROR: cannot find call-site in execute_bootc_install", + file=sys.stderr) + sys.exit(1) + + with open(filepath, "w") as fh: + fh.write(content) + print("Patched %s" % filepath) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: %s " % sys.argv[0], file=sys.stderr) + sys.exit(1) + patch(sys.argv[1]) diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index e366284f..2cc9c7d3 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -12,6 +12,32 @@ - job: name: openstack-baremetal-operator-crc-podified-edpm-baremetal parent: cifmw-crc-podified-edpm-baremetal + +- job: + name: openstack-baremetal-operator-crc-podified-edpm-baremetal-bootc + parent: cifmw-crc-podified-edpm-baremetal-bootc + vars: + cifmw_update_containers: true + cifmw_update_containers_openstack: true + cifmw_update_containers_edpm_image_url: oci://quay.io/openstack-k8s-operators/edpm-bootc:latest + pre_deploy: + - name: 61 Bootc IPA artifacts + type: playbook + source: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/openstack-baremetal-operator/ci/playbooks/bootc-ipa-pre-deploy.yml" + extra_vars: + cifmw_bootc_ipa_build_enabled: true + post_ctlplane_deploy: + - name: 61 Bootc IPA reconfigure ironic + type: playbook + source: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/openstack-baremetal-operator/ci/playbooks/bootc-ipa-pre-deploy.yml" + extra_vars: + cifmw_bootc_ipa_build_enabled: false + cifmw_install_yamls_vars: + BAREMETAL_OS_IMG_TYPE: PassThrough + BMO_DEFAULT_DEPLOY_INTERFACE: bootc + BM_INSTANCE_MEMORY: 16384 + BMAAS_INSTANCE_DISK_SIZE: 40 + - job: name: openstack-baremetal-operator-edpm-baremetal-minor-update parent: cifmw-crc-podified-edpm-baremetal-minor-update diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 2821b20a..e6a4e565 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -9,6 +9,9 @@ - openstack-baremetal-operator-crc-podified-edpm-baremetal: dependencies: - openstack-baremetal-operator-content-provider + - openstack-baremetal-operator-crc-podified-edpm-baremetal-bootc: + dependencies: + - openstack-baremetal-operator-content-provider - openstack-baremetal-operator-edpm-baremetal-minor-update: dependencies: - openstack-baremetal-operator-content-provider