diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 75d1910980..62f9e4677d 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -306,6 +306,26 @@ jobs: env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Pull container images + run: | + # Try up to 3 times to pull container images. + for attempt in $(seq 3); do + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh + if [ "$?" = "0" ]; then + echo "Pulled container images on attempt $attempt" + exit 0 + fi + echo "Failed to pull container images on attempt $attempt" + done + echo "Failed to pull container imagers after $attempt attempts" + exit 1 + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Service deploy run: | docker run -t --rm \ diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index ed3a7a2db0..9995a7d2c5 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -38,7 +38,12 @@ on: type: boolean required: false default: true - push-dirty: + sbom: + description: Generate SBOM? + type: boolean + required: false + default: true + push-critical: description: Push scanned images that have critical vulnerabilities? type: boolean required: false @@ -82,14 +87,14 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT" # Generate a tag to apply to all built container images. # Without this, each kayobe * container image build command would use a different tag. - name: Generate container datetime tag id: datetime_tag run: | - echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT" # Dynamically define job matrix. # We need a separate matrix entry for each distribution, when the relevant input is true. @@ -113,7 +118,7 @@ jobs: # remove trailing comma output="${output%,}" output+="]}" - echo "matrix=$output" >> $GITHUB_OUTPUT + echo "matrix=$output" | tee -a "$GITHUB_OUTPUT" - name: Display container datetime tag run: | @@ -187,7 +192,7 @@ jobs: - name: Get Kolla tag id: write-kolla-tag - run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" >> $GITHUB_OUTPUT + run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT" - name: Configure localhost as a seed run: | @@ -198,17 +203,6 @@ jobs: localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 EOF - # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details. - # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the - # runner and dind containers. - - name: Deploy an authenticating package repository mirror proxy - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} - - name: Create build logs output directory run: mkdir image-build-logs @@ -230,6 +224,9 @@ jobs: args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" args="$args -e kolla_build_log_path=$GITHUB_WORKSPACE/image-build-logs/kolla-build-overcloud.log" args="$args -e base_path=$GITHUB_WORKSPACE/opt/kayobe" + # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the + # runner and dind containers. + args="$args -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy" source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe overcloud container image build $args @@ -261,20 +258,21 @@ jobs: if: inputs.seed && matrix.distro.arch == 'amd64' - name: Get built container images - run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images" - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi - name: Scan built container images - run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom' }} - name: Move image scan logs to output artifact run: mv image-scan-output image-build-logs/image-scan-output + if: ${{ !cancelled() }} - - name: Fail if no images have passed scanning - run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi - if: ${{ !inputs.push-dirty }} + - name: Fail if any images have critical vulnerabilities + run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi + if: ${{ !inputs.push-critical }} - name: Copy clean images to push-attempt-images list run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt @@ -284,13 +282,13 @@ jobs: # This should be reverted when it's decided to filter high level CVEs as well. - name: Append dirty images to push list run: | - cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt if: ${{ inputs.push }} - name: Append images with critical vulnerabilities to push list run: | cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt - if: ${{ inputs.push && inputs.push-dirty }} + if: ${{ inputs.push && inputs.push-critical }} - name: Push images run: | @@ -332,19 +330,19 @@ jobs: if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} - name: Fail when images failed to push - run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi - if: ${{ !cancelled() }} + run: if [ -e image-build-logs/push-failed-images.txt ] && [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi + if: ${{ inputs.push && !cancelled() }} # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected. # This can be used again instead of "Fail when critical vulnerabilities are found" when it's # decided to fail the job on detecting high CVEs as well. # - name: Fail when images failed scanning - # run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi - # if: ${{ !inputs.push-dirty && !cancelled() }} + # run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi + # if: ${{ !inputs.push-critical && !cancelled() }} - name: Fail when critical vulnerabilities are found - run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi - if: ${{ !inputs.push-dirty && !cancelled() }} + run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi + if: ${{ !inputs.push-critical && !cancelled() }} - name: Remove locally built images for this run if: always() && runner.arch == 'ARM64' diff --git a/etc/kayobe/ansible/install-doca.yml b/etc/kayobe/ansible/install-doca.yml index 1a0fee8c37..fdf57a5de2 100644 --- a/etc/kayobe/ansible/install-doca.yml +++ b/etc/kayobe/ansible/install-doca.yml @@ -3,26 +3,50 @@ become: true hosts: mlnx gather_facts: true + vars: + # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built, + # but not 5.14.0-611.20.1 + doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] }}" tasks: - - name: Get running kernel - ansible.builtin.command: - cmd: "uname -r" - register: kernel - - name: Install kernel repo ansible.builtin.dnf: name: doca-kernel-repo state: latest update_cache: true + # not the same as doca_kernel_version: some dots changed to underscore or dash + - name: Discover kernel repo filename + ansible.builtin.shell: | + set -o pipefail + rpm -ql doca-kernel-repo | grep /etc/yum.repos.d/ + register: kernel_repo_filename + changed_when: false + - name: Ensure correct priority for DOCA modules ansible.builtin.lineinfile: line: "priority=-2" insertafter: EOF - path: "/etc/yum.repos.d/doca-kernel-{{ kernel.stdout }}.repo" + path: "{{ kernel_repo_filename.stdout }}" + + # This is required by mlnx-ofa_kernel, and comes from the doca repository. + # It is already present when doca-ofed is installed, but will be upgraded. + - name: Ensure mlnx-tools is installed + ansible.builtin.dnf: + name: mlnx-tools + state: latest + update_cache: true + become: true + + - name: Ensure appropriate doca-kernel is installed + ansible.builtin.dnf: + name: "doca-kernel-{{ doca_kernel_version }}" + state: latest + disablerepo: doca + become: true - - name: Install DOCA OFED + - name: Ensure DOCA OFED userspace is installed ansible.builtin.dnf: - name: doca-ofed + name: + - doca-ofed-userspace state: latest update_cache: true diff --git a/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml new file mode 100644 index 0000000000..063529bc3b --- /dev/null +++ b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml @@ -0,0 +1,104 @@ +--- +# This playbook performs the system upgrade from RockyLinux 9.6 to 9.7 on hosts +# using doca ofed kernel modules and utilities. +# It prevents the install of the dkms toolchain and dkms modules that would be +# used instead or our precompiled modules if a direct `dnf upgrade` was run. +# +# It must be run after new package snapshots have been merged and +# `pulp-repo-sync.yml` and `pulp-repo-publish.yml` have been run. +# Also `kayobe overcloud host configure -t dnf` must have been run for the new +# `doca.repo` to be present (the doca version is in the url) on the mlnx hosts. + +- name: Prepare upgrade from Rocky Linux 9.6 to 9.7 + hosts: mlnx + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}" + gather_facts: true + tags: pre + vars: + # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built, + # but not 5.14.0-611.20.1. + doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] }}" + tasks: + - name: Assert that hosts are running Rocky Linux 9.6 + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Rocky' + - ansible_facts.distribution_version == '9.6' + - os_distribution == 'rocky' + fail_msg: >- + This playbook is only designed for Rocky Linux 9.6 hosts. Ensure + that you are limiting it to only run on Rocky Linux 9.6 hosts and + os_distribution is set to rocky. + + - name: Ensure doca kernel repo is up to date + ansible.builtin.dnf: + name: doca-kernel-repo + state: latest + update_cache: true + become: true + + # This is required by mlnx-ofa_kernel, and comes from the doca repository. + # It is already present when doca-ofed is installed, but will be upgraded. + - name: Ensure mlnx-tools is installed + ansible.builtin.dnf: + name: mlnx-tools + state: latest + update_cache: true + become: true + + - name: Ensure appropriate doca-kernel is installed + ansible.builtin.dnf: + name: "doca-kernel-{{ doca_kernel_version }}" + state: latest + disablerepo: doca + become: true + + # doca-ofed 3.2 starts to depend on the dkms modules. It was not the case + # in doca-ofed 2.9.3. + - name: Ensure doca-ofed is not present (upgrading it brings dkms) + ansible.builtin.dnf: + name: doca-ofed + state: absent + autoremove: false + become: true + + - name: Ensure latest doca-ofed-userspace instead of doca-ofed + ansible.builtin.dnf: + name: doca-ofed-userspace + state: latest + become: true + + - name: Upgrade all + ansible.builtin.dnf: + name: "*" + state: latest + become: true + +- name: Fix potential grub config preventing new kernel from being used + ansible.builtin.import_playbook: reset-bls-entries.yml + vars: + reset_bls_hosts: mlnx + +- name: Reboot to apply updates + ansible.builtin.import_playbook: reboot.yml + vars: + reboot_hosts: mlnx + +- name: Confirm the host is upgraded to Rocky Linux 9.7 + hosts: mlnx + tags: post + tasks: + - name: Update distribution facts + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + + # Can fail (eg in multinode) when there are bad entries in grub config + # Fixed by `kayobe playbook run ansible/maintenance/reset-bls-entries.yml` + # and manual reboot + - name: Assert that hosts are now using Rocky Linux 9.7 + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Rocky' + - ansible_facts.distribution_version == '9.7' + - os_distribution == 'rocky' diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml index cdc94c2cd9..f13912ff80 100644 --- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml +++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml @@ -142,7 +142,7 @@ # Inclusive min sct_docker_version_min: "24.0.0" # Exclusive max - sct_docker_version_max: "28.1.0" + sct_docker_version_max: "30.0.0" sct_selinux_state: "{{ selinux_state }}" failed_when: host_results.rc not in [0, 1] register: host_results diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml index 26fac13464..85d3886d87 100644 --- a/etc/kayobe/docker.yml +++ b/etc/kayobe/docker.yml @@ -38,6 +38,15 @@ docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" # Enable live-restore on docker daemon docker_daemon_live_restore: true +# Avoid docker 29 for the moment in Caracal +docker_packages_version: "{{ '-28.*' if os_distribution == 'rocky' else ('=5:28*' if os_release == 'noble' else '=5:27*') }}" +# variable from https://github.com/stackhpc/ansible-role-docker/blob/master/defaults/main.yml +docker_packages: + - "docker-ce{{ docker_packages_version }}" + - "docker-ce-cli{{ docker_packages_version }}" + - "docker-ce-rootless-extras{{ docker_packages_version }}" + - "containerd.io" + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 0000000000..6e84a71044 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 0000000000..6e84a71044 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 0000000000..6e84a71044 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index 6f3a400548..5c00f815a7 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -38,6 +38,7 @@ kolla_build_neutron_ovs: true # Host and port of a package repository mirror. # Build against the development Pulp service repositories. # Use Ark's package repositories to install packages. +stackhpc_repo_mirror_auth_proxy_enabled: true stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "skc-ci-builder-reader" stackhpc_repo_mirror_password: !vault | diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 24a7b7663d..8885ff1bd4 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -4,91 +4,75 @@ # where the key is the OS distro and the value is the tag to deploy. kolla_image_tags: openstack: - rocky-9: 2024.1-rocky-9-20250716T041633 + rocky-9: 2024.1-rocky-9-20260223T075405 ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809 ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323 barbican: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 blazar: - rocky-9: 2024.1-rocky-9-20250627T102805 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 bifrost: ubuntu-jammy: 2024.1-ubuntu-jammy-20250325T141125 + # not buildable anymore, see https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a + rocky-9: 2024.1-rocky-9-20250716T041633 cinder: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 cloudkitty: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 designate: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 heat: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 glance: - rocky-9: 2024.1-rocky-9-20250903T130503 ubuntu-jammy: 2024.1-ubuntu-jammy-20250903T130503 ubuntu-noble: 2024.1-ubuntu-noble-20250903T130503 horizon: ubuntu-jammy: 2024.1-ubuntu-jammy-20250227T091118 ironic: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 ironic_dnsmasq: - rocky-9: 2024.1-rocky-9-20250716T041633 ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809 ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323 ironic_prometheus_exporter: ubuntu-jammy: 2024.1-ubuntu-jammy-20250124T081816 iscsid: - rocky-9: 2024.1-rocky-9-20251103T065408 ubuntu-jammy: 2024.1-ubuntu-jammy-20251103T065408 ubuntu-noble: 2024.1-ubuntu-noble-20251103T065408 keystone: - rocky-9: 2024.1-rocky-9-20251031T092724 ubuntu-jammy: 2024.1-ubuntu-jammy-20251031T092724 ubuntu-noble: 2024.1-ubuntu-noble-20251031T092724 kolla_toolbox: - rocky-9: 2024.1-rocky-9-20250908T155427 ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 magnum: - rocky-9: 2024.1-rocky-9-20250922T105039 ubuntu-jammy: 2024.1-ubuntu-jammy-20250922T105039 ubuntu-noble: 2024.1-ubuntu-noble-20250922T105039 manila: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 neutron: - rocky-9: 2024.1-rocky-9-20260128T142402 ubuntu-jammy: 2024.1-ubuntu-jammy-20260128T065158 ubuntu-noble: 2024.1-ubuntu-noble-20260128T065158 neutron_bgp_dragent: ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 nova: - rocky-9: 2024.1-rocky-9-20260220T123443 ubuntu-jammy: 2024.1-ubuntu-jammy-20260220T123443 ubuntu-noble: 2024.1-ubuntu-noble-20260220T123443 octavia: - rocky-9: 2024.1-rocky-9-20250717T094248 ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 openvswitch: ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 ovn: - rocky-9: 2024.1-rocky-9-20250925T130625 ubuntu-jammy: 2024.1-ubuntu-jammy-20250925T130625 ubuntu-noble: 2024.1-ubuntu-noble-20250925T130625 prometheus: @@ -96,10 +80,8 @@ kolla_image_tags: prometheus_alertmanager: ubuntu-jammy: 2024.1-ubuntu-jammy-20250422T103147 prometheus_libvirt_exporter: - rocky-9: 2024.1-rocky-9-20250819T151035 ubuntu-jammy: 2024.1-ubuntu-jammy-20250819T151035 ubuntu-noble: 2024.1-ubuntu-noble-20250819T151035 rabbitmq: - rocky-9: 2024.1-rocky-9-20250908T155427 ubuntu-jammy: 2024.1-ubuntu-jammy-20250722T083943 ubuntu-noble: 2024.1-ubuntu-noble-20250722T083943 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 9702f7d718..370ec19581 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}" # Path in which to generate kolla configuration. #kolla_build_config_path: +# View debug logs, like source downloads, during image build. +kolla_build_extra_config: | + [DEFAULT] + debug = True + ############################################################################### # Kolla-ansible installation. diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index 95692e5ef4..f8daae9ebe 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -18,9 +18,18 @@ location = https://github.com/stackhpc/requirements reference = stackhpc/{{ openstack_release }} [etcd] -version = 3.5.21 -sha256 = amd64:adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c,arm64:95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f +version = 3.5.27 +sha256 = amd64:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc,arm64:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa [letsencrypt-lego] version = v4.23.1 sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,arm64:1114745108343286d4bff189b4bdee3cba9d07ebcacc673860d91ab951d31e0d + +# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available +[prometheus-cadvisor] +version = 0.56.2 +sha256 = amd64:ad92930f16a2f9da15190675e09eeaceb8fd38637d07a686bb0dd68695f692af,arm64:b7a707379496fd7a7b5d2768c5c494427112f534ba5069f889af28ffe6ad11bb + +[prometheus-msteams] +version = 1.5.3 +sha256 = amd64:8eae63d89338f53a990fa2720b3fabf58c916e2648c948ce1e0f29942459a491 diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml index 7867206f64..6c58b5db96 100644 --- a/etc/kayobe/ofed.yml +++ b/etc/kayobe/ofed.yml @@ -3,7 +3,15 @@ ############################################################################### # DOCA host version -stackhpc_pulp_doca_version: "{{ '2.9.3' if stackhpc_pulp_repo_rocky_9_minor_version == '6' else '2.9.1' }}" +stackhpc_pulp_doca_version_matrix: + "6": 2.9.3 + "7": 3.2.1 +stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] | default('2.9.1') }}" + +# Available and tested versions of the pre-compiled doca-ofed kernel modules +stackhpc_doca_kernel_version_matrix: + "6": 5.14.0.570.21.1.el9.6.x86.64 + "7": 5.14.0.611.13.1.el9.7.x86.64 ############################################################################### # Pulp configuration for DOCA OFED @@ -12,7 +20,9 @@ stackhpc_pulp_doca_version: "{{ '2.9.3' if stackhpc_pulp_repo_rocky_9_minor_vers stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}" # DOCA Snapshot lookup vars -doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_version" +doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}9{% else %}9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}" +# eg stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version or stackhpc_pulp_repo_doca_3_2_1_rhel9_version +doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version" doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version" # DOCA Snapshot versions. The defaults use the appropriate version from diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index db0316a37d..b7e13a44da 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,6 +1,6 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_rocky_9_overcloud_host_image_version: "2024.1-20241209T151515" +stackhpc_rocky_9_overcloud_host_image_version: "2024.1-20260205T145550" stackhpc_ubuntu_jammy_overcloud_host_image_version: "2024.1-20250408T082408" stackhpc_ubuntu_noble_overcloud_host_image_version: "2024.1-20250417T104113" diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 7ade702c41..537b366745 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -1,36 +1,44 @@ --- # This file is autogenerated by Ansible using the following workflow: # https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml -stackhpc_pulp_repo_centos_stream_9_docker_version: 20241210T000909 -stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20241210T012225 -stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20250528T022338 -stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20250528T015409 -stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20241212T022636 -stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20241212T020643 +stackhpc_pulp_repo_centos_stream_9_docker_version: 20260202T201529 +stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20260202T213000 +stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20260202T211958 +stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20260202T213000 +stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20250603T020143 +stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20250603T013132 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318 stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838 -stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20240923T233036 -stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20240927T073838 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20250617T023108 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20250617T020101 +stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620 +stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245 +stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301 +stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314 +stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021 +stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841 +stackhpc_pulp_repo_doca_3_2_1_rhel9_version: 20251219T132023 +stackhpc_pulp_repo_doca_3_2_1_rhel9_7_modules_version: 20251222T160623 stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022 stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20241218T154614 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20250401T001425 -stackhpc_pulp_repo_elrepo_9_version: 20250610T235426 +stackhpc_pulp_repo_elrepo_9_version: 20260127T212055 stackhpc_pulp_repo_elrepo_9_aarch64_version: 20250408T030629 -stackhpc_pulp_repo_epel_9_version: 20250615T000221 -stackhpc_pulp_repo_epel_9_aarch64_version: 20250615T015805 -stackhpc_pulp_repo_grafana_version: 20250615T005738 -stackhpc_pulp_repo_opensearch_2_x_version: 20250430T014638 -stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20250430T014638 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20250718T004955 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20250905T114106 -stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20250905T114106 -stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20250708T003546 -stackhpc_pulp_repo_rhel_9_influxdb_version: 20241217T002152 -stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20241217T012754 -stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20241102T004913 -stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20241102T015940 -stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20241115T002028 -stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20241115T010217 +stackhpc_pulp_repo_epel_9_version: 20260204T220346 +stackhpc_pulp_repo_epel_9_aarch64_version: 20260204T223146 +stackhpc_pulp_repo_grafana_version: 20260214T213531 +stackhpc_pulp_repo_opensearch_2_x_version: 20251106T202313 +stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20251106T202313 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20251130T213625 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20260122T202504 +stackhpc_pulp_repo_rhel_9_influxdb_version: 20260113T202459 +stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20260204T213517 +stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20260204T223146 +stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20251219T202129 +stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20251219T213344 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328 @@ -51,36 +59,46 @@ stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610 stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610 stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610 stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610 -stackhpc_pulp_repo_rocky_9_5_appstream_version: 20241217T005008 -stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20241217T012754 -stackhpc_pulp_repo_rocky_9_5_baseos_version: 20241216T013503 -stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20241216T011954 -stackhpc_pulp_repo_rocky_9_5_crb_version: 20241217T005008 -stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20241217T012754 -stackhpc_pulp_repo_rocky_9_5_extras_version: 20241216T004230 -stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20241216T011954 -stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20241202T003154 -stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20241203T010516 -stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20241127T003858 -stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20241127T011228 +stackhpc_pulp_repo_rocky_9_5_appstream_version: 20250202T120931 +stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20250203T124809 +stackhpc_pulp_repo_rocky_9_5_baseos_version: 20250201T125442 +stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20250201T123411 +stackhpc_pulp_repo_rocky_9_5_crb_version: 20250204T095037 +stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20250206T042027 +stackhpc_pulp_repo_rocky_9_5_extras_version: 20250122T025402 +stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20250124T030113 +stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20250204T095037 +stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20250206T042027 stackhpc_pulp_repo_rocky_9_6_appstream_version: 20250614T015933 stackhpc_pulp_repo_rocky_9_6_appstream_aarch64_version: 20250614T024523 stackhpc_pulp_repo_rocky_9_6_baseos_version: 20250614T030644 stackhpc_pulp_repo_rocky_9_6_baseos_aarch64_version: 20250614T024523 stackhpc_pulp_repo_rocky_9_6_crb_version: 20250614T015933 stackhpc_pulp_repo_rocky_9_6_crb_aarch64_version: 20250614T024523 -stackhpc_pulp_repo_rocky_9_6_extras_version: 20250605T150141 -stackhpc_pulp_repo_rocky_9_6_extras_aarch64_version: 20250605T150141 +stackhpc_pulp_repo_rocky_9_6_extras_version: 20250726T040613 +stackhpc_pulp_repo_rocky_9_6_extras_aarch64_version: 20250726T045704 stackhpc_pulp_repo_rocky_9_6_highavailability_version: 20250605T150141 stackhpc_pulp_repo_rocky_9_6_highavailability_aarch64_version: 20250605T150141 +stackhpc_pulp_repo_rocky_9_7_appstream_version: 20260130T205631 +stackhpc_pulp_repo_rocky_9_7_appstream_aarch64_version: 20260130T210847 +stackhpc_pulp_repo_rocky_9_7_appstream_source_version: 20260130T211649 +stackhpc_pulp_repo_rocky_9_7_baseos_version: 20260130T212743 +stackhpc_pulp_repo_rocky_9_7_baseos_aarch64_version: 20260130T210847 +stackhpc_pulp_repo_rocky_9_7_baseos_source_version: 20260130T211649 +stackhpc_pulp_repo_rocky_9_7_crb_version: 20260130T205631 +stackhpc_pulp_repo_rocky_9_7_crb_aarch64_version: 20260130T210847 +stackhpc_pulp_repo_rocky_9_7_crb_source_version: 20260130T211649 +stackhpc_pulp_repo_rocky_9_7_extras_version: 20251229T144024 +stackhpc_pulp_repo_rocky_9_7_extras_aarch64_version: 20251229T153210 +stackhpc_pulp_repo_rocky_9_7_extras_source_version: 20251229T150242 +stackhpc_pulp_repo_rocky_9_7_highavailability_version: 20260108T204618 +stackhpc_pulp_repo_rocky_9_7_highavailability_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rocky_9_7_highavailability_source_version: 20251220T210727 +stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20251108T211425 +stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20251109T211404 +stackhpc_pulp_repo_rocky_9_sig_security_common_source_version: 20251203T103807 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20250416T042645 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20250417T070229 stackhpc_pulp_repo_ubuntu_jammy_version: 20250417T070229 stackhpc_pulp_repo_ubuntu_noble_security_version: 20250417T033501 stackhpc_pulp_repo_ubuntu_noble_version: 20250417T033501 -stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620 -stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245 -stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301 -stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314 -stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021 -stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 478aa275ee..0ab2694ab7 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -242,8 +242,8 @@ stackhpc_pulp_distribution_deb_production: >- # Whether to sync Rocky Linux 9 packages. stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}" -# Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6. Default is 6 -stackhpc_pulp_repo_rocky_9_minor_version: '6' +# Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6, 7. Default is 7 +stackhpc_pulp_repo_rocky_9_minor_version: '7' # Rocky 9 Snapshot versions. The defaults use the appropriate version from # pulp-repo-versions.yml for the selected minor release. @@ -424,10 +424,10 @@ stackhpc_pulp_rpm_repos: base_path: "opensearch-dashboards/2.x/yum/" required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" - - name: DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} - url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_version }}" - distribution_name: "doca-{{ stackhpc_pulp_doca_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-" - base_path: "doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/" + - name: DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL {{ doca_rocky_9_minor_dot }} + url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_version }}" + distribution_name: "doca-{{ stackhpc_pulp_doca_version }}-rhel{{ doca_rocky_9_minor_dot }}-" + base_path: "doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/" required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}" - name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_version }} - RHEL 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} @@ -611,9 +611,13 @@ stackhpc_pulp_images_kolla: # List of images for each base distribution which should not/cannot be built. stackhpc_kolla_unbuildable_images: - ubuntu-jammy: [] - ubuntu-noble: [] - rocky-9: [] + # See https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a + ubuntu-jammy: + - bifrost-deploy + ubuntu-noble: + - bifrost-deploy + rocky-9: + - bifrost-deploy # Whitespace-separated list of regular expressions matching Kolla image names. # Usage is similar to kolla-build CLI arguments. diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 44e295fca4..c3db6f1309 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -69,7 +69,7 @@ stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_repo_distribution }}" # RPMs # DOCA -stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}" +stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}" stackhpc_repo_rhel9_doca_version: "{{ stackhpc_repo_distribution }}" # DOCA Modules diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml index 579ed31941..26ee5977dd 100644 --- a/etc/kayobe/trivy/allowed-vulnerabilities.yml +++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml @@ -16,31 +16,82 @@ fluentd_allowed_vulnerabilities: - CVE-2024-27280 grafana_allowed_vulnerabilities: - CVE-2024-8986 + - CVE-2025-68121 # the opensearch datasource plugin is still vulnerable influxdb_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 +ironic_neutron_agent_allowed_vulnerabilities: + - CVE-2025-68121 +letsencrypt_lego_allowed_vulnerabilities: + - CVE-2025-68121 magnum_conductor_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 +neutron_base_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_bgp_dragent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_dhcp_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_l3_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_linuxbridge_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_metadata_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_mlnx_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_openvswitch_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_ovn_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_server_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_sriov_agent_allowed_vulnerabilities: + - CVE-2025-68121 +opensearch_dashboards_allowed_vulnerabilities: + - CVE-2025-68428 +prometheus_alertmanager_allowed_vulnerabilities: + - CVE-2025-68121 prometheus_blackbox_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 + - CVE-2025-68121 prometheus_memcached_exporter_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 prometheus_mysqld_exporter_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 prometheus_elasticsearch_exporter_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 prometheus_node_exporter_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 prometheus_openstack_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 + - CVE-2025-68121 +prometheus_ovn_exporter_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2025-68121 prometheus_libvirt_exporter_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 prometheus_cadvisor_allowed_vulnerabilities: - CVE-2024-41110 - CVE-2024-45337 + - CVE-2025-68121 prometheus_msteams_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 +prometheus_mtail_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2025-68121 prometheus_v2_server_allowed_vulnerabilities: - CVE-2024-45337 + - CVE-2025-68121 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml new file mode 100644 index 0000000000..6b6fa315fa --- /dev/null +++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Added support for Rocky Linux 9.7, including host packages and a full + container image refresh. + - | + 9.7 is now the default release for Rocky Linux. + +upgrade: + - | + The upgrade to DOCA 3.2.1 is required for Rocky Linux 9.7. diff --git a/tools/scan-images.sh b/tools/scan-images.sh index e313cbc41a..14da5a53aa 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -1,104 +1,213 @@ #!/usr/bin/env bash set -eo pipefail -# Check correct usage -if [[ ! $2 ]]; then - echo "Usage: scan-images.sh " +# Disable telemetry and version check: +# https://github.com/aquasecurity/trivy/discussions/8945 +export TRIVY_DISABLE_TELEMETRY=true +export TRIVY_SKIP_VERSION_CHECK=true + +# Global variables +scan_common_args=" \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --db-repository ghcr.io/aquasecurity/trivy-db:2 \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ + --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db " + +# Print usage instructions and error with wrong inputs +usage() { + echo "Usage: scan-images.sh [--sbom]" exit 2 -fi - -set -u - -# Check that trivy is installed -if ! trivy --version; then - echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.68.2' -fi - -# Clear any previous outputs -rm -rf image-scan-output - -# Make a fresh output directory -mkdir -p image-scan-output - -# Get built container images -images=$(docker image ls \ - --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" \ - --format "{{.Repository}}:{{.Tag}}") - -# Save list of images to file -echo "$images" > "$1-scanned-container-images.txt" - -# Ensure output files exist -touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt - -# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. -# If there are vulnerabilities detected, add it to dirty-images.txt and -# generate a csv summary -# If the image contains at least one critical vulnerabilities, add it to -# critical-images.txt -for image in $images; do - filename=$(basename $image | sed 's/:/\./g') - imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') - global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) - image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml) - touch .trivyignore +} + +# Check dependencies are installed, print installation instructions otherwise +check_deps_installed() { + if ! trivy --version > /dev/null; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.68.2' + exit 1 + fi + if ! yq --version > /dev/null; then + echo 'Please install yq: sudo dnf/apt install yq' + exit 1 + fi +} + +# Prepare output files +file_prep() { + rm -rf image-scan-output + mkdir -p image-scan-output + touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt +} + +# Gather image lists +get_images() { + local output_file="$1-scanned-container-images.txt" + + docker image ls \ + --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" \ + --format "{{.Repository}}:{{.Tag}}" \ + > "$output_file" + + cat "$output_file" +} + +# Generate ignored vulnerabilities file +generate_trivy_ignore() { + local imagename=$1 + local global_vulnerabilities + global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities + image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + + truncate -s 0 .trivyignore # ensure we start from a clean slate for vulnerability in $global_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done for vulnerability in $image_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done - if $(trivy image \ - --quiet \ - --exit-code 1 \ - --scanners vuln \ - --format json \ - --severity HIGH,CRITICAL \ - --output image-scan-output/${filename}.json \ - --ignore-unfixed \ - --db-repository ghcr.io/aquasecurity/trivy-db:2 \ - --db-repository public.ecr.aws/aquasecurity/trivy-db \ - --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ - --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \ - $image); then - # Clean up the output file for any images with no vulnerabilities - rm -f image-scan-output/${filename}.json - - # Add the image to the clean list +} + +# Put results into CSV +generate_summary_csv() { + local scan="$1" + local summary="$2" + + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary" + + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' "$scan" >> "$summary" +} + +# Categorise images based on severity +categorise_image() { + local summary="$1" + local image="$2" + + if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then + echo "${image}" >> image-scan-output/critical-images.txt + else + echo "${image}" >> image-scan-output/high-images.txt + fi +} + +# Generate SBOM, return correct scan command for SBOM +generate_sbom() { + local sbom="$1" + local scan="$2" + local image="$3" + trivy image \ + --debug \ + --format spdx-json \ + --output "$sbom" \ + "$image" &> "$sbom.log" + if [ ! -e "$sbom" ]; then + ( + echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2 + echo "==== trivy log ====" + cat "$sbom.log" + ) 1>&2 + exit 1 + elif grep -q FATAL "$sbom.log"; then + ( + echo "ERROR: trivy image encountered a fatal error producing $sbom for $image" + echo "==== trivy log ====" + cat "$sbom.log" + echo "==== sbom.json ====" + cat "$sbom" + ) 1>&2 + exit 1 + else + echo "trivy sbom $scan_common_args --output $scan $sbom" + fi +} + +# Scan images, generate SBOMs if requested +scan_image() { + local image=$1 + local filename + filename=$(basename "$image" | sed 's/:/\./g') + local imagename + imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g') + local sbom="image-scan-output/${imagename}/${filename}-sbom.json" + local scan="image-scan-output/${imagename}/${filename}-scan.json" + local summary="image-scan-output/${imagename}/${filename}-summary.csv" + + mkdir -p "image-scan-output/$imagename" + generate_trivy_ignore "$imagename" + + # If SBOM is required, generate it first and scan the results, otherwise we + # scan the image directly. + if $generate_sbom; then + echo "Generating SBOM for $imagename" + scan_command="$(generate_sbom "$sbom" "$scan" "$image")" + else + scan_command="trivy image $scan_common_args --output $scan $image" + fi + + # Run scan against image or SBOM, format output. If no results, delete files. + echo "Scanning $imagename for vulnerabilities" + if $scan_command >& "$scan.log"; then + rm -f "$scan" echo "${image}" >> image-scan-output/clean-images.txt + elif [ ! -f "$scan" ]; then + ( + echo "ERROR: trivy scan encountered an error producing $scan" + echo "Command: $scan_command" + echo "==== trivy log ====" + cat "$scan.log" + if $generate_sbom; then + echo "==== sbom.json ====" + cat "$sbom" + fi + ) 1>&2 + exit 1 else + generate_summary_csv "$scan" "$summary" + categorise_image "$summary" "$image" + fi +} + +# Main function +main() { + if [[ ! $2 ]]; then + usage + fi - # Write a header for the summary CSV - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv - - # Write the summary CSV data - jq -r '.Results[] - | select(.Vulnerabilities) - | .Vulnerabilities - # Ignore packages with "kernel" in the PkgName - | map(select(.PkgName | test("kernel") | not )) - | group_by(.VulnerabilityID) - | map( - [ - (map(.PkgName) | unique | join(";")), - (map(.PkgPath | select( . != null )) | join(";")), - .[0].PkgID, - .[0].VulnerabilityID, - .[0].FixedVersion, - .[0].PrimaryURL, - .[0].Severity - ] - ) - | .[] - | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv - - if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then - # If the image contains critical vulnerabilities, add the image to critical list - echo "${image}" >> image-scan-output/critical-images.txt - else - # Otherwise, add the image to the dirty list - echo "${image}" >> image-scan-output/dirty-images.txt - fi + generate_sbom=false + if [[ "$3" == "--sbom" ]]; then + generate_sbom=true fi - rm .trivyignore -done + + set -u + + check_deps_installed + file_prep + + images=$(get_images "$1" "$2") + for image in $images; do + scan_image "$image" + done +} + +main "$@"