From c92b91aa92642396ab85864a156ad680e8145e58 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Tue, 3 Feb 2026 09:26:37 +0100
Subject: [PATCH 01/19] Switch to Rocky Linux 9.7 and DOCA 3.2.1

- DOCA 3.2.1 for RL 9.7
- Bump Rocky 9 Security SIG repo, add source
---
 etc/kayobe/ofed.yml                           |   9 +-
 etc/kayobe/pulp-repo-versions.yml             | 106 ++++++++++--------
 etc/kayobe/pulp.yml                           |   4 +-
 etc/kayobe/stackhpc.yml                       |   2 +-
 .../notes/rl9.7-c8d033f11097b76b.yaml         |  10 ++
 5 files changed, 82 insertions(+), 49 deletions(-)
 create mode 100644 releasenotes/notes/rl9.7-c8d033f11097b76b.yaml

diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 7867206f64..f37e793d02 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -3,7 +3,10 @@
 
 ###############################################################################
 # DOCA host version
-stackhpc_pulp_doca_version: "{{ '2.9.3' if stackhpc_pulp_repo_rocky_9_minor_version == '6' else '2.9.1' }}"
+stackhpc_pulp_doca_version_matrix:
+    "6": 2.9.3
+    "7": 3.2.1
+stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] | default('2.9.1') }}"
 
 ###############################################################################
 # Pulp configuration for DOCA OFED
@@ -12,7 +15,9 @@ stackhpc_pulp_doca_version: "{{ '2.9.3' if stackhpc_pulp_repo_rocky_9_minor_vers
 stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}"
 
 # DOCA Snapshot lookup vars
-doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_version"
+doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}rhel9{% else %}rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}"
+# eg stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version or stackhpc_pulp_repo_doca_3_2_1_rhel9_version
+doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version"
 doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version"
 
 # DOCA Snapshot versions. The defaults use the appropriate version from
diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml
index 7ade702c41..dd5f7dbab3 100644
--- a/etc/kayobe/pulp-repo-versions.yml
+++ b/etc/kayobe/pulp-repo-versions.yml
@@ -1,36 +1,44 @@
 ---
 # This file is autogenerated by Ansible using the following workflow:
 # https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml
-stackhpc_pulp_repo_centos_stream_9_docker_version: 20241210T000909
-stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20241210T012225
-stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20250528T022338
-stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20250528T015409
-stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20241212T022636
-stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20241212T020643
+stackhpc_pulp_repo_centos_stream_9_docker_version: 20260202T201529
+stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20260202T213000
+stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20260202T211958
+stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20260202T213000
+stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20250603T020143
+stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20250603T013132
 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318
 stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838
-stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20240923T233036
-stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20240927T073838
+stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20250617T023108
+stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20250617T020101
+stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620
+stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245
+stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301
+stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314
+stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021
+stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841
+stackhpc_pulp_repo_doca_3_2_1_rhel9_version: 20251219T132023
+stackhpc_pulp_repo_doca_3_2_1_rhel9_7_modules_version: 20251222T160623
 stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022
 stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20241218T154614
 stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20250401T001425
-stackhpc_pulp_repo_elrepo_9_version: 20250610T235426
+stackhpc_pulp_repo_elrepo_9_version: 20260127T212055
 stackhpc_pulp_repo_elrepo_9_aarch64_version: 20250408T030629
-stackhpc_pulp_repo_epel_9_version: 20250615T000221
-stackhpc_pulp_repo_epel_9_aarch64_version: 20250615T015805
-stackhpc_pulp_repo_grafana_version: 20250615T005738
-stackhpc_pulp_repo_opensearch_2_x_version: 20250430T014638
-stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20250430T014638
-stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20250718T004955
-stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20250905T114106
-stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20250905T114106
-stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20250708T003546
-stackhpc_pulp_repo_rhel_9_influxdb_version: 20241217T002152
-stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20241217T012754
-stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20241102T004913
-stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20241102T015940
-stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20241115T002028
-stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20241115T010217
+stackhpc_pulp_repo_epel_9_version: 20260204T220346
+stackhpc_pulp_repo_epel_9_aarch64_version: 20260204T223146
+stackhpc_pulp_repo_grafana_version: 20260204T212232
+stackhpc_pulp_repo_opensearch_2_x_version: 20251106T202313
+stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20251106T202313
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20251130T213625
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20260112T224827
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20260112T224827
+stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20260122T202504
+stackhpc_pulp_repo_rhel_9_influxdb_version: 20260113T202459
+stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20260112T224827
+stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20260204T213517
+stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20260204T223146
+stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20251219T202129
+stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20251219T213344
 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715
 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015
 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328
@@ -51,36 +59,46 @@ stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610
 stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610
-stackhpc_pulp_repo_rocky_9_5_appstream_version: 20241217T005008
-stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20241217T012754
-stackhpc_pulp_repo_rocky_9_5_baseos_version: 20241216T013503
-stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20241216T011954
-stackhpc_pulp_repo_rocky_9_5_crb_version: 20241217T005008
-stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20241217T012754
-stackhpc_pulp_repo_rocky_9_5_extras_version: 20241216T004230
-stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20241216T011954
-stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20241202T003154
-stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20241203T010516
-stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20241127T003858
-stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20241127T011228
+stackhpc_pulp_repo_rocky_9_5_appstream_version: 20250202T120931
+stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20250203T124809
+stackhpc_pulp_repo_rocky_9_5_baseos_version: 20250201T125442
+stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20250201T123411
+stackhpc_pulp_repo_rocky_9_5_crb_version: 20250204T095037
+stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20250206T042027
+stackhpc_pulp_repo_rocky_9_5_extras_version: 20250122T025402
+stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20250124T030113
+stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20250204T095037
+stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20250206T042027
 stackhpc_pulp_repo_rocky_9_6_appstream_version: 20250614T015933
 stackhpc_pulp_repo_rocky_9_6_appstream_aarch64_version: 20250614T024523
 stackhpc_pulp_repo_rocky_9_6_baseos_version: 20250614T030644
 stackhpc_pulp_repo_rocky_9_6_baseos_aarch64_version: 20250614T024523
 stackhpc_pulp_repo_rocky_9_6_crb_version: 20250614T015933
 stackhpc_pulp_repo_rocky_9_6_crb_aarch64_version: 20250614T024523
-stackhpc_pulp_repo_rocky_9_6_extras_version: 20250605T150141
-stackhpc_pulp_repo_rocky_9_6_extras_aarch64_version: 20250605T150141
+stackhpc_pulp_repo_rocky_9_6_extras_version: 20250726T040613
+stackhpc_pulp_repo_rocky_9_6_extras_aarch64_version: 20250726T045704
 stackhpc_pulp_repo_rocky_9_6_highavailability_version: 20250605T150141
 stackhpc_pulp_repo_rocky_9_6_highavailability_aarch64_version: 20250605T150141
+stackhpc_pulp_repo_rocky_9_7_appstream_version: 20260130T205631
+stackhpc_pulp_repo_rocky_9_7_appstream_aarch64_version: 20260130T210847
+stackhpc_pulp_repo_rocky_9_7_appstream_source_version: 20260130T211649
+stackhpc_pulp_repo_rocky_9_7_baseos_version: 20260130T212743
+stackhpc_pulp_repo_rocky_9_7_baseos_aarch64_version: 20260130T210847
+stackhpc_pulp_repo_rocky_9_7_baseos_source_version: 20260130T211649
+stackhpc_pulp_repo_rocky_9_7_crb_version: 20260130T205631
+stackhpc_pulp_repo_rocky_9_7_crb_aarch64_version: 20260130T210847
+stackhpc_pulp_repo_rocky_9_7_crb_source_version: 20260130T211649
+stackhpc_pulp_repo_rocky_9_7_extras_version: 20251229T144024
+stackhpc_pulp_repo_rocky_9_7_extras_aarch64_version: 20251229T153210
+stackhpc_pulp_repo_rocky_9_7_extras_source_version: 20251229T150242
+stackhpc_pulp_repo_rocky_9_7_highavailability_version: 20260108T204618
+stackhpc_pulp_repo_rocky_9_7_highavailability_aarch64_version: 20260112T224827
+stackhpc_pulp_repo_rocky_9_7_highavailability_source_version: 20251220T210727
+stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20251108T211425
+stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20251109T211404
+stackhpc_pulp_repo_rocky_9_sig_security_common_source_version: 20251203T103807
 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20250416T042645
 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20250417T070229
 stackhpc_pulp_repo_ubuntu_jammy_version: 20250417T070229
 stackhpc_pulp_repo_ubuntu_noble_security_version: 20250417T033501
 stackhpc_pulp_repo_ubuntu_noble_version: 20250417T033501
-stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620
-stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245
-stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301
-stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314
-stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021
-stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 478aa275ee..d5711fb83e 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -242,8 +242,8 @@ stackhpc_pulp_distribution_deb_production: >-
 
 # Whether to sync Rocky Linux 9 packages.
 stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
-# Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6. Default is 6
-stackhpc_pulp_repo_rocky_9_minor_version: '6'
+# Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6, 7. Default is 7
+stackhpc_pulp_repo_rocky_9_minor_version: '7'
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
 
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 44e295fca4..5ddb9f0e59 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -69,7 +69,7 @@ stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_repo_distribution }}"
 # RPMs
 
 # DOCA
-stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}"
+stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}"
 stackhpc_repo_rhel9_doca_version: "{{ stackhpc_repo_distribution }}"
 
 # DOCA Modules
diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml
new file mode 100644
index 0000000000..dcdb765ec7
--- /dev/null
+++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Added support for Rocky Linux 9.7.
+  - |
+    9.7 is now the default release for Rocky Linux.
+
+upgrade:
+  - |
+    The upgrade to DOCA 3.2.1 is required for Rocky Linux 9.7.

From ba592188a0d2e81f53732c17fd9c81d660a9774d Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Thu, 8 Jan 2026 10:27:21 +0100
Subject: [PATCH 02/19] [skip ci] Fix DOCA url for local pulp servers

removes the RockyLinux minor version in the name and path
when DOCA version is greater than 3.2.0.
Doesn't apply to DOCA modules because they are still compiled
for a specific RL minor version.
---
 etc/kayobe/ofed.yml     | 4 ++--
 etc/kayobe/pulp.yml     | 8 ++++----
 etc/kayobe/stackhpc.yml | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index f37e793d02..21c92d3152 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -15,9 +15,9 @@ stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_
 stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}"
 
 # DOCA Snapshot lookup vars
-doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}rhel9{% else %}rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}"
+doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}9{% else %}9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}"
 # eg stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version or stackhpc_pulp_repo_doca_3_2_1_rhel9_version
-doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version"
+doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version"
 doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version"
 
 # DOCA Snapshot versions. The defaults use the appropriate version from
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index d5711fb83e..07ecc37942 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -424,10 +424,10 @@ stackhpc_pulp_rpm_repos:
     base_path: "opensearch-dashboards/2.x/yum/"
     required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
-  - name: DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
-    url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_version }}"
-    distribution_name: "doca-{{ stackhpc_pulp_doca_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
-    base_path: "doca/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
+  - name: DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL {{ doca_rocky_9_minor_dot }}
+    url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_version }}"
+    distribution_name: "doca-{{ stackhpc_pulp_doca_version }}-rhel{{ doca_rocky_9_minor_dot }}-"
+    base_path: "doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/"
     required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
   - name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_version }} - RHEL 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 5ddb9f0e59..c3db6f1309 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -69,7 +69,7 @@ stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_repo_distribution }}"
 # RPMs
 
 # DOCA
-stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}"
+stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}"
 stackhpc_repo_rhel9_doca_version: "{{ stackhpc_repo_distribution }}"
 
 # DOCA Modules

From a700d562f43fd6955224c1fa03a902b6efee400c Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Fri, 16 Jan 2026 09:58:47 +0100
Subject: [PATCH 03/19] stackhpc-cloud-tests: bump docker max version

Latest version for RockyLinux is 29.2
---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index cdc94c2cd9..f13912ff80 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -142,7 +142,7 @@
             # Inclusive min
             sct_docker_version_min: "24.0.0"
             # Exclusive max
-            sct_docker_version_max: "28.1.0"
+            sct_docker_version_max: "30.0.0"
             sct_selinux_state: "{{ selinux_state }}"
           failed_when: host_results.rc not in [0, 1]
           register: host_results

From c33337370304f32a3b5e7b15f3878e5a85f30388 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Fri, 16 Jan 2026 18:29:50 +0100
Subject: [PATCH 04/19] Add playbook to automate Rocky 9.7 with OFED upgrade

Tested on multinode.

Fix install-doca.yml to not install doca-ofed anymore (avoid dkms).

The stackhpc_doca_kernel_version_matrix variable contains kernel module
versions to install for last 2 supported minor RockyLinux versions.
It must be changed after a new pre-compiled kernel module version has been built.
---
 etc/kayobe/ansible/install-doca.yml          |  40 +++++--
 etc/kayobe/ansible/rocky-97-ofed-upgrade.yml | 104 +++++++++++++++++++
 etc/kayobe/ofed.yml                          |   5 +
 3 files changed, 141 insertions(+), 8 deletions(-)
 create mode 100644 etc/kayobe/ansible/rocky-97-ofed-upgrade.yml

diff --git a/etc/kayobe/ansible/install-doca.yml b/etc/kayobe/ansible/install-doca.yml
index 1a0fee8c37..fdf57a5de2 100644
--- a/etc/kayobe/ansible/install-doca.yml
+++ b/etc/kayobe/ansible/install-doca.yml
@@ -3,26 +3,50 @@
   become: true
   hosts: mlnx
   gather_facts: true
+  vars:
+    # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
+    # but not 5.14.0-611.20.1
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] }}"
   tasks:
-    - name: Get running kernel
-      ansible.builtin.command:
-        cmd: "uname -r"
-      register: kernel
-
     - name: Install kernel repo
       ansible.builtin.dnf:
         name: doca-kernel-repo
         state: latest
         update_cache: true
 
+    # not the same as doca_kernel_version: some dots changed to underscore or dash
+    - name: Discover kernel repo filename
+      ansible.builtin.shell: |
+        set -o pipefail
+        rpm -ql doca-kernel-repo | grep /etc/yum.repos.d/
+      register: kernel_repo_filename
+      changed_when: false
+
     - name: Ensure correct priority for DOCA modules
       ansible.builtin.lineinfile:
         line: "priority=-2"
         insertafter: EOF
-        path: "/etc/yum.repos.d/doca-kernel-{{ kernel.stdout }}.repo"
+        path: "{{ kernel_repo_filename.stdout }}"
+
+    # This is required by mlnx-ofa_kernel, and comes from the doca repository.
+    # It is already present when doca-ofed is installed, but will be upgraded.
+    - name: Ensure mlnx-tools is installed
+      ansible.builtin.dnf:
+        name: mlnx-tools
+        state: latest
+        update_cache: true
+      become: true
+
+    - name: Ensure appropriate doca-kernel is installed
+      ansible.builtin.dnf:
+        name: "doca-kernel-{{ doca_kernel_version }}"
+        state: latest
+        disablerepo: doca
+      become: true
 
-    - name: Install DOCA OFED
+    - name: Ensure DOCA OFED userspace is installed
       ansible.builtin.dnf:
-        name: doca-ofed
+        name:
+          - doca-ofed-userspace
         state: latest
         update_cache: true
diff --git a/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml
new file mode 100644
index 0000000000..063529bc3b
--- /dev/null
+++ b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml
@@ -0,0 +1,104 @@
+---
+# This playbook performs the system upgrade from RockyLinux 9.6 to 9.7 on hosts
+# using doca ofed kernel modules and utilities.
+# It prevents the install of the dkms toolchain and dkms modules that would be
+# used instead or our precompiled modules if a direct `dnf upgrade` was run.
+#
+# It must be run after new package snapshots have been merged and
+# `pulp-repo-sync.yml` and `pulp-repo-publish.yml` have been run.
+# Also `kayobe overcloud host configure -t dnf` must have been run for the new
+# `doca.repo` to be present (the doca version is in the url) on the mlnx hosts.
+
+- name: Prepare upgrade from Rocky Linux 9.6 to 9.7
+  hosts: mlnx
+  serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}"
+  gather_facts: true
+  tags: pre
+  vars:
+    # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
+    # but not 5.14.0-611.20.1.
+    doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] }}"
+  tasks:
+    - name: Assert that hosts are running Rocky Linux 9.6
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.distribution == 'Rocky'
+          - ansible_facts.distribution_version == '9.6'
+          - os_distribution == 'rocky'
+        fail_msg: >-
+          This playbook is only designed for Rocky Linux 9.6 hosts. Ensure
+          that you are limiting it to only run on Rocky Linux 9.6 hosts and
+          os_distribution is set to rocky.
+
+    - name: Ensure doca kernel repo is up to date
+      ansible.builtin.dnf:
+        name: doca-kernel-repo
+        state: latest
+        update_cache: true
+      become: true
+
+    # This is required by mlnx-ofa_kernel, and comes from the doca repository.
+    # It is already present when doca-ofed is installed, but will be upgraded.
+    - name: Ensure mlnx-tools is installed
+      ansible.builtin.dnf:
+        name: mlnx-tools
+        state: latest
+        update_cache: true
+      become: true
+
+    - name: Ensure appropriate doca-kernel is installed
+      ansible.builtin.dnf:
+        name: "doca-kernel-{{ doca_kernel_version }}"
+        state: latest
+        disablerepo: doca
+      become: true
+
+    # doca-ofed 3.2 starts to depend on the dkms modules. It was not the case
+    # in doca-ofed 2.9.3.
+    - name: Ensure doca-ofed is not present (upgrading it brings dkms)
+      ansible.builtin.dnf:
+        name: doca-ofed
+        state: absent
+        autoremove: false
+      become: true
+
+    - name: Ensure latest doca-ofed-userspace instead of doca-ofed
+      ansible.builtin.dnf:
+        name: doca-ofed-userspace
+        state: latest
+      become: true
+
+    - name: Upgrade all
+      ansible.builtin.dnf:
+        name: "*"
+        state: latest
+      become: true
+
+- name: Fix potential grub config preventing new kernel from being used
+  ansible.builtin.import_playbook: reset-bls-entries.yml
+  vars:
+    reset_bls_hosts: mlnx
+
+- name: Reboot to apply updates
+  ansible.builtin.import_playbook: reboot.yml
+  vars:
+    reboot_hosts: mlnx
+
+- name: Confirm the host is upgraded to Rocky Linux 9.7
+  hosts: mlnx
+  tags: post
+  tasks:
+    - name: Update distribution facts
+      ansible.builtin.setup:
+        filter: "{{ kayobe_ansible_setup_filter }}"
+        gather_subset: "{{ kayobe_ansible_setup_gather_subset }}"
+
+    # Can fail (eg in multinode) when there are bad entries in grub config
+    # Fixed by `kayobe playbook run ansible/maintenance/reset-bls-entries.yml`
+    # and manual reboot
+    - name: Assert that hosts are now using Rocky Linux 9.7
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.distribution == 'Rocky'
+          - ansible_facts.distribution_version == '9.7'
+          - os_distribution == 'rocky'
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 21c92d3152..6c58b5db96 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -8,6 +8,11 @@ stackhpc_pulp_doca_version_matrix:
     "7": 3.2.1
 stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version] | default('2.9.1') }}"
 
+# Available and tested versions of the pre-compiled doca-ofed kernel modules
+stackhpc_doca_kernel_version_matrix:
+    "6": 5.14.0.570.21.1.el9.6.x86.64
+    "7": 5.14.0.611.13.1.el9.7.x86.64
+
 ###############################################################################
 # Pulp configuration for DOCA OFED
 

From 321eb1d5cd6231ab75877b4c127225de0ee94fca Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 8 Dec 2025 11:34:01 +0100
Subject: [PATCH 05/19] debug kolla container image build

to see which sources are downloaded before docker build
---
 etc/kayobe/kolla.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 9702f7d718..370ec19581 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}"
 # Path in which to generate kolla configuration.
 #kolla_build_config_path:
 
+# View debug logs, like source downloads, during image build.
+kolla_build_extra_config: |
+  [DEFAULT]
+  debug = True
+
 ###############################################################################
 # Kolla-ansible installation.
 

From 66c7846466d71151b2e1b6e0434eacc5a044497f Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Tue, 9 Dec 2025 10:44:57 +0100
Subject: [PATCH 06/19] aio workflow: add container image pull

to accomodate temporary errors from ark (was getting a 500 error)
---
 .github/workflows/stackhpc-all-in-one.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 75d1910980..62f9e4677d 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -306,6 +306,26 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Pull container images
+        run: |
+          # Try up to 3 times to pull container images.
+          for attempt in $(seq 3); do
+              docker run -t --rm \
+                -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+                -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+                ${{ steps.kayobe_image.outputs.kayobe_image }} \
+                /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh
+              if [ "$?" = "0" ]; then
+                  echo "Pulled container images on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to pull container images on attempt $attempt"
+          done
+          echo "Failed to pull container imagers after $attempt attempts"
+          exit 1
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Service deploy
         run: |
           docker run -t --rm \

From 9c677fb576558b038c6f7b53f00894bda5976bc7 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Fri, 19 Dec 2025 15:44:29 +0100
Subject: [PATCH 07/19] release notes

---
 releasenotes/notes/rl9.7-c8d033f11097b76b.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml
index dcdb765ec7..6b6fa315fa 100644
--- a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml
+++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml
@@ -1,7 +1,8 @@
 ---
 features:
   - |
-    Added support for Rocky Linux 9.7.
+    Added support for Rocky Linux 9.7, including host packages and a full
+    container image refresh.
   - |
     9.7 is now the default release for Rocky Linux.
 

From e9608d3ebba33a172bb7fc1912e7ffdf41f7ebb4 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 19 Jan 2026 16:52:28 +0100
Subject: [PATCH 08/19] Build kolla container images: show workflow values

---
 .github/workflows/stackhpc-container-image-build.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index ed3a7a2db0..21d91996d2 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -82,14 +82,14 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT"
 
       # Generate a tag to apply to all built container images.
       # Without this, each kayobe * container image build command would use a different tag.
       - name: Generate container datetime tag
         id: datetime_tag
         run: |
-          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT"
 
       # Dynamically define job matrix.
       # We need a separate matrix entry for each distribution, when the relevant input is true.
@@ -113,7 +113,7 @@ jobs:
           # remove trailing comma
           output="${output%,}"
           output+="]}"
-          echo "matrix=$output" >> $GITHUB_OUTPUT
+          echo "matrix=$output" | tee -a "$GITHUB_OUTPUT"
 
       - name: Display container datetime tag
         run: |
@@ -187,7 +187,7 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT"
 
       - name: Configure localhost as a seed
         run: |
@@ -261,7 +261,7 @@ jobs:
         if: inputs.seed && matrix.distro.arch == 'amd64'
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images"
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi

From 3db4139b9c88f60f755f270167467ea703ad26b0 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 19 Jan 2026 18:46:04 +0100
Subject: [PATCH 09/19] [skipci] Build kolla container images: always upload
 scan output directory

---
 .github/workflows/stackhpc-container-image-build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 21d91996d2..da96a842a6 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -271,6 +271,7 @@ jobs:
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
+        if: ${{ !cancelled() }}
 
       - name: Fail if no images have passed scanning
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi

From 56ec6d16b6a274e9c0832b9b9f1e80f116169d61 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 19 Jan 2026 18:49:49 +0100
Subject: [PATCH 10/19] [skipci] Show trivy errors when scanning images

---
 tools/scan-images.sh | 291 +++++++++++++++++++++++++++++--------------
 1 file changed, 200 insertions(+), 91 deletions(-)

diff --git a/tools/scan-images.sh b/tools/scan-images.sh
index e313cbc41a..b885b51062 100755
--- a/tools/scan-images.sh
+++ b/tools/scan-images.sh
@@ -1,104 +1,213 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
-# Check correct usage
-if [[ ! $2 ]]; then
-  echo "Usage: scan-images.sh <os-distribution> <image-tag>"
+# Disable telemetry and version check:
+# https://github.com/aquasecurity/trivy/discussions/8945
+export TRIVY_DISABLE_TELEMETRY=true
+export TRIVY_SKIP_VERSION_CHECK=true
+
+# Global variables
+scan_common_args=" \
+                  --exit-code 1 \
+                  --scanners vuln \
+                  --format json \
+                  --severity HIGH,CRITICAL \
+                  --ignore-unfixed \
+                  --db-repository ghcr.io/aquasecurity/trivy-db:2 \
+                  --db-repository public.ecr.aws/aquasecurity/trivy-db \
+                  --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
+                  --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db "
+
+# Print usage instructions and error with wrong inputs
+usage() {
+  echo "Usage: scan-images.sh <os-distribution> <image-tag> [--sbom]"
   exit 2
-fi
-
-set -u
-
-# Check that trivy is installed
-if ! trivy --version; then
-  echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.68.2'
-fi
-
-# Clear any previous outputs
-rm -rf image-scan-output
-
-# Make a fresh output directory
-mkdir -p image-scan-output
-
-# Get built container images
-images=$(docker image ls \
-  --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" \
-  --format "{{.Repository}}:{{.Tag}}")
-
-# Save list of images to file
-echo "$images" > "$1-scanned-container-images.txt"
-
-# Ensure output files exist
-touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt
-
-# If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
-# If there are vulnerabilities detected, add it to dirty-images.txt and
-# generate a csv summary
-# If the image contains at least one critical vulnerabilities, add it to
-# critical-images.txt
-for image in $images; do
-  filename=$(basename $image | sed 's/:/\./g')
-  imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
-  global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
-  image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
+}
+
+# Check dependencies are installed, print installation instructions otherwise
+check_deps_installed() {
+  if ! trivy --version > /dev/null; then
+    echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.68.2'
+    exit 1
+  fi
+  if ! yq --version > /dev/null; then
+    echo 'Please install yq: sudo dnf/apt install yq'
+    exit 1
+  fi
+}
+
+# Prepare output files
+file_prep() {
+  rm -rf image-scan-output
+  mkdir -p image-scan-output
+  touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt
+}
+
+# Gather image lists
+get_images() {
+  local output_file="$1-scanned-container-images.txt"
+
+  docker image ls \
+    --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" \
+    --format "{{.Repository}}:{{.Tag}}" \
+    > "$output_file"
+
+  cat "$output_file"
+}
+
+# Generate ignored vulnerabilities file
+generate_trivy_ignore() {
+  local imagename=$1
+  local global_vulnerabilities
+  global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
+  local image_vulnerabilities
+  image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
+
   touch .trivyignore
   for vulnerability in $global_vulnerabilities; do
-    echo $vulnerability >> .trivyignore
+    echo "$vulnerability" >> .trivyignore
   done
   for vulnerability in $image_vulnerabilities; do
-    echo $vulnerability >> .trivyignore
+    echo "$vulnerability" >> .trivyignore
   done
-  if $(trivy image \
-          --quiet \
-          --exit-code 1 \
-          --scanners vuln \
-          --format json \
-          --severity HIGH,CRITICAL \
-          --output image-scan-output/${filename}.json \
-          --ignore-unfixed \
-          --db-repository ghcr.io/aquasecurity/trivy-db:2 \
-          --db-repository public.ecr.aws/aquasecurity/trivy-db \
-          --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
-          --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \
-          $image); then
-    # Clean up the output file for any images with no vulnerabilities
-    rm -f image-scan-output/${filename}.json
-
-    # Add the image to the clean list
+}
+
+# Put results into CSV
+generate_summary_csv() {
+  local scan="$1"
+  local summary="$2"
+
+  echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary"
+
+  jq -r '.Results[]
+      | select(.Vulnerabilities)
+      | .Vulnerabilities
+      | map(select(.PkgName | test("kernel") | not ))
+      | group_by(.VulnerabilityID)
+      | map(
+        [
+          (map(.PkgName) | unique | join(";")),
+          (map(.PkgPath | select( . != null )) | join(";")),
+          .[0].PkgID,
+          .[0].VulnerabilityID,
+          .[0].FixedVersion,
+          .[0].PrimaryURL,
+          .[0].Severity
+          ]
+        )
+      | .[]
+      | @csv' "$scan" >> "$summary"
+}
+
+# Categorise images based on severity
+categorise_image() {
+  local summary="$1"
+  local image="$2"
+
+  if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then
+    echo "${image}" >> image-scan-output/critical-images.txt
+  else
+    echo "${image}" >> image-scan-output/high-images.txt
+  fi
+}
+
+# Generate SBOM, return correct scan command for SBOM
+generate_sbom() {
+  local sbom="$1"
+  local scan="$2"
+  local image="$3"
+  trivy image \
+        --debug \
+        --format spdx-json \
+        --output "$sbom" \
+        "$image" &> "$sbom.log"
+  if [ ! -e "$sbom" ]; then
+    (
+      echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2
+      echo "==== trivy log ===="
+      cat "$sbom.log"
+    ) 1>&2
+    exit 1
+  elif grep -q FATAL "$sbom.log"; then
+    (
+      echo "ERROR: trivy image encountered a fatal error producing $sbom for $image"
+      echo "==== trivy log ===="
+      cat "$sbom.log"
+      echo "==== sbom.json ===="
+      cat "$sbom"
+    ) 1>&2
+    exit 1
+  else
+    echo "trivy sbom $scan_common_args --output $scan $sbom"
+  fi
+}
+
+# Scan images, generate SBOMs if requested
+scan_image() {
+  local image=$1
+  local filename
+  filename=$(basename "$image" | sed 's/:/\./g')
+  local imagename
+  imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g')
+  local sbom="image-scan-output/${imagename}/${filename}-sbom.json"
+  local scan="image-scan-output/${imagename}/${filename}-scan.json"
+  local summary="image-scan-output/${imagename}/${filename}-summary.csv"
+
+  mkdir -p "image-scan-output/$imagename"
+  generate_trivy_ignore "$imagename"
+
+  # If SBOM is required, generate it first and scan the results, otherwise we
+  # scan the image directly.
+  if $generate_sbom; then
+    echo "Generating SBOM for $imagename"
+    scan_command="$(generate_sbom "$sbom" "$scan" "$image")"
+  else
+    scan_command="trivy image $scan_common_args --output $scan $image"
+  fi
+
+  # Run scan against image or SBOM, format output. If no results, delete files.
+  echo "Scanning $imagename for vulnerabilities"
+  if $scan_command >& "$scan.log"; then
+    rm -f "$scan"
     echo "${image}" >> image-scan-output/clean-images.txt
+  elif [ ! -f "$scan" ]; then
+    (
+      echo "ERROR: trivy scan encountered an error producing $scan"
+      echo "Command: $scan_command"
+      echo "==== trivy log ===="
+      cat "$scan.log"
+      if $generate_sbom; then
+        echo "==== sbom.json ===="
+        cat "$sbom"
+      fi
+    ) 1>&2
+    exit 1
   else
+    generate_summary_csv "$scan" "$summary"
+    categorise_image "$summary" "$image"
+  fi
+}
+
+# Main function
+main() {
+  if [[ ! $2 ]]; then
+    usage
+  fi
 
-    # Write a header for the summary CSV
-    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv
-
-    # Write the summary CSV data
-    jq -r '.Results[]
-            | select(.Vulnerabilities)
-            | .Vulnerabilities
-            # Ignore packages with "kernel" in the PkgName
-            | map(select(.PkgName | test("kernel") | not ))
-            | group_by(.VulnerabilityID)
-            | map(
-                  [
-                    (map(.PkgName) | unique | join(";")),
-                    (map(.PkgPath | select( . != null )) | join(";")),
-                    .[0].PkgID,
-                    .[0].VulnerabilityID,
-                    .[0].FixedVersion,
-                    .[0].PrimaryURL,
-                    .[0].Severity
-                    ]
-                )
-            | .[]
-            | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
-
-    if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then
-      # If the image contains critical vulnerabilities, add the image to critical list
-      echo "${image}" >> image-scan-output/critical-images.txt
-    else
-      # Otherwise, add the image to the dirty list
-      echo "${image}" >> image-scan-output/dirty-images.txt
-    fi
+  generate_sbom=false
+  if [[ "$3" == "--sbom" ]]; then
+    generate_sbom=true
   fi
-  rm .trivyignore
-done
+
+  set -u
+
+  check_deps_installed
+  file_prep
+
+  images=$(get_images "$1" "$2")
+  for image in $images; do
+    scan_image "$image"
+  done
+}
+
+main "$@"

From a9629e28a0e8a68f3fce45af3c6d68800e29ea0b Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Tue, 17 Feb 2026 15:15:42 +0100
Subject: [PATCH 11/19] backport stackhpc-container-image-build.yml from 2025.1

---
 .../stackhpc-container-image-build.yml        | 39 +++++++++----------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index da96a842a6..1fe2034eed 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -38,7 +38,12 @@ on:
         type: boolean
         required: false
         default: true
-      push-dirty:
+      sbom:
+        description: Generate SBOM?
+        type: boolean
+        required: false
+        default: true
+      push-critical:
         description: Push scanned images that have critical vulnerabilities?
         type: boolean
         required: false
@@ -198,17 +203,6 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
-      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
-      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
-      # runner and dind containers.
-      - name: Deploy an authenticating package repository mirror proxy
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
-        env:
-          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
-
       - name: Create build logs output directory
         run: mkdir image-build-logs
 
@@ -230,6 +224,9 @@ jobs:
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           args="$args -e kolla_build_log_path=$GITHUB_WORKSPACE/image-build-logs/kolla-build-overcloud.log"
           args="$args -e base_path=$GITHUB_WORKSPACE/opt/kayobe"
+          # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+          # runner and dind containers.
+          args="$args -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud container image build $args
@@ -267,15 +264,15 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom' }}
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
         if: ${{ !cancelled() }}
 
-      - name: Fail if no images have passed scanning
+      - name: Fail if any images have critical vulnerabilities
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
-        if: ${{ !inputs.push-dirty }}
+        if: ${{ !inputs.push-critical }}
 
       - name: Copy clean images to push-attempt-images list
         run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
@@ -285,13 +282,13 @@ jobs:
       # This should be reverted when it's decided to filter high level CVEs as well.
       - name: Append dirty images to push list
         run: |
-          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+          cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt
         if: ${{ inputs.push }}
 
       - name: Append images with critical vulnerabilities to push list
         run: |
           cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
-        if: ${{ inputs.push && inputs.push-dirty }}
+        if: ${{ inputs.push && inputs.push-critical }}
 
       - name: Push images
         run: |
@@ -334,18 +331,18 @@ jobs:
 
       - name: Fail when images failed to push
         run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
-        if: ${{ !cancelled() }}
+        if: ${{ inputs.push && !cancelled() }}
 
       # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
       # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
       # decided to fail the job on detecting high CVEs as well.
       # - name: Fail when images failed scanning
-      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
-      #   if: ${{ !inputs.push-dirty && !cancelled() }}
+      #   run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi
+      #   if: ${{ !inputs.push-critical && !cancelled() }}
 
       - name: Fail when critical vulnerabilities are found
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
-        if: ${{ !inputs.push-dirty && !cancelled() }}
+        if: ${{ !inputs.push-critical && !cancelled() }}
 
       - name: Remove locally built images for this run
         if: always() && runner.arch == 'ARM64'

From 3b661596227d0c5c3d83390af662ce500c7092d1 Mon Sep 17 00:00:00 2001
From: Jack Hodgkiss <jack@stackhpc.com>
Date: Mon, 4 Aug 2025 12:16:29 +0100
Subject: [PATCH 12/19] Use env hooks to setup Pulp proxy in CI

Use the authenticating pulp_proxy for all CI build jobs that need packages from Ark -
host images, Kolla images and the IPA image.
---
 .../overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml | 1 +
 .../pre.d/10-pulp-auth-proxy.yml                                 | 1 +
 .../overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml      | 1 +
 etc/kayobe/environments/ci-builder/stackhpc-ci.yml               | 1 +
 4 files changed, 4 insertions(+)
 create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml
 create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml
 create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml

diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml
new file mode 120000
index 0000000000..6e84a71044
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml
@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml
new file mode 120000
index 0000000000..6e84a71044
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml
@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml
new file mode 120000
index 0000000000..6e84a71044
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml
@@ -0,0 +1 @@
+../../../../../ansible/pulp-auth-proxy.yml
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 6f3a400548..5c00f815a7 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -38,6 +38,7 @@ kolla_build_neutron_ovs: true
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
+stackhpc_repo_mirror_auth_proxy_enabled: true
 stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "skc-ci-builder-reader"
 stackhpc_repo_mirror_password: !vault |

From 9b2281b8d7055788c239ca10d08e656affa35f16 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Thu, 5 Feb 2026 17:33:29 +0100
Subject: [PATCH 13/19] Rebuilt overcloud host image

See actions/runs/21713574987
---
 etc/kayobe/pulp-host-image-versions.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml
index db0316a37d..b7e13a44da 100644
--- a/etc/kayobe/pulp-host-image-versions.yml
+++ b/etc/kayobe/pulp-host-image-versions.yml
@@ -1,6 +1,6 @@
 ---
 # Overcloud host image versioning tags
 # These images must be in SMS, since they are used by our AIO CI runners
-stackhpc_rocky_9_overcloud_host_image_version: "2024.1-20241209T151515"
+stackhpc_rocky_9_overcloud_host_image_version: "2024.1-20260205T145550"
 stackhpc_ubuntu_jammy_overcloud_host_image_version: "2024.1-20250408T082408"
 stackhpc_ubuntu_noble_overcloud_host_image_version: "2024.1-20250417T104113"

From dae78e6a208b33b11f13c4e59cb6c3ba77f8497e Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 9 Feb 2026 09:26:13 +0100
Subject: [PATCH 14/19] bifrost unbuildable since switch to unmaintained

See https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a
---
 etc/kayobe/kolla-image-tags.yml |  2 ++
 etc/kayobe/pulp.yml             | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index 24a7b7663d..c1b0e9ef35 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -17,6 +17,8 @@ kolla_image_tags:
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   bifrost:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250325T141125
+    # not buildable anymore, see https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a
+    rocky-9: 2024.1-rocky-9-20250716T041633
   cinder:
     rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 07ecc37942..0ab2694ab7 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -611,9 +611,13 @@ stackhpc_pulp_images_kolla:
 
 # List of images for each base distribution which should not/cannot be built.
 stackhpc_kolla_unbuildable_images:
-  ubuntu-jammy: []
-  ubuntu-noble: []
-  rocky-9: []
+  # See https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a
+  ubuntu-jammy:
+    - bifrost-deploy
+  ubuntu-noble:
+    - bifrost-deploy
+  rocky-9:
+    - bifrost-deploy
 
 # Whitespace-separated list of regular expressions matching Kolla image names.
 # Usage is similar to kolla-build CLI arguments.

From d64e434db2a63ffb916bed7991b17d574c6faa44 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 9 Feb 2026 17:51:21 +0100
Subject: [PATCH 15/19] Fix allowed vulnerability list not reset between images

---
 tools/scan-images.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/scan-images.sh b/tools/scan-images.sh
index b885b51062..14da5a53aa 100755
--- a/tools/scan-images.sh
+++ b/tools/scan-images.sh
@@ -63,7 +63,7 @@ generate_trivy_ignore() {
   local image_vulnerabilities
   image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
 
-  touch .trivyignore
+  truncate -s 0 .trivyignore  # ensure we start from a clean slate
   for vulnerability in $global_vulnerabilities; do
     echo "$vulnerability" >> .trivyignore
   done

From 5551bceacc336ea94e73c4022a9fd65cf150c6b9 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Wed, 11 Feb 2026 13:19:44 +0100
Subject: [PATCH 16/19] Fix wc error when file doesn't exist

---
 .github/workflows/stackhpc-container-image-build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 1fe2034eed..9995a7d2c5 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -271,7 +271,7 @@ jobs:
         if: ${{ !cancelled() }}
 
       - name: Fail if any images have critical vulnerabilities
-        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
+        run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
         if: ${{ !inputs.push-critical }}
 
       - name: Copy clean images to push-attempt-images list
@@ -330,7 +330,7 @@ jobs:
         if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
 
       - name: Fail when images failed to push
-        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
+        run: if [ -e image-build-logs/push-failed-images.txt ] && [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
         if: ${{ inputs.push && !cancelled() }}
 
       # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
@@ -341,7 +341,7 @@ jobs:
       #   if: ${{ !inputs.push-critical && !cancelled() }}
 
       - name: Fail when critical vulnerabilities are found
-        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
+        run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-critical && !cancelled() }}
 
       - name: Remove locally built images for this run

From 18027cac09dc03eae5cb623ad722832f04266b62 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Tue, 17 Feb 2026 15:14:09 +0100
Subject: [PATCH 17/19] limit docker-ce to 28 on Caracal

---
 etc/kayobe/docker.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml
index 26fac13464..85d3886d87 100644
--- a/etc/kayobe/docker.yml
+++ b/etc/kayobe/docker.yml
@@ -38,6 +38,15 @@ docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}"
 # Enable live-restore on docker daemon
 docker_daemon_live_restore: true
 
+# Avoid docker 29 for the moment in Caracal
+docker_packages_version: "{{ '-28.*' if os_distribution == 'rocky' else ('=5:28*' if os_release == 'noble' else '=5:27*') }}"
+# variable from https://github.com/stackhpc/ansible-role-docker/blob/master/defaults/main.yml
+docker_packages:
+  - "docker-ce{{ docker_packages_version }}"
+  - "docker-ce-cli{{ docker_packages_version }}"
+  - "docker-ce-rootless-extras{{ docker_packages_version }}"
+  - "containerd.io"
+
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.
 workaround_ansible_issue_8743: yes

From 3c961f2885e9334cb0a6577c73444e2736d57f46 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Fri, 5 Dec 2025 17:37:23 +0100
Subject: [PATCH 18/19] Updated or ignored for CVE vulnerabilities

- bump cadvisor to 0.56.2

- Ignore CVE-2024-24790 in prometheus exporters
  control plane is trusted

- Upgrade prometheus-msteams to 1.5.3
  to fix CVE-2023-24538 CVE-2023-24540

- opensearch-dashboard: ignore CVE-2025-68428
  CVE-2025-68428 is still present in opensearch-dashboards 2.19.4
  because jspdf is still in version 3.0.1

- Ignore CVE-2024-24790 in prometheus-mtail
  control plane is trusted

- Bump grafana to 12.3.3 to fix CVE-2025-68121
  grafana server 12.3.3 is fixed but the opensearch-datasource plugin
  is still affected.

- Bump etcd to 3.5.27 to fix CVE-2025-68121

- Ignore CVE-2025-68121 for prometheus images
  - server-side: exporters and server are not listening with tls
  - as client: only querying known services

- Ignore CVE-2025-68121 for influxdb
  No new version is available and it runs on a secure network

- Ignore CVE-2025-68121 for letsencrypt-lego
  it only talks to known servers

- Ignore CVE-2025-68121 for neutron
  it is the docker client that triggers it and we don't speak to remote
  docker over tls
---
 etc/kayobe/kolla/kolla-build.conf            | 13 ++++-
 etc/kayobe/pulp-repo-versions.yml            |  2 +-
 etc/kayobe/trivy/allowed-vulnerabilities.yml | 51 ++++++++++++++++++++
 3 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf
index 95692e5ef4..f8daae9ebe 100644
--- a/etc/kayobe/kolla/kolla-build.conf
+++ b/etc/kayobe/kolla/kolla-build.conf
@@ -18,9 +18,18 @@ location = https://github.com/stackhpc/requirements
 reference = stackhpc/{{ openstack_release }}
 
 [etcd]
-version = 3.5.21
-sha256 = amd64:adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c,arm64:95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f
+version = 3.5.27
+sha256 = amd64:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc,arm64:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa
 
 [letsencrypt-lego]
 version = v4.23.1
 sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,arm64:1114745108343286d4bff189b4bdee3cba9d07ebcacc673860d91ab951d31e0d
+
+# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available
+[prometheus-cadvisor]
+version = 0.56.2
+sha256 = amd64:ad92930f16a2f9da15190675e09eeaceb8fd38637d07a686bb0dd68695f692af,arm64:b7a707379496fd7a7b5d2768c5c494427112f534ba5069f889af28ffe6ad11bb
+
+[prometheus-msteams]
+version = 1.5.3
+sha256 = amd64:8eae63d89338f53a990fa2720b3fabf58c916e2648c948ce1e0f29942459a491
diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml
index dd5f7dbab3..537b366745 100644
--- a/etc/kayobe/pulp-repo-versions.yml
+++ b/etc/kayobe/pulp-repo-versions.yml
@@ -26,7 +26,7 @@ stackhpc_pulp_repo_elrepo_9_version: 20260127T212055
 stackhpc_pulp_repo_elrepo_9_aarch64_version: 20250408T030629
 stackhpc_pulp_repo_epel_9_version: 20260204T220346
 stackhpc_pulp_repo_epel_9_aarch64_version: 20260204T223146
-stackhpc_pulp_repo_grafana_version: 20260204T212232
+stackhpc_pulp_repo_grafana_version: 20260214T213531
 stackhpc_pulp_repo_opensearch_2_x_version: 20251106T202313
 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20251106T202313
 stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20251130T213625
diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml
index 579ed31941..26ee5977dd 100644
--- a/etc/kayobe/trivy/allowed-vulnerabilities.yml
+++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml
@@ -16,31 +16,82 @@ fluentd_allowed_vulnerabilities:
   - CVE-2024-27280
 grafana_allowed_vulnerabilities:
   - CVE-2024-8986
+  - CVE-2025-68121  # the opensearch datasource plugin is still vulnerable
 influxdb_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
+ironic_neutron_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+letsencrypt_lego_allowed_vulnerabilities:
+  - CVE-2025-68121
 magnum_conductor_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
+neutron_base_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_bgp_dragent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_dhcp_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_l3_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_linuxbridge_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_metadata_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_mlnx_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_openvswitch_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_ovn_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_server_allowed_vulnerabilities:
+  - CVE-2025-68121
+neutron_sriov_agent_allowed_vulnerabilities:
+  - CVE-2025-68121
+opensearch_dashboards_allowed_vulnerabilities:
+  - CVE-2025-68428
+prometheus_alertmanager_allowed_vulnerabilities:
+  - CVE-2025-68121
 prometheus_blackbox_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_memcached_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_mysqld_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_elasticsearch_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_node_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_openstack_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
+  - CVE-2025-68121
+prometheus_ovn_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
+  - CVE-2025-68121
 prometheus_libvirt_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_cadvisor_allowed_vulnerabilities:
   - CVE-2024-41110
   - CVE-2024-45337
+  - CVE-2025-68121
 prometheus_msteams_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
+prometheus_mtail_allowed_vulnerabilities:
+  - CVE-2024-24790
+  - CVE-2025-68121
 prometheus_v2_server_allowed_vulnerabilities:
   - CVE-2024-45337
+  - CVE-2025-68121
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

From 1e349593e06a7d0a532e136b6176662a74a71ef3 Mon Sep 17 00:00:00 2001
From: Eric Le Lay <eric@stackhpc.com>
Date: Mon, 23 Feb 2026 11:29:06 +0100
Subject: [PATCH 19/19] Rebuilt kolla container images

See https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/22297360173
---
 etc/kayobe/kolla-image-tags.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
index c1b0e9ef35..8885ff1bd4 100644
--- a/etc/kayobe/kolla-image-tags.yml
+++ b/etc/kayobe/kolla-image-tags.yml
@@ -4,15 +4,13 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky-9: 2024.1-rocky-9-20250716T041633
+    rocky-9: 2024.1-rocky-9-20260223T075405
     ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809
     ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323
   barbican:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   blazar:
-    rocky-9: 2024.1-rocky-9-20250627T102805
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   bifrost:
@@ -20,77 +18,61 @@ kolla_image_tags:
     # not buildable anymore, see https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a
     rocky-9: 2024.1-rocky-9-20250716T041633
   cinder:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   cloudkitty:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   designate:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   heat:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   glance:
-    rocky-9: 2024.1-rocky-9-20250903T130503
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250903T130503
     ubuntu-noble: 2024.1-ubuntu-noble-20250903T130503
   horizon:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250227T091118
   ironic:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   ironic_dnsmasq:
-    rocky-9: 2024.1-rocky-9-20250716T041633
     ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809
     ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323
   ironic_prometheus_exporter:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250124T081816
   iscsid:
-    rocky-9: 2024.1-rocky-9-20251103T065408
     ubuntu-jammy: 2024.1-ubuntu-jammy-20251103T065408
     ubuntu-noble: 2024.1-ubuntu-noble-20251103T065408
   keystone:
-    rocky-9: 2024.1-rocky-9-20251031T092724
     ubuntu-jammy: 2024.1-ubuntu-jammy-20251031T092724
     ubuntu-noble: 2024.1-ubuntu-noble-20251031T092724
   kolla_toolbox:
-    rocky-9: 2024.1-rocky-9-20250908T155427
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147
     ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147
   magnum:
-    rocky-9: 2024.1-rocky-9-20250922T105039
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250922T105039
     ubuntu-noble: 2024.1-ubuntu-noble-20250922T105039
   manila:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   neutron:
-    rocky-9: 2024.1-rocky-9-20260128T142402
     ubuntu-jammy: 2024.1-ubuntu-jammy-20260128T065158
     ubuntu-noble: 2024.1-ubuntu-noble-20260128T065158
   neutron_bgp_dragent:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147
     ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147
   nova:
-    rocky-9: 2024.1-rocky-9-20260220T123443
     ubuntu-jammy: 2024.1-ubuntu-jammy-20260220T123443
     ubuntu-noble: 2024.1-ubuntu-noble-20260220T123443
   octavia:
-    rocky-9: 2024.1-rocky-9-20250717T094248
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805
     ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805
   openvswitch:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147
     ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147
   ovn:
-    rocky-9: 2024.1-rocky-9-20250925T130625
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250925T130625
     ubuntu-noble: 2024.1-ubuntu-noble-20250925T130625
   prometheus:
@@ -98,10 +80,8 @@ kolla_image_tags:
   prometheus_alertmanager:
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250422T103147
   prometheus_libvirt_exporter:
-    rocky-9: 2024.1-rocky-9-20250819T151035
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250819T151035
     ubuntu-noble: 2024.1-ubuntu-noble-20250819T151035
   rabbitmq:
-    rocky-9: 2024.1-rocky-9-20250908T155427
     ubuntu-jammy: 2024.1-ubuntu-jammy-20250722T083943
     ubuntu-noble: 2024.1-ubuntu-noble-20250722T083943