Skip to content

Commit 64f1913

Browse files
authored
Merge pull request #2061 from stackhpc/rl-9.7-containers
Rocky Linux 9.7 containers
2 parents 49744cc + e3743c5 commit 64f1913

9 files changed

Lines changed: 113 additions & 50 deletions

File tree

.github/workflows/stackhpc-all-in-one.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,26 @@ jobs:
314314
env:
315315
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
316316

317+
- name: Pull container images
318+
run: |
319+
# Try up to 3 times to pull container images.
320+
for attempt in $(seq 3); do
321+
docker run -t --rm \
322+
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
323+
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
324+
${{ steps.kayobe_image.outputs.kayobe_image }} \
325+
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh
326+
if [ "$?" = "0" ]; then
327+
echo "Pulled container images on attempt $attempt"
328+
exit 0
329+
fi
330+
echo "Failed to pull container images on attempt $attempt"
331+
done
332+
echo "Failed to pull container imagers after $attempt attempts"
333+
exit 1
334+
env:
335+
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
336+
317337
- name: Service deploy
318338
run: |
319339
docker run -t --rm \

.github/workflows/stackhpc-container-image-build.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,14 @@ jobs:
8282
id: openstack_release
8383
run: |
8484
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
85-
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
85+
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT"
8686
8787
# Generate a tag to apply to all built container images.
8888
# Without this, each kayobe * container image build command would use a different tag.
8989
- name: Generate container datetime tag
9090
id: datetime_tag
9191
run: |
92-
echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
92+
echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT"
9393
9494
# Dynamically define job matrix.
9595
# We need a separate matrix entry for each distribution, when the relevant input is true.
@@ -110,7 +110,7 @@ jobs:
110110
# remove trailing comma
111111
output="${output%,}"
112112
output+="]}"
113-
echo "matrix=$output" >> $GITHUB_OUTPUT
113+
echo "matrix=$output" | tee -a "$GITHUB_OUTPUT"
114114
115115
- name: Display container datetime tag
116116
run: |
@@ -184,7 +184,7 @@ jobs:
184184
185185
- name: Get Kolla tag
186186
id: write-kolla-tag
187-
run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" >> $GITHUB_OUTPUT
187+
run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT"
188188

189189
- name: Configure localhost as a seed
190190
run: |
@@ -250,7 +250,7 @@ jobs:
250250
if: inputs.seed && matrix.distro.arch == 'amd64'
251251

252252
- name: Get built container images
253-
run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
253+
run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images"
254254

255255
- name: Fail if no images have been built
256256
run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
@@ -260,6 +260,7 @@ jobs:
260260

261261
- name: Move image scan logs to output artifact
262262
run: mv image-scan-output image-build-logs/image-scan-output
263+
if: ${{ !cancelled() }}
263264

264265
- name: Fail if any images have critical vulnerabilities
265266
run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi

etc/kayobe/kolla-image-tags.yml

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,19 @@
44
# where the key is the OS distro and the value is the tag to deploy.
55
kolla_image_tags:
66
openstack:
7-
rocky-9: 2025.1-rocky-9-20250730T105631
7+
rocky-9: 2025.1-rocky-9-20260123T132550
88
ubuntu-noble: 2025.1-ubuntu-noble-20250730T105631
99
iscsid:
10-
rocky-9: 2025.1-rocky-9-20251013T135156
1110
ubuntu-noble: 2025.1-ubuntu-noble-20251013T135156
1211
keystone:
13-
rocky-9: 2025.1-rocky-9-20251031T094539
1412
ubuntu-noble: 2025.1-ubuntu-noble-20251031T094539
1513
kolla_toolbox:
16-
rocky-9: 2025.1-rocky-9-20250905T151507
1714
ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507
1815
magnum:
19-
rocky-9: 2025.1-rocky-9-20251016T073648
2016
ubuntu-noble: 2025.1-ubuntu-noble-20251016T073648
2117
neutron:
22-
rocky-9: 2025.1-rocky-9-20260116T134029
2318
ubuntu-noble: 2025.1-ubuntu-noble-20260116T134029
24-
ovn:
25-
rocky-9: 2025.1-rocky-9-20250915T100822
2619
prometheus_libvirt_exporter:
27-
rocky-9: 2025.1-rocky-9-20250822T151934
2820
ubuntu-noble: 2025.1-ubuntu-noble-20250822T151934
2921
rabbitmq:
30-
rocky-9: 2025.1-rocky-9-20250905T151507
3122
ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507

etc/kayobe/kolla.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}"
2323
# Path in which to generate kolla configuration.
2424
#kolla_build_config_path:
2525

26+
# View debug logs, like source downloads, during image build.
27+
kolla_build_extra_config: |
28+
[DEFAULT]
29+
debug = True
30+
2631
###############################################################################
2732
# Kolla-ansible installation.
2833

etc/kayobe/kolla/kolla-build.conf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,8 @@ sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,
2929
[magnum-conductor-plugin-helm]
3030
version = v3.18.2
3131
sha256 = amd64:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce,arm64:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d
32+
33+
# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available
34+
[prometheus-cadvisor]
35+
version = 0.54.1
36+
sha256 = amd64:21be8d2797433048474e676d37c215c28fb171509448ef9b1c4648a564e39595,arm64:21f7bac786f6c53a8091964b4d3ff2486a0c460e5a410000b59a9a565b4183a9
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
22
# Overcloud host image versioning tags
33
# These images must be in SMS, since they are used by our AIO CI runners
4-
stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260113T105634"
4+
stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260119T110913"
55
stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250930T144255"

etc/kayobe/trivy/allowed-vulnerabilities.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ influxdb_allowed_vulnerabilities:
2020
- CVE-2024-45337
2121
magnum_conductor_allowed_vulnerabilities:
2222
- CVE-2024-45337
23+
opensearch_dashboards_allowed_vulnerabilities:
24+
- CVE-2025-68428
2325
prometheus_blackbox_exporter_allowed_vulnerabilities:
26+
- CVE-2024-24790
2427
- CVE-2024-45337
2528
prometheus_memcached_exporter_allowed_vulnerabilities:
2629
- CVE-2024-45337
@@ -31,7 +34,10 @@ prometheus_elasticsearch_exporter_allowed_vulnerabilities:
3134
prometheus_node_exporter_allowed_vulnerabilities:
3235
- CVE-2024-45337
3336
prometheus_openstack_exporter_allowed_vulnerabilities:
37+
- CVE-2024-24790
3438
- CVE-2024-45337
39+
prometheus_ovn_exporter_allowed_vulnerabilities:
40+
- CVE-2024-24790
3541
prometheus_libvirt_exporter_allowed_vulnerabilities:
3642
- CVE-2024-45337
3743
prometheus_cadvisor_allowed_vulnerabilities:

releasenotes/notes/rl9.7-c8d033f11097b76b.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
---
22
features:
33
- |
4-
Added support for Rocky Linux 9.7.
4+
Added support for Rocky Linux 9.7, including host packages and a full
5+
container image refresh.
56
- |
67
9.7 is now the default release for Rocky Linux.
78

tools/scan-images.sh

Lines changed: 67 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -58,24 +58,26 @@ get_images() {
5858
# Generate ignored vulnerabilities file
5959
generate_trivy_ignore() {
6060
local imagename=$1
61-
local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
62-
local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
61+
local global_vulnerabilities
62+
global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
63+
local image_vulnerabilities
64+
image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
6365

6466
touch .trivyignore
6567
for vulnerability in $global_vulnerabilities; do
66-
echo $vulnerability >> .trivyignore
68+
echo "$vulnerability" >> .trivyignore
6769
done
6870
for vulnerability in $image_vulnerabilities; do
69-
echo $vulnerability >> .trivyignore
71+
echo "$vulnerability" >> .trivyignore
7072
done
7173
}
7274

7375
# Put results into CSV
7476
generate_summary_csv() {
75-
local imagename=$1
76-
local filename=$2
77+
local scan="$1"
78+
local summary="$2"
7779

78-
echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv
80+
echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary"
7981

8082
jq -r '.Results[]
8183
| select(.Vulnerabilities)
@@ -94,16 +96,15 @@ generate_summary_csv() {
9496
]
9597
)
9698
| .[]
97-
| @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv
99+
| @csv' "$scan" >> "$summary"
98100
}
99101

100102
# Categorise images based on severity
101103
categorise_image() {
102-
local imagename=$1
103-
local filename=$2
104-
local image=$3
104+
local summary="$1"
105+
local image="$2"
105106

106-
if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then
107+
if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then
107108
echo "${image}" >> image-scan-output/critical-images.txt
108109
else
109110
echo "${image}" >> image-scan-output/high-images.txt
@@ -112,45 +113,78 @@ categorise_image() {
112113

113114
# Generate SBOM, return correct scan command for SBOM
114115
generate_sbom() {
115-
local imagename=$1
116-
local filename=$2
117-
local image=$3
116+
local sbom="$1"
117+
local scan="$2"
118+
local image="$3"
118119
trivy image \
120+
--debug \
119121
--format spdx-json \
120-
--output image-scan-output/${imagename}/${filename}-sbom.json \
121-
$image > /dev/null 2>&1
122-
echo "trivy sbom $scan_common_args \
123-
--output image-scan-output/${imagename}/${filename}-scan.json \
124-
image-scan-output/${imagename}/${filename}-sbom.json"
122+
--output "$sbom" \
123+
"$image" &> "$sbom.log"
124+
if [ ! -e "$sbom" ]; then
125+
(
126+
echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2
127+
echo "==== trivy log ===="
128+
cat "$sbom.log"
129+
) 1>&2
130+
exit 1
131+
elif grep -q FATAL "$sbom.log"; then
132+
(
133+
echo "ERROR: trivy image encountered a fatal error producing $sbom for $image"
134+
echo "==== trivy log ===="
135+
cat "$sbom.log"
136+
echo "==== sbom.json ===="
137+
cat "$sbom"
138+
) 1>&2
139+
exit 1
140+
else
141+
echo "trivy sbom $scan_common_args --output $scan $sbom"
142+
fi
125143
}
126144

127145
# Scan images, generate SBOMs if requested
128146
scan_image() {
129147
local image=$1
130-
local filename=$(basename $image | sed 's/:/\./g')
131-
local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
148+
local filename
149+
filename=$(basename "$image" | sed 's/:/\./g')
150+
local imagename
151+
imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g')
152+
local sbom="image-scan-output/${imagename}/${filename}-sbom.json"
153+
local scan="image-scan-output/${imagename}/${filename}-scan.json"
154+
local summary="image-scan-output/${imagename}/${filename}-summary.csv"
132155

133-
mkdir -p image-scan-output/$imagename
134-
generate_trivy_ignore $imagename
156+
mkdir -p "image-scan-output/$imagename"
157+
generate_trivy_ignore "$imagename"
135158

136159
# If SBOM is required, generate it first and scan the results, otherwise we
137160
# scan the image directly.
138161
if $generate_sbom; then
139162
echo "Generating SBOM for $imagename"
140-
scan_command=$(generate_sbom $imagename $filename $image)
163+
scan_command="$(generate_sbom "$sbom" "$scan" "$image")"
141164
else
142-
scan_command="trivy image $scan_common_args \
143-
--output image-scan-output/${imagename}/${filename}-scan.json $image"
165+
scan_command="trivy image $scan_common_args --output $scan $image"
144166
fi
145167

146168
# Run scan against image or SBOM, format output. If no results, delete files.
147169
echo "Scanning $imagename for vulnerabilities"
148-
if $scan_command > /dev/null 2>&1; then
149-
rm -f image-scan-output/${imagename}/${filename}-scan.json
170+
if $scan_command >& "$scan.log"; then
171+
rm -f "$scan"
150172
echo "${image}" >> image-scan-output/clean-images.txt
173+
elif [ ! -f "$scan" ]; then
174+
(
175+
echo "ERROR: trivy scan encountered an error producing $scan"
176+
echo "Command: $scan_command"
177+
echo "==== trivy log ===="
178+
cat "$scan.log"
179+
if $generate_sbom; then
180+
echo "==== sbom.json ===="
181+
cat "$sbom"
182+
fi
183+
) 1>&2
184+
exit 1
151185
else
152-
generate_summary_csv $imagename $filename
153-
categorise_image $imagename $filename $image
186+
generate_summary_csv "$scan" "$summary"
187+
categorise_image "$summary" "$image"
154188
fi
155189
}
156190

@@ -170,9 +204,9 @@ main() {
170204
check_deps_installed
171205
file_prep
172206

173-
images=$(get_images $1 $2)
207+
images=$(get_images "$1" "$2")
174208
for image in $images; do
175-
scan_image $image
209+
scan_image "$image"
176210
done
177211
}
178212

0 commit comments

Comments
 (0)