Skip to content

Commit a0fb79b

Browse files
bbezakbbezak
committed
Add Rocky 9 aarch64 DOCA OFED support
Signed-off-by: Bartosz Bezak <bartosz@stackhpc.com>
1 parent 7c1f3be commit a0fb79b

10 files changed

Lines changed: 327 additions & 19 deletions

File tree

‎.github/workflows/package-build-ofed.yml‎

Lines changed: 289 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,17 @@ on:
77
description: Build Rocky Linux 9
88
type: boolean
99
default: true
10+
rocky9-aarch64:
11+
description: Build Rocky Linux 9 aarch64
12+
type: boolean
13+
default: true
14+
runner_env:
15+
description: Which cloud to run on?
16+
type: choice
17+
default: SMS Lab
18+
options:
19+
- SMS Lab
20+
- Leafcloud
1021
secrets:
1122
KAYOBE_VAULT_PASSWORD_CI_BUILDER:
1223
required: true
@@ -22,17 +33,55 @@ env:
2233
KAYOBE_ENVIRONMENT: ci-doca-builder
2334
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
2435
jobs:
25-
overcloud-ofed-packages:
26-
name: Build OFED kernel modules
36+
runner-selection:
37+
uses: ./.github/workflows/runner-selector.yml
38+
with:
39+
runner_env: ${{ inputs.runner_env }}
40+
41+
create-tag:
42+
name: Create a tag to be added to resulting packages
2743
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
28-
runs-on: arc-skc-host-image-builder-runner
44+
environment: ${{ inputs.runner_env }}
45+
runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
46+
needs:
47+
- runner-selection
2948
permissions: {}
49+
outputs:
50+
ofed_tag: ${{ steps.ofed_tag.outputs.ofed_tag }}
3051
steps:
52+
- name: Validate inputs
53+
run: |
54+
if [[
55+
"${{ inputs.rocky9 }}" == "false" &&
56+
"${{ inputs.rocky9-aarch64 }}" == "false"
57+
]]; then
58+
echo "At least one distribution must be selected"
59+
exit 1
60+
fi
61+
62+
if [[
63+
"${{ inputs.rocky9-aarch64 }}" == "true" &&
64+
"${{ inputs.runner_env }}" != "SMS Lab"
65+
]]; then
66+
echo "aarch64 builds are only supported on SMS Lab"
67+
exit 1
68+
fi
69+
3170
- name: Generate OFED tag
3271
id: ofed_tag
3372
run: |
3473
echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
3574
75+
overcloud-ofed-packages:
76+
name: Build OFED kernel modules
77+
if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.rocky9
78+
environment: ${{ inputs.runner_env }}
79+
runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
80+
needs:
81+
- runner-selection
82+
- create-tag
83+
permissions: {}
84+
steps:
3685
- name: Install Package dependencies
3786
run: |
3887
sudo apt update &&
@@ -93,9 +142,9 @@ jobs:
93142
ssh_username = "cloud-user"
94143
aio_vm_name = "skc-ofed-builder"
95144
aio_vm_image = "${{ env.VM_IMAGE }}"
96-
aio_vm_flavor = "en1.medium"
97-
aio_vm_network = "stackhpc-ci"
98-
aio_vm_subnet = "stackhpc-ci"
145+
aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
146+
aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
147+
aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
99148
aio_vm_interface = "ens3"
100149
aio_vm_tags = ${{ env.VM_TAGS }}
101150
EOF
@@ -108,7 +157,7 @@ jobs:
108157
run: terraform plan
109158
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
110159
env:
111-
OS_CLOUD: "openstack"
160+
OS_CLOUD: ${{ vars.OS_CLOUD }}
112161
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
113162
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
114163

@@ -128,7 +177,7 @@ jobs:
128177
exit 1
129178
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
130179
env:
131-
OS_CLOUD: "openstack"
180+
OS_CLOUD: ${{ vars.OS_CLOUD }}
132181
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
133182
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
134183

@@ -236,15 +285,245 @@ jobs:
236285
source venvs/kayobe/bin/activate &&
237286
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
238287
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/tools/push-ofed.yml \
239-
-e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
288+
-e "ofed_tag=${{ needs.create-tag.outputs.ofed_tag }}"
289+
env:
290+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
291+
292+
- name: Destroy
293+
run: terraform destroy -auto-approve
294+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
295+
env:
296+
OS_CLOUD: ${{ vars.OS_CLOUD }}
297+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
298+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
299+
if: always()
300+
301+
overcloud-ofed-packages-aarch64:
302+
name: Build aarch64 OFED kernel modules
303+
if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.rocky9-aarch64
304+
environment: ${{ inputs.runner_env }}
305+
runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }}
306+
needs:
307+
- runner-selection
308+
- create-tag
309+
permissions: {}
310+
steps:
311+
- name: Install Package dependencies
312+
run: |
313+
sudo apt update &&
314+
sudo apt install -y git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
315+
316+
- name: Start the SSH service
317+
run: |
318+
sudo /etc/init.d/ssh start
319+
320+
- name: Checkout
321+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
322+
with:
323+
path: src/kayobe-config
324+
325+
- name: Install Kayobe
326+
run: |
327+
mkdir -p venvs &&
328+
pushd venvs &&
329+
python3 -m venv kayobe &&
330+
source kayobe/bin/activate &&
331+
pip install -U pip &&
332+
pip install -r ../src/kayobe-config/requirements.txt
333+
334+
- name: Install terraform
335+
uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0
336+
337+
- name: Initialise terraform
338+
run: terraform init
339+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
340+
341+
- name: Generate SSH keypair
342+
run: ssh-keygen -f id_rsa -N ''
343+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
344+
345+
- name: Generate clouds.yaml
346+
run: |
347+
cat << EOF > clouds.yaml
348+
${{ secrets.CLOUDS_YAML }}
349+
EOF
350+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
351+
352+
- name: Output image tag
353+
id: image_tag
354+
run: |
355+
echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version_aarch64: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
356+
working-directory: ${{ github.workspace }}/src/kayobe-config
357+
358+
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
359+
- name: Output image name
360+
id: image_name
361+
run: |
362+
echo image_name=overcloud-rocky-9-aarch64-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
363+
364+
- name: Generate terraform.tfvars
365+
run: |
366+
cat << EOF > terraform.tfvars
367+
ssh_public_key = "id_rsa.pub"
368+
ssh_username = "cloud-user"
369+
aio_vm_name = "skc-ofed-builder-arm64"
370+
aio_vm_image = "${{ env.VM_IMAGE }}"
371+
aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
372+
aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
373+
aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
374+
aio_vm_interface = "ens3"
375+
aio_vm_tags = ${{ env.VM_TAGS }}
376+
EOF
377+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
378+
env:
379+
VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
380+
VM_TAGS: '["skc-ci-doca-builder", "PR=${{ github.event.number }}"]'
381+
382+
- name: Terraform Plan
383+
run: terraform plan
384+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
385+
env:
386+
OS_CLOUD: ${{ vars.OS_CLOUD }}
387+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
388+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
389+
390+
- name: Terraform Apply
391+
run: |
392+
for attempt in $(seq 5); do
393+
if terraform apply -auto-approve; then
394+
echo "Created infrastructure on attempt $attempt"
395+
exit 0
396+
fi
397+
echo "Failed to create infrastructure on attempt $attempt"
398+
sleep 10
399+
terraform destroy -auto-approve
400+
sleep 60
401+
done
402+
echo "Failed to create infrastructure after $attempt attempts"
403+
exit 1
404+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
405+
env:
406+
OS_CLOUD: ${{ vars.OS_CLOUD }}
407+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
408+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
409+
410+
- name: Get Terraform outputs
411+
id: tf_outputs
412+
run: |
413+
terraform output -json
414+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
415+
416+
- name: Write Terraform outputs
417+
run: |
418+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-outputs.yml
419+
${{ steps.tf_outputs.outputs.stdout }}
420+
EOF
421+
422+
- name: Write Terraform network config
423+
run: |
424+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-network-allocation.yml
425+
---
426+
aio_ips:
427+
builder: "{{ access_ip_v4.value }}"
428+
EOF
429+
430+
- name: Write Terraform network interface config
431+
run: |
432+
mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
433+
rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
434+
cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
435+
admin_interface: "{{ access_interface.value }}"
436+
aio_interface: "{{ access_interface.value }}"
437+
EOF
438+
439+
- name: Manage SSH keys
440+
run: |
441+
mkdir -p ~/.ssh
442+
touch ~/.ssh/authorized_keys
443+
cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
444+
cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
445+
446+
- name: Bootstrap the control host
447+
run: |
448+
source venvs/kayobe/bin/activate &&
449+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
450+
kayobe control host bootstrap
451+
452+
- name: Run growroot playbook
453+
run: |
454+
source venvs/kayobe/bin/activate &&
455+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
456+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/tools/growroot.yml
457+
env:
458+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
459+
460+
- name: Configure the seed host (Builder VM)
461+
run: |
462+
source venvs/kayobe/bin/activate &&
463+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
464+
kayobe seed host configure --skip-tags network,docker,docker-registry \
465+
-e kolla_base_arch="aarch64"
466+
env:
467+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
468+
469+
- name: Run a distro-sync
470+
run: |
471+
source venvs/kayobe/bin/activate &&
472+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
473+
kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes" \
474+
-e kolla_base_arch="aarch64"
475+
env:
476+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
477+
478+
- name: Reset BLS entries on the seed host
479+
run: |
480+
source venvs/kayobe/bin/activate &&
481+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
482+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/maintenance/reset-bls-entries.yml \
483+
-e "reset_bls_host=ofed-builder"
484+
env:
485+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
486+
487+
- name: Disable noexec in /var/tmp
488+
run: |
489+
source venvs/kayobe/bin/activate &&
490+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
491+
kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab"
492+
env:
493+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
494+
495+
- name: Reboot to apply the kernel update
496+
run: |
497+
source venvs/kayobe/bin/activate &&
498+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
499+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/maintenance/reboot.yml
500+
env:
501+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
502+
503+
- name: Run OFED builder playbook
504+
run: |
505+
source venvs/kayobe/bin/activate &&
506+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
507+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/tools/build-ofed-rocky.yml \
508+
-e kolla_base_arch="aarch64"
509+
env:
510+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
511+
512+
- name: Run OFED upload playbook
513+
run: |
514+
source venvs/kayobe/bin/activate &&
515+
source src/kayobe-config/kayobe-env --environment ci-doca-builder &&
516+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/tools/push-ofed.yml \
517+
-e "ofed_tag=${{ needs.create-tag.outputs.ofed_tag }}" \
518+
-e kolla_base_arch="aarch64"
240519
env:
241520
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }}
242521

243522
- name: Destroy
244523
run: terraform destroy -auto-approve
245524
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
246525
env:
247-
OS_CLOUD: openstack
526+
OS_CLOUD: ${{ vars.OS_CLOUD }}
248527
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
249528
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
250529
if: always()

‎doc/source/contributor/ofed.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ build-ofed
3131
----------
3232

3333
Currently we only support building Rocky Linux 9 OFED kernel module packages.
34+
The workflow can build packages for ``x86_64`` and ``aarch64``.
3435

3536
The Build OFED module workflow will check that the filesystem is configured (noexec disabled)
3637
to allow the DOCA build script to run. The workflow will also install any necessary dependencies

‎etc/kayobe/ansible/maintenance/rocky-97-ofed-upgrade.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
vars:
1818
# we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
1919
# but not 5.14.0-611.20.1.
20-
doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}"
20+
doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string][ansible_facts.architecture] }}"
2121
tasks:
2222
- name: Assert that hosts are running Rocky Linux 9.6
2323
ansible.builtin.assert:

‎etc/kayobe/ansible/tools/install-doca.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
vars:
77
# we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built,
88
# but not 5.14.0-611.20.1
9-
doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}"
9+
doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string][ansible_facts.architecture] }}"
1010
tasks:
1111
- name: Install kernel repo
1212
ansible.builtin.dnf:

‎etc/kayobe/ansible/tools/push-ofed.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
tasks:
88
- name: Get OFED module repo variables
99
ansible.builtin.set_fact:
10-
doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='name') | join('') }}"
11-
doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='base_path') | join('') }}"
12-
doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='distribution_name') | join('') }}"
10+
doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | selectattr('base_path', 'search', '/' ~ kolla_base_arch ~ '/') | map(attribute='name') | join('') }}"
11+
doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | selectattr('base_path', 'search', '/' ~ kolla_base_arch ~ '/') | map(attribute='base_path') | join('') }}"
12+
doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | selectattr('base_path', 'search', '/' ~ kolla_base_arch ~ '/') | map(attribute='distribution_name') | join('') }}"
1313

1414
- name: Install python dependencies
1515
ansible.builtin.pip:

0 commit comments

Comments
 (0)