diff --git a/.github/workflows/cross-build-azl3-arm-raw.yml b/.github/workflows/cross-build-azl3-arm-raw.yml new file mode 100644 index 000000000..ac6874897 --- /dev/null +++ b/.github/workflows/cross-build-azl3-arm-raw.yml @@ -0,0 +1,168 @@ +name: Cross Build Azure Linux Raw Image - ARM +on: + workflow_dispatch: # Manual runs + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - fix-broken-arm64-builds + pull_request: + branches: + - main +permissions: + contents: read + +jobs: + cross-build-azl3-arm-raw: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + #update-binfmts --display qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build AZL3 AArch64 Raw image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/azl3-aarch64-edge-raw.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + - name: GitHub Upload Release Artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: azl3-arm-raw-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/.github/workflows/cross-build-debian13-aarch64-minimal-raw.yml b/.github/workflows/cross-build-debian13-aarch64-minimal-raw.yml new file mode 100644 index 000000000..d0a7c0655 --- /dev/null +++ b/.github/workflows/cross-build-debian13-aarch64-minimal-raw.yml @@ -0,0 +1,169 @@ +name: Cross Build Debian13 AArch64 Minimal Raw +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + cross-build-debian13-aarch64-minimal-raw: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build Debian13 AArch64 Minimal Raw image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/debian13-aarch64-minimal-raw.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: debian13-aarch64-minimal-raw-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/.github/workflows/cross-build-elxr12-aarch64-minimal-raw.yml b/.github/workflows/cross-build-elxr12-aarch64-minimal-raw.yml new file mode 100644 index 000000000..18f07832f --- /dev/null +++ b/.github/workflows/cross-build-elxr12-aarch64-minimal-raw.yml @@ -0,0 +1,169 @@ +name: Cross Build ELXR12 AArch64 Minimal Raw +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + cross-build-elxr12-aarch64-minimal-raw: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build ELXR12 AArch64 Minimal Raw image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/elxr12-aarch64-minimal-raw.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: elxr12-aarch64-minimal-raw-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/.github/workflows/cross-build-ubuntu24-aarch64-edge-raw.yml b/.github/workflows/cross-build-ubuntu24-aarch64-edge-raw.yml new file mode 100644 index 000000000..95c263e95 --- /dev/null +++ b/.github/workflows/cross-build-ubuntu24-aarch64-edge-raw.yml @@ -0,0 +1,169 @@ +name: Cross Build Ubuntu24 AArch64 Edge Raw +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + cross-build-ubuntu24-aarch64-edge-raw: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build Ubuntu24 AArch64 Edge Raw image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/ubuntu24-aarch64-edge-raw.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: ubuntu24-aarch64-edge-raw-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/.github/workflows/cross-build-ubuntu24-aarch64-minimal-raw.yml b/.github/workflows/cross-build-ubuntu24-aarch64-minimal-raw.yml new file mode 100644 index 000000000..c8abb1c0e --- /dev/null +++ b/.github/workflows/cross-build-ubuntu24-aarch64-minimal-raw.yml @@ -0,0 +1,169 @@ +name: Cross Build Ubuntu24 AArch64 Minimal Raw +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + cross-build-ubuntu24-aarch64-minimal-raw: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build Ubuntu24 AArch64 Minimal Raw image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/ubuntu24-aarch64-minimal-raw.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: ubuntu24-aarch64-minimal-raw-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/.github/workflows/cross-build-ubuntu24-aarch64-minimal-uki.yml b/.github/workflows/cross-build-ubuntu24-aarch64-minimal-uki.yml new file mode 100644 index 000000000..6a69e80ba --- /dev/null +++ b/.github/workflows/cross-build-ubuntu24-aarch64-minimal-uki.yml @@ -0,0 +1,169 @@ +name: Cross Build Ubuntu24 AArch64 Minimal UKI +on: + workflow_dispatch: + inputs: + ref: + description: "Branch or SHA to test (e.g. feature/x or a1b2c3)" + required: false + run_qemu_test: + description: "Run QEMU boot test after build" + required: false + default: "false" + type: choice + options: + - "true" + - "false" + push: + branches: + - main + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + cross-build-ubuntu24-aarch64-minimal-uki: + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + with: + persist-credentials: false + ref: ${{ github.event.inputs.ref || github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd #v4.0.0 + + - name: Install Earthly + uses: earthly/actions-setup@43211c7a0eae5344d6d79fb4aaf209c8f8866203 #v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + version: "latest" + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-aarch64 qemu-user-static binfmt-support ovmf tree jq systemd-ukify mmdebstrap systemd-boot qemu-efi-aarch64 + sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc || true + sudo update-binfmts --enable qemu-aarch64 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c #v6.4.0 + with: + go-version: stable + + - name: Build image-composer-tool + run: go build ./cmd/image-composer-tool + + - name: Build Ubuntu24 AArch64 Minimal UKI image + run: | + sudo -E ./image-composer-tool --verbose build image-templates/ubuntu24-aarch64-minimal-uki.yml + + - name: Run QEMU boot test + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.run_qemu_test == 'true' }} + run: | + set -euo pipefail + FOUND_PATH=$(find workspace -type f \( -name "*.raw.gz" -o -name "*.raw" \) | head -n 1) + if [ -z "$FOUND_PATH" ]; then + echo "No raw image artifact found under workspace/" + exit 1 + fi + + TEST_IMAGE="/tmp/test-arm-image.raw" + if [[ "$FOUND_PATH" == *.gz ]]; then + gunzip -c "$FOUND_PATH" > "$TEST_IMAGE" + else + cp -f "$FOUND_PATH" "$TEST_IMAGE" + fi + + CODE_FD="" + VARS_FD="" + BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in /usr/share/AAVMF/AAVMF_VARS.fd /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files" + exit 1 + fi + + LOGFILE="qemu-boot.log" + SUCCESS_STRING="login:" + TIMEOUT=60 + + QEMU_ARGS=( + -machine virt,accel=tcg + -cpu max + -m 2048 + -drive if=none,file="$TEST_IMAGE",format=raw,id=nvme0 + -device nvme,drive=nvme0,serial=deadbeef + -nographic + -serial mon:stdio + ) + + if [ -n "$CODE_FD" ] && [ -n "$VARS_FD" ]; then + VARS_WORK="/tmp/AAVMF_VARS.fd" + cp -f "$VARS_FD" "$VARS_WORK" + QEMU_ARGS+=( + -drive if=pflash,format=raw,readonly=on,file="$CODE_FD" + -drive if=pflash,format=raw,file="$VARS_WORK" + ) + else + QEMU_ARGS+=(-bios "$BIOS_FD") + fi + + sudo nohup qemu-system-aarch64 "${QEMU_ARGS[@]}" > "$LOGFILE" 2>&1 & + QEMU_PID=$! + + elapsed=0 + while ! grep -q "$SUCCESS_STRING" "$LOGFILE" && [ $elapsed -lt $TIMEOUT ]; do + sleep 1 + elapsed=$((elapsed + 1)) + done + + sudo kill "$QEMU_PID" 2>/dev/null || true + sudo pkill -f "qemu-system-aarch64.*$TEST_IMAGE" 2>/dev/null || true + + if ! grep -q "$SUCCESS_STRING" "$LOGFILE"; then + echo "QEMU boot test failed or timed out" + tail -n 200 "$LOGFILE" || true + exit 1 + fi + + echo "QEMU boot test passed" + rm -f "$TEST_IMAGE" /tmp/AAVMF_VARS.fd + + - name: Set file permissions for artifacts + run: | + sudo chmod -R 755 workspace/ || true + find workspace/*/imagebuild/*/ -name "*.raw*" -exec chmod 777 {} \; || true + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + name: ubuntu24-aarch64-minimal-uki-image + path: | + workspace/*/imagebuild/*/*.raw* + retention-days: 30 diff --git a/internal/image/imageboot/imageboot_test.go b/internal/image/imageboot/imageboot_test.go index f4fbac98c..6614554fc 100644 --- a/internal/image/imageboot/imageboot_test.go +++ b/internal/image/imageboot/imageboot_test.go @@ -1567,13 +1567,31 @@ func TestInstallImageBoot_GrubWithEnableExtraModulesUbuntu(t *testing.T) { } } +type noSudoExecutor struct{} + +func (e *noSudoExecutor) ExecCmd(cmdStr string, _ bool, chrootPath string, envVal []string) (string, error) { + return (&shell.DefaultExecutor{}).ExecCmd(cmdStr, false, chrootPath, envVal) +} + +func (e *noSudoExecutor) ExecCmdSilent(cmdStr string, _ bool, chrootPath string, envVal []string) (string, error) { + return (&shell.DefaultExecutor{}).ExecCmdSilent(cmdStr, false, chrootPath, envVal) +} + +func (e *noSudoExecutor) ExecCmdWithStream(cmdStr string, _ bool, chrootPath string, envVal []string) (string, error) { + return (&shell.DefaultExecutor{}).ExecCmdWithStream(cmdStr, false, chrootPath, envVal) +} + +func (e *noSudoExecutor) ExecCmdWithInput(inputStr string, cmdStr string, _ bool, chrootPath string, envVal []string) (string, error) { + return (&shell.DefaultExecutor{}).ExecCmdWithInput(inputStr, cmdStr, false, chrootPath, envVal) +} + func TestUpdateBootConfigTemplate_GrubCmdlineRootAndExtraArgs(t *testing.T) { originalGlobal := config.Global() t.Cleanup(func() { config.SetGlobal(originalGlobal) }) originalExecutor := shell.Default - shell.Default = &shell.DefaultExecutor{} + shell.Default = &noSudoExecutor{} t.Cleanup(func() { shell.Default = originalExecutor }) diff --git a/internal/provider/azl/azl.go b/internal/provider/azl/azl.go index 66579c3e7..997222c8e 100644 --- a/internal/provider/azl/azl.go +++ b/internal/provider/azl/azl.go @@ -226,13 +226,15 @@ func (p *AzureLinux) PostProcess(template *config.ImageTemplate, err error) erro func (p *AzureLinux) installHostDependency() error { var dependencyInfo = map[string]string{ - "rpm": "rpm", // For the chroot env build RPM pkg installation - "mkfs.fat": "dosfstools", // For the FAT32 boot partition creation - "qemu-img": "qemu-utils", // For image file format conversion - "mformat": "mtools", // For writing files to FAT32 partition - "xorriso": "xorriso", // For ISO image creation - "grub-mkimage": "grub-common", // For ISO image UEFI Grub binary creation - "sbsign": "sbsigntool", // For the UKI image creation + "rpm": "rpm", // For the chroot env build RPM pkg installation + "qemu-user-static": "qemu-user-static", // For cross-architecture binary execution support + "update-binfmts": "binfmt-support", // For registering qemu-user-static with the kernel + "mkfs.fat": "dosfstools", // For the FAT32 boot partition creation + "qemu-img": "qemu-utils", // For image file format conversion + "mformat": "mtools", // For writing files to FAT32 partition + "xorriso": "xorriso", // For ISO image creation + "grub-mkimage": "grub-common", // For ISO image UEFI Grub binary creation + "sbsign": "sbsigntool", // For the UKI image creation } hostPkgManager, err := system.GetHostOsPkgManager() if err != nil { @@ -254,6 +256,21 @@ func (p *AzureLinux) installHostDependency() error { log.Debugf("Host dependency %s is already installed", pkg) } } + + // Ensure arm64 binfmt handler is active on x86_64 hosts for cross-arch chroot execution. + hostInfo, err := system.GetHostOsInfo() + if err != nil { + return fmt.Errorf("failed to get host OS information: %w", err) + } + if hostInfo["arch"] == "x86_64" { + if _, err := shell.ExecCmd("mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null || true", true, shell.HostPath, nil); err != nil { + log.Debugf("binfmt_misc mount attempt: %v", err) + } + if _, err := shell.ExecCmd("update-binfmts --enable qemu-aarch64", true, shell.HostPath, nil); err != nil { + return fmt.Errorf("failed to enable qemu-aarch64 binfmt handler: %w", err) + } + } + return nil } diff --git a/internal/provider/azl/azl_test.go b/internal/provider/azl/azl_test.go index da145e0dd..8c1bfea44 100644 --- a/internal/provider/azl/azl_test.go +++ b/internal/provider/azl/azl_test.go @@ -716,8 +716,17 @@ func TestAzlPreProcess(t *testing.T) { t.Skip("PreProcess requires proper Azure Linux initialization with chrootEnv - function exists and is callable") } +func requireHostDependencyTestOptIn(t *testing.T) { + t.Helper() + if os.Getenv("ICT_RUN_HOST_DEP_TESTS") != "1" { + t.Skip("Skipping host dependency tests by default; set ICT_RUN_HOST_DEP_TESTS=1 to enable") + } +} + // TestAzlInstallHostDependency tests the installHostDependency function func TestAzlInstallHostDependency(t *testing.T) { + requireHostDependencyTestOptIn(t) + azl := &AzureLinux{} // Test that the function exists and can be called @@ -743,6 +752,8 @@ func TestAzlDownloadImagePkgs(t *testing.T) { // TestAzlPreProcessWithMockEnv tests PreProcess with proper mock chrootEnv func TestAzlPreProcessWithMockEnv(t *testing.T) { + requireHostDependencyTestOptIn(t) + azl := &AzureLinux{ chrootEnv: &mockChrootEnv{}, } @@ -951,6 +962,8 @@ func TestRegisterSuccess(t *testing.T) { // TestInstallHostDependencyMapping tests the dependency mapping logic func TestInstallHostDependencyMapping(t *testing.T) { + requireHostDependencyTestOptIn(t) + azl := &AzureLinux{} // Call installHostDependency @@ -1041,6 +1054,8 @@ func TestBuildImageEdgeCases(t *testing.T) { // TestPreProcessErrorPropagation tests error propagation in PreProcess func TestPreProcessErrorPropagation(t *testing.T) { + requireHostDependencyTestOptIn(t) + azl := &AzureLinux{ chrootEnv: &mockChrootEnv{}, } diff --git a/internal/utils/shell/shell.go b/internal/utils/shell/shell.go index 7ef816b4b..c92f2b8d3 100644 --- a/internal/utils/shell/shell.go +++ b/internal/utils/shell/shell.go @@ -124,6 +124,7 @@ var commandMap = map[string][]string{ "ukify": {"/usr/bin/ukify", "/usr/local/bin/ukify"}, "umount": {"/usr/bin/umount"}, "uname": {"/usr/bin/uname"}, + "update-binfmts": {"/usr/sbin/update-binfmts", "/usr/bin/update-binfmts"}, "uniq": {"/usr/bin/uniq"}, "veritysetup": {"/usr/sbin/veritysetup"}, "vgcreate": {"/usr/sbin/vgcreate"}, @@ -449,7 +450,6 @@ func verifyCmdWithFullPath(cmd, chrootPath string) (string, error) { func GetFullCmdStr(cmdStr string, sudo bool, chrootPath string, envVal []string) (string, error) { var fullCmdStr string envValStr := "" - runningAsRoot := os.Geteuid() == 0 for _, env := range envVal { envValStr += env + " " } @@ -470,11 +470,7 @@ func GetFullCmdStr(cmdStr string, sudo bool, chrootPath string, envVal []string) envValStr += key + "=" + value + " " } - if sudo && !runningAsRoot { - fullCmdStr = "sudo " + envValStr + "chroot " + chrootPath + " " + fullPathCmdStr - } else { - fullCmdStr = envValStr + "chroot " + chrootPath + " " + fullPathCmdStr - } + fullCmdStr = "sudo " + envValStr + "chroot " + chrootPath + " " + fullPathCmdStr chrootDir := filepath.Base(chrootPath) // log.Debugf("Chroot " + chrootDir + " Exec: [" + fullPathCmdStr + "]") // Avoid logging full command string to prevent leaking sensitive data. @@ -488,15 +484,10 @@ func GetFullCmdStr(cmdStr string, sudo bool, chrootPath string, envVal []string) envValStr += key + "=" + value + " " } - if runningAsRoot { - fullCmdStr = envValStr + fullPathCmdStr - log.Debugf("Exec without sudo: [already running as root]") - } else { - fullCmdStr = "sudo " + envValStr + fullPathCmdStr - // log.Debugf("Exec: [sudo " + fullPathCmdStr + "]") - // Avoid logging full command string to prevent leaking sensitive data. - log.Debugf("Exec with sudo: [command executed]") - } + fullCmdStr = "sudo " + envValStr + fullPathCmdStr + // log.Debugf("Exec: [sudo " + fullPathCmdStr + "]") + // Avoid logging full command string to prevent leaking sensitive data. + log.Debugf("Exec with sudo: [command executed]") } else { fullCmdStr = fullPathCmdStr // log.Debugf("Exec: [" + fullPathCmdStr + "]") diff --git a/internal/utils/shell/shell_retry_internal_test.go b/internal/utils/shell/shell_retry_internal_test.go index f67037edf..4bcd38648 100644 --- a/internal/utils/shell/shell_retry_internal_test.go +++ b/internal/utils/shell/shell_retry_internal_test.go @@ -49,3 +49,45 @@ echo "updated" t.Fatalf("expected state file to exist after first failed attempt, got: %v", err) } } + +func TestExecCmdWithStream_RetriesAptGetLock(t *testing.T) { + tempDir := t.TempDir() + stateFile := filepath.Join(tempDir, "apt-get-state") + fakeAptGetPath := filepath.Join(tempDir, "apt-get") + script := fmt.Sprintf(`#!/bin/sh +if [ ! -f %q ]; then + touch %q + echo "E: Could not get lock /var/lib/dpkg/lock-frontend. It is held by process 913 (apt-get)" 1>&2 + echo "E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?" 1>&2 + exit 100 +fi +echo "apt-get-updated" +`, stateFile, stateFile) + if err := os.WriteFile(fakeAptGetPath, []byte(script), 0755); err != nil { + t.Fatalf("failed to write fake apt-get script: %v", err) + } + + originalAptGetPaths := commandMap["apt-get"] + originalAttempts := aptLockRetryAttempts + originalDelay := aptLockRetryDelay + defer func() { + commandMap["apt-get"] = originalAptGetPaths + aptLockRetryAttempts = originalAttempts + aptLockRetryDelay = originalDelay + }() + + commandMap["apt-get"] = []string{fakeAptGetPath} + aptLockRetryAttempts = 2 + aptLockRetryDelay = 10 * time.Millisecond + + output, err := (&DefaultExecutor{}).ExecCmdWithStream("apt-get update", false, HostPath, nil) + if err != nil { + t.Fatalf("expected apt-get lock retry to succeed, got: %v", err) + } + if output != "apt-get-updated\n" { + t.Fatalf("expected output to contain final successful apt-get output, got %q", output) + } + if _, err := os.Stat(stateFile); err != nil { + t.Fatalf("expected state file to exist after first failed attempt, got: %v", err) + } +} diff --git a/internal/utils/shell/shell_test.go b/internal/utils/shell/shell_test.go index c01470153..c73d4bf01 100644 --- a/internal/utils/shell/shell_test.go +++ b/internal/utils/shell/shell_test.go @@ -347,23 +347,6 @@ func TestGetFullCmdStr_CommandNotFoundInChroot(t *testing.T) { } } -func TestGetFullCmdStr_Sudo(t *testing.T) { - cmd := "ls" - fullCmd, err := shell.GetFullCmdStr(cmd, true, shell.HostPath, nil) - if err != nil { - t.Fatalf("GetFullCmdStr failed with sudo: %v", err) - } - if os.Geteuid() == 0 { - if strings.Contains(fullCmd, "sudo") { - t.Errorf("Expected sudo to be omitted when already root, got: %s", fullCmd) - } - return - } - if !strings.Contains(fullCmd, "sudo") { - t.Errorf("Expected sudo in command when not root, got: %s", fullCmd) - } -} - func TestGetFullCmdStr_Env(t *testing.T) { cmd := "ls" env := []string{"VAR=VALUE"} diff --git a/scripts/cross-build_azl3_arm_raw.sh b/scripts/cross-build_azl3_arm_raw.sh new file mode 100644 index 000000000..0a06f199e --- /dev/null +++ b/scripts/cross-build_azl3_arm_raw.sh @@ -0,0 +1,312 @@ +#!/bin/bash +set -e + +# Parse command line arguments +RUN_QEMU_TESTS=false +WORKING_DIR="$(pwd)" + +while [[ $# -gt 0 ]]; do + case $1 in + --qemu-test|--with-qemu) + RUN_QEMU_TESTS=true + shift + ;; + --working-dir) + WORKING_DIR="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [--qemu-test|--with-qemu] [--working-dir DIR]" + echo " --qemu-test, --with-qemu Run QEMU boot tests after image build" + echo " --working-dir DIR Set the working directory" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +# Centralized cleanup function for image files +cleanup_image_files() { + local cleanup_type="${1:-all}" # Options: all, raw, extracted + + case "$cleanup_type" in + "raw") + echo "Cleaning up raw image files from build directories..." + sudo rm -rf ./tmp/*/imagebuild/*/*.raw 2>/dev/null || true + sudo rm -rf ./workspace/*/imagebuild/*/*.raw 2>/dev/null || true + ;; + "extracted") + echo "Cleaning up extracted image files in current directory..." + rm -f *.raw 2>/dev/null || true + ;; + "all"|*) + echo "Cleaning up all temporary image files..." + sudo rm -rf ./tmp/*/imagebuild/*/*.raw 2>/dev/null || true + sudo rm -rf ./workspace/*/imagebuild/*/*.raw 2>/dev/null || true + rm -f *.raw 2>/dev/null || true + ;; + esac +} + +run_qemu_boot_test() { + local IMAGE_PATTERN="$1" + if [ -z "$IMAGE_PATTERN" ]; then + echo "Error: Image pattern not provided to run_qemu_boot_test" + return 1 + fi + + # Prefer AArch64 firmware files and fall back to generic qemu-efi image if needed. + local CODE_FD="" + local VARS_FD="" + local BIOS_FD="" + for code_path in \ + /usr/share/AAVMF/AAVMF_CODE.fd \ + /usr/share/AAVMF/AAVMF_CODE.ms.fd \ + /usr/share/qemu-efi-aarch64/QEMU_EFI.fd; do + if [ -f "$code_path" ]; then + if [[ "$code_path" == *QEMU_EFI.fd ]]; then + BIOS_FD="$code_path" + else + CODE_FD="$code_path" + fi + break + fi + done + + if [ -n "$CODE_FD" ]; then + for vars_path in \ + /usr/share/AAVMF/AAVMF_VARS.fd \ + /usr/share/AAVMF/AAVMF_VARS.ms.fd; do + if [ -f "$vars_path" ]; then + VARS_FD="$vars_path" + break + fi + done + fi + + if [ -z "$BIOS_FD" ] && { [ -z "$CODE_FD" ] || [ -z "$VARS_FD" ]; }; then + echo "Unable to find ARM UEFI firmware files for qemu-system-aarch64" + echo "Checked AAVMF CODE/VARS and qemu-efi-aarch64 QEMU_EFI.fd paths" + return 1 + fi + + local QEMU_MACHINE="virt,accel=tcg" + local QEMU_CPU="max" + TIMEOUT=30 + SUCCESS_STRING="login:" + LOGFILE="qemu_serial.log" + + ORIGINAL_DIR=$(pwd) + # Find compressed raw image path using pattern, handle permission issues + FOUND_PATH=$(sudo -S find . -type f -name "*${IMAGE_PATTERN}*.raw.gz" 2>/dev/null | head -n 1) + if [ -n "$FOUND_PATH" ]; then + echo "Found compressed image at: $FOUND_PATH" + IMAGE_DIR=$(dirname "$FOUND_PATH") + + # Fix permissions for the image directory recursively to allow access + IMAGE_ROOT_DIR=$(echo "$IMAGE_DIR" | cut -d'/' -f2) # Get the root directory (workspace or tmp) + echo "Setting permissions recursively for ./$IMAGE_ROOT_DIR directory" + sudo chmod -R 777 "./$IMAGE_ROOT_DIR" + + cd "$IMAGE_DIR" + + # Extract the .raw.gz file + COMPRESSED_IMAGE=$(basename "$FOUND_PATH") + RAW_IMAGE="${COMPRESSED_IMAGE%.gz}" + echo "Extracting $COMPRESSED_IMAGE to $RAW_IMAGE..." + + # Check available disk space before extraction + AVAILABLE_SPACE=$(df . | tail -1 | awk '{print $4}') + COMPRESSED_SIZE=$(stat -c%s "$COMPRESSED_IMAGE" 2>/dev/null || echo "0") + # Estimate uncompressed size (typically 4-6x larger for these images, being conservative) + ESTIMATED_SIZE=$((COMPRESSED_SIZE * 6 / 1024)) + + echo "Disk space check: Available=${AVAILABLE_SPACE}KB, Estimated needed=${ESTIMATED_SIZE}KB" + + # Always try aggressive cleanup first to ensure maximum space + echo "Performing aggressive cleanup before extraction..." + sudo rm -f *.raw 2>/dev/null || true + sudo rm -f /tmp/*.raw 2>/dev/null || true + sudo rm -rf ../../../cache/ 2>/dev/null || true + sudo rm -rf ../../../tmp/*/imagebuild/*/*.raw 2>/dev/null || true + sudo rm -rf ../../../workspace/*/imagebuild/*/*.raw 2>/dev/null || true + + # Force filesystem sync and check space again + sync + AVAILABLE_SPACE=$(df . | tail -1 | awk '{print $4}') + echo "Available space after cleanup: ${AVAILABLE_SPACE}KB" + + if [ "$AVAILABLE_SPACE" -lt "$ESTIMATED_SIZE" ]; then + echo "Warning: Still insufficient disk space after cleanup" + echo "Attempting extraction to /tmp with streaming..." + + # Check /tmp space + TMP_AVAILABLE=$(df /tmp | tail -1 | awk '{print $4}') + echo "/tmp available space: ${TMP_AVAILABLE}KB" + + if [ "$TMP_AVAILABLE" -gt "$ESTIMATED_SIZE" ]; then + TMP_RAW="/tmp/$RAW_IMAGE" + echo "Extracting to /tmp first..." + if gunzip -c "$COMPRESSED_IMAGE" > "$TMP_RAW"; then + echo "Successfully extracted to /tmp, moving to final location..." + if mv "$TMP_RAW" "$RAW_IMAGE"; then + echo "Successfully moved extracted image to current directory" + else + echo "Failed to move from /tmp, will try to use /tmp location directly" + ln -sf "$TMP_RAW" "$RAW_IMAGE" 2>/dev/null || cp "$TMP_RAW" "$RAW_IMAGE" + fi + else + echo "Failed to extract to /tmp" + rm -f "$TMP_RAW" 2>/dev/null || true + return 1 + fi + else + echo "ERROR: Insufficient space in both current directory and /tmp" + echo "Current: ${AVAILABLE_SPACE}KB, /tmp: ${TMP_AVAILABLE}KB, Needed: ${ESTIMATED_SIZE}KB" + return 1 + fi + else + echo "Sufficient space available, extracting directly..." + if ! gunzip -c "$COMPRESSED_IMAGE" > "$RAW_IMAGE"; then + echo "Direct extraction failed, cleaning up partial file..." + rm -f "$RAW_IMAGE" 2>/dev/null || true + return 1 + fi + fi + + if [ ! -f "$RAW_IMAGE" ]; then + echo "Failed to extract image!" + # Clean up any partially extracted files + sudo rm -f "$RAW_IMAGE" /tmp/"$RAW_IMAGE" 2>/dev/null || true + cd "$ORIGINAL_DIR" + return 1 + fi + + IMAGE="$RAW_IMAGE" + else + echo "Compressed raw image file matching pattern '*${IMAGE_PATTERN}*.raw.gz' not found!" + return 1 + fi + + + echo "Booting image: $IMAGE " + #create log file ,boot image into qemu , return the pass or fail after boot sucess + sudo bash -c " + LOGFILE=\"$LOGFILE\" + SUCCESS_STRING=\"$SUCCESS_STRING\" + TIMEOUT=\"$TIMEOUT\" + IMAGE=\"$IMAGE\" + RAW_IMAGE=\"$RAW_IMAGE\" + ORIGINAL_DIR=\"$ORIGINAL_DIR\" + CODE_FD=\"$CODE_FD\" + VARS_FD=\"$VARS_FD\" + BIOS_FD=\"$BIOS_FD\" + QEMU_MACHINE=\"$QEMU_MACHINE\" + QEMU_CPU=\"$QEMU_CPU\" + #-enable-kvm \\ + + touch \"\$LOGFILE\" && chmod 666 \"\$LOGFILE\" + + qemu_cmd=\"qemu-system-aarch64 \\ + -machine \$QEMU_MACHINE \\ + -m 2048 \\ + -cpu \$QEMU_CPU \\ + -drive if=none,file=\"\$IMAGE\",format=raw,id=nvme0 \\ + -device nvme,drive=nvme0,serial=deadbeef \\ + -nographic \\ + -serial mon:stdio\" + + if [ -n \"\$CODE_FD\" ] && [ -n \"\$VARS_FD\" ]; then + qemu_cmd=\"\$qemu_cmd -drive if=pflash,format=raw,readonly=on,file=\"\$CODE_FD\" -drive if=pflash,format=raw,file=\"\$VARS_FD\"\" + else + qemu_cmd=\"\$qemu_cmd -bios \"\$BIOS_FD\"\" + fi + + nohup sh -c \"\$qemu_cmd\" > \"\$LOGFILE\" 2>&1 & + + qemu_pid=\$! + echo \"QEMU launched as root with PID \$qemu_pid\" + echo \"Current working dir: \$(pwd)\" + + # Wait for SUCCESS_STRING or timeout + elapsed=0 + while ! grep -q \"\$SUCCESS_STRING\" \"\$LOGFILE\" && [ \$elapsed -lt \$TIMEOUT ]; do + sleep 1 + elapsed=\$((elapsed + 1)) + done + echo \"\$elapsed\" + kill \$qemu_pid + cat \"\$LOGFILE\" + + if grep -q \"\$SUCCESS_STRING\" \"\$LOGFILE\"; then + echo \"Boot success!\" + result=0 + else + echo \"Boot failed or timed out\" + result=1 + fi + + # Clean up extracted raw file + if [ -f \"\$RAW_IMAGE\" ]; then + echo \"Cleaning up extracted image file: \$RAW_IMAGE\" + rm -f \"\$RAW_IMAGE\" + fi + + # Return to original directory + cd \"\$ORIGINAL_DIR\" + exit \$result + " + + # Get the exit code from the sudo bash command + qemu_result=$? + return $qemu_result +} + +git branch +#Build the ICT +echo "Building the ICT..." +echo "Generating binary with go build..." +go build ./cmd/image-composer-tool + +build_azl3_raw_image() { + echo "Building AZL3 raw Image for ARM64. (using image-composer-tool binary)" + # Ensure we're in the working directory before starting builds + echo "Ensuring we're in the working directory before starting builds..." + cd "$WORKING_DIR" + echo "Current working directory: $(pwd)" + + # Temporarily disable exit on error for the build command to capture output + set +e + output=$( sudo -S ./image-composer-tool --verbose build image-templates/azl3-aarch64-edge-raw.yml 2>&1) + build_exit_code=$? + set -e + + # Check for the success message in the output + if [ $build_exit_code -eq 0 ] && echo "$output" | grep -q "image build completed successfully"; then + echo "AZL3 raw Image build passed." + if [ "$RUN_QEMU_TESTS" = true ]; then + echo "Running QEMU boot test for AZL3 raw image..." + if run_qemu_boot_test "azl3-aarch64-edge"; then + echo "QEMU boot test PASSED for AZL3 raw image" + else + echo "QEMU boot test FAILED for AZL3 raw image" + exit 1 + fi + # Clean up after QEMU test to free space + cleanup_image_files raw + fi + else + echo "AZL3 raw Image build failed." + echo "Build output:" + echo "$output" + exit 1 # Exit with error if build fails + fi +} + +# Run the main function +build_azl3_raw_image