ci.yml

# CI Workflow for bootc
#
# Core principles:
# - Everything done here should be easy to replicate locally. Most tasks
#   should invoke `just <something>`. Read the Justfile for more explanation
#   of this.
# - Most additions to this should be extending existing tasks; e.g.
#   there's places for unit and integration tests already.
name: CI

permissions:
  actions: read
  contents: read

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
    types: [opened, synchronize, reopened, labeled]
  merge_group:
  workflow_dispatch: {}

env:
  CARGO_TERM_COLOR: always
  # Something seems to be setting this in the default GHA runners, which breaks bcvk
  # as the default runner user doesn't have access
  LIBVIRT_DEFAULT_URI: "qemu:///session"
  DEV_IMAGE: ghcr.io/bootc-dev/dev-bootc
  # Retry parameters for `just build-fetch` (transient Koji/Copr/quay.io failures)
  BOOTC_CI_RETRIES: "10"
  BOOTC_CI_DELAY: "60"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  # Determine the OS matrix for CI jobs based on context:
  #   - merge_group / workflow_dispatch / ci/merge label: all OSes
  #   - ci/tier-1 label: centos-10 only (fast feedback on the primary target)
  #   - plain PR: no heavy jobs
  compute-ci-level:
    runs-on: ubuntu-24.04
    outputs:
      package_os_matrix: ${{ steps.matrix.outputs.package_os_matrix }}
      integration_os_matrix: ${{ steps.matrix.outputs.integration_os_matrix }}
      upgrade_os_matrix: ${{ steps.matrix.outputs.upgrade_os_matrix }}
      run_heavy: ${{ steps.matrix.outputs.run_heavy }}
    steps:
      - name: Compute OS matrices
        id: matrix
        run: |
          LABELS='${{ toJson(github.event.pull_request.labels.*.name) }}'
          EVENT='${{ github.event_name }}'

          if [[ "$EVENT" == "merge_group" || "$EVENT" == "workflow_dispatch" ]] \
              || echo "$LABELS" | jq -e 'index("ci/merge")' > /dev/null; then
            # Full suite: all OSes
            echo 'package_os_matrix=["fedora-43","fedora-44","fedora-45","centos-9","centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'integration_os_matrix=["fedora-43","fedora-44","centos-9","centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'upgrade_os_matrix=["fedora-44","centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'run_heavy=true' >> "$GITHUB_OUTPUT"
          elif echo "$LABELS" | jq -e 'index("ci/tier-1")' > /dev/null; then
            # Tier-1 only: centos-10
            echo 'package_os_matrix=["centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'integration_os_matrix=["centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'upgrade_os_matrix=["centos-10"]' >> "$GITHUB_OUTPUT"
            echo 'run_heavy=true' >> "$GITHUB_OUTPUT"
          else
            # Plain PR: skip heavy jobs entirely
            echo 'package_os_matrix=[]' >> "$GITHUB_OUTPUT"
            echo 'integration_os_matrix=[]' >> "$GITHUB_OUTPUT"
            echo 'upgrade_os_matrix=[]' >> "$GITHUB_OUTPUT"
            echo 'run_heavy=false' >> "$GITHUB_OUTPUT"
          fi

  # Run basic validation checks (linting, formatting, etc)
  validate:
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
      - name: Validate (default)
        run: just validate
  # Check for security vulnerabilities and license compliance
  cargo-deny:
    runs-on: ubuntu-24.04
    steps:
    - uses: actions/checkout@v7
    - uses: EmbarkStudios/cargo-deny-action@v2
      with:
        log-level: warn
        command: check -A duplicate bans sources licenses
  # Test bootc installation scenarios and fsverity support
  # TODO convert to be an integration test: this would also
  # ensure we inherit the `package` job (and in theory
  # this could be a proper matrix)
  install-tests:
    name: "Test install"
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: compute-ci-level
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
      - name: Enable fsverity for /
        run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
      - name: Install utils
        run: sudo apt -y install fsverity just
      - name: Fetch external dependencies (with retry)
        run: just build-fetch

      - name: Integration tests
        run: |
          set -xeu
          # Build images as regular user, then copy to root's podman storage
          # This avoids cargo cache permission issues when running cargo as root
          just build
          just build-install-test-image
          just copy-to-rootful localhost/bootc
          just copy-to-rootful localhost/bootc-install
          # Copy bound images (LBI) to root's storage for tests that need them
          just copy-lbi-to-rootful

          # Build test binaries before any sudo commands to avoid cargo permission issues
          cargo build --release -p tests-integration

          sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity

          df -h /
          sudo install -m 0755 target/release/tests-integration /usr/bin/bootc-integration-tests
          sudo rm target -rf
          df -h /
          # The ostree-container tests
          sudo podman run --privileged --pid=host -v /:/run/host -v $(pwd):/src:ro -v /var/tmp:/var/tmp \
            --tmpfs /var/lib/containers \
            -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/crates/ostree-ext/ci/priv-integration.sh
          # Nondestructive but privileged tests
          sudo bootc-integration-tests host-privileged localhost/bootc-install
          # Install tests
          sudo bootc-integration-tests install-alongside localhost/bootc-install

          # inspect system state after the install tests.
          sudo lsblk
          sudo mount
          
          # system-reinstall-bootc tests
          cargo build --release -p system-reinstall-bootc

          # not sure why this is missing in the ubuntu image but just creating this directory allows the tests to pass
          sudo mkdir -p /run/sshd

          sudo install -m 0755 target/release/system-reinstall-bootc /usr/bin/system-reinstall-bootc
          # These tests may mutate the system live so we can't run in parallel
          sudo bootc-integration-tests system-reinstall localhost/bootc --test-threads=1

          # And the fsverity case
          sudo podman run --privileged --pid=host localhost/bootc-fsverity bootc install to-existing-root --stateroot=other \
            --acknowledge-destructive --skip-fetch-check
          # Crude cross check
          sudo find /ostree/repo/objects -name '*.file' -type f | while read f; do
            sudo fsverity measure $f >/dev/null
          done
  # Test that we can build documentation
  docs:
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
      - name: Build mdbook
        run: just build-mdbook
  # Build packages for each test OS
  package:
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: compute-ci-level
    strategy:
      fail-fast: false
      matrix:
        test_os: ${{ fromJson(needs.compute-ci-level.outputs.package_os_matrix) }}

    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main

      - name: Setup env
        run: |
          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV

      - name: Build packages (and verify build system)
        run: just check-buildsys

      - name: Upload package artifacts
        uses: actions/upload-artifact@v7
        with:
          name: packages-${{ matrix.test_os }}
          path: target/packages/*.rpm
          retention-days: 1

  # Build bootc from source into a container image FROM each specified base `test_os`
  # running unit and integration tests (using TMT, leveraging the support for nested virtualization
  # in the GHA runners)
  test-integration:
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: [compute-ci-level, package]
    strategy:
      fail-fast: false
      matrix:
        test_os: ${{ fromJson(needs.compute-ci-level.outputs.integration_os_matrix) }}
        variant: [ostree, composefs]
        filesystem: ["ext4", "xfs"]
        # TODO: Remove "grub" once "grub-cc" is stable
        bootloader: ["grub", "grub-cc", "systemd"]
        boot_type: ["bls", "uki"]
        seal_state: ["sealed", "unsealed"]

        exclude:
          # https://github.com/bootc-dev/bootc/issues/1812
          - test_os: centos-9
            variant: composefs
          - seal_state: "sealed"
            boot_type: bls
          - seal_state: "sealed"
            bootloader: grub
          - seal_state: "sealed"
            filesystem: xfs
          - seal_state: "unsealed"
            filesystem: ext4
            boot_type: uki # we still want to test ext4 unsealed bls
          - bootloader: grub
            boot_type: "uki"

          # We only test filesystems for composefs to test if composefs backend will work on fs
          # without fsverity
          - variant: ostree
            filesystem: ext4
          - variant: ostree
            boot_type: uki
          - variant: ostree
            bootloader: systemd

          # For now only have grub-cc tests in F44
          - test_os: fedora-45
            bootloader: grub-cc
          - test_os: fedora-43
            bootloader: grub-cc
          - test_os: centos-9
            bootloader: grub-cc
          - test_os: centos-10
            bootloader: grub-cc
          # Not in ostree
          - variant: ostree
            bootloader: grub-cc
          # Not yet "sealed"
          - bootloader: grub-cc
            seal_state: sealed

    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
        with:
          libvirt: true
      - name: Install tmt
        run: pip install --user "tmt[provision-virtual]"

      - name: Setup env
        run: |
          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV
          echo "RUST_LOG=debug" >> $GITHUB_ENV

          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
          echo "BOOTC_filesystem=${{ matrix.filesystem }}" >> $GITHUB_ENV
          echo "BOOTC_bootloader=${{ matrix.bootloader }}" >> $GITHUB_ENV
          echo "BOOTC_boot_type=${{ matrix.boot_type }}" >> $GITHUB_ENV
          echo "BOOTC_seal_state=${{ matrix.seal_state }}" >> $GITHUB_ENV

      - name: Download package artifacts
        uses: actions/download-artifact@v8
        with:
          name: packages-${{ matrix.test_os }}
          path: target/packages/

      - name: Fetch external dependencies (with retry)
        run: BOOTC_SKIP_PACKAGE=1 just build-fetch

      - name: Build container
        run: |
          BOOTC_SKIP_PACKAGE=1 just bootloader=$BOOTC_bootloader build

          # Extra cross-check (duplicating the integration test) that we're using the right base
          used_vid=$(podman run --rm localhost/bootc bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
          test ${{ matrix.test_os }} = "${used_vid}"

      - name: Unit and container integration tests
        run: just test-container

      - name: Validate composefs digest (UKI only)
        if: matrix.boot_type == 'uki'
        run: just validate-composefs-digest

      - name: Run TMT integration tests
        run: |
          if [[ "${{ matrix.variant }}" = composefs ]]; then
            just test-composefs "${{ matrix.bootloader }}"  "${{ matrix.filesystem }}" "${{ matrix.boot_type }}" "${{ matrix.seal_state }}"
          else
            just test-tmt integration
          fi

          just clean-local-images

      - name: Disk usage summary
        if: always()
        run: |
          echo "### Disk usage" >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"
          df -h >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"

      - name: Archive TMT logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: "tmt-log-\
            ${{ matrix.test_os }}-\
            ${{ matrix.variant }}-\
            ${{ matrix.bootloader }}-\
            ${{ matrix.boot_type }}-\
            ${{ matrix.filesystem }}-\
            ${{ matrix.seal_state }}-\
            ${{ env.ARCH }}"
          path: /var/tmp/tmt

  # Test the upgrade path: boot from published base image, upgrade to locally-built image,
  # then run readonly tests to verify the upgrade worked.
  # Excluded: centos-9 (lacks systemd.extra-unit.* support needed for --bind-storage-ro)
  test-upgrade:
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: [compute-ci-level, package]
    strategy:
      fail-fast: false
      matrix:
        test_os: ${{ fromJson(needs.compute-ci-level.outputs.upgrade_os_matrix) }}
        variant: [ostree, composefs]

    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
        with:
          libvirt: true
      - name: Install tmt
        run: pip install --user "tmt[provision-virtual]"

      - name: Setup env
        run: |
          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
          echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV
          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV

      - name: Download package artifacts
        uses: actions/download-artifact@v8
        with:
          name: packages-${{ matrix.test_os }}
          path: target/packages/

      - name: Fetch external dependencies (with retry)
        run: just build-fetch

      - name: Run upgrade test
        run: just test-upgrade

      - name: Disk usage summary
        if: always()
        run: |
          echo "### Disk usage" >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"
          df -h >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"

      - name: Archive TMT logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: "tmt-log-${{ matrix.test_os }}-${{ matrix.variant }}-upgrade-${{ env.ARCH }}"
          path: /var/tmp/tmt

  # Test readonly behaviour with baseconfigs (transient mounts) baked into the image.
  # Composefs-only: setup-root-conf.toml is a composefs concept; ostree uses a
  # different config format (prepare-root.conf) and is not covered here.
  # Runs once per distro × baseconfig — no bootloader/filesystem/boot_type matrix.
  test-baseconfigs:
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: [compute-ci-level, package]
    strategy:
      fail-fast: false
      matrix:
        test_os: ${{ fromJson(needs.compute-ci-level.outputs.integration_os_matrix) }}
        baseconfigs: ["etc-transient", "root-transient", "var-volatile"]
        exclude:
          # centos-9 ships an older dracut that lacks the auto-install of setup-root-conf.toml
          - test_os: centos-9

    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
        with:
          libvirt: true
      - name: Install tmt
        run: pip install --user "tmt[provision-virtual]"

      - name: Setup env
        run: |
          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
          echo "BOOTC_variant=composefs" >> $GITHUB_ENV
          echo "BOOTC_baseconfigs=${{ matrix.baseconfigs }}" >> $GITHUB_ENV
          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV

      - name: Download package artifacts
        uses: actions/download-artifact@v8
        with:
          name: packages-${{ matrix.test_os }}
          path: target/packages/

      - name: Build container with baseconfig
        run: BOOTC_SKIP_PACKAGE=1 just build

      - name: Build upgrade image
        run: just _build-upgrade-image

      - name: Run TMT readonly tests
        run: |
          cargo xtask run-tmt \
            --env=BOOTC_variant=composefs \
            --env=BOOTC_baseconfigs=${{ matrix.baseconfigs }} \
            --composefs-backend --bootloader=grub --filesystem=ext4 \
            --seal-state=unsealed --boot-type=bls \
            --upgrade-image=localhost/bootc-upgrade \
            localhost/bootc readonly
          just clean-local-images

      - name: Disk usage summary
        if: always()
        run: |
          echo "### Disk usage" >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"
          df -h >> "$GITHUB_STEP_SUMMARY"
          echo '```' >> "$GITHUB_STEP_SUMMARY"

      - name: Archive TMT logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: "tmt-log-${{ matrix.test_os }}-composefs-baseconfigs-${{ matrix.baseconfigs }}-${{ env.ARCH }}"
          path: /var/tmp/tmt

  # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues
  # when run in the same job as test-integration).
  # Uses fedora-43 as it's the current stable Fedora release matching CoreOS.
  test-coreos:
    # https://github.com/ostreedev/ostree/pull/3571 broke this because /boot
    # without a separate /boot partition isn't mounted in the initramfs anymore.
    # We need to change to use coreos-assembler.
    if: false
    needs: [compute-ci-level, package]
    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
        with:
          libvirt: true
      - name: Install tmt
        run: pip install --user "tmt[provision-virtual]"

      - name: Setup env
        run: |
          BASE=$(just pullspec-for-os base fedora-43)
          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
          echo "BOOTC_variant=ostree" >> $GITHUB_ENV

      - name: Download package artifacts
        uses: actions/download-artifact@v8
        with:
          name: packages-fedora-43
          path: target/packages/

      - name: Build container and test on CoreOS
        run: |
          BOOTC_SKIP_PACKAGE=1 just build
          just build-testimage-coreos target/packages
          just test-tmt-on-coreos plan-bootc-install-on-coreos
          just clean-local-images

      - name: Archive TMT logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: tmt-log-fedora-43-coreos-${{ env.ARCH }}
          path: /var/tmp/tmt

  # Test the container export -> Anaconda liveimg install path.
  # Builds localhost/bootc, exports as tarball, installs via Anaconda in QEMU,
  # and verifies the resulting disk boots.
  test-container-export:
    if: needs.compute-ci-level.outputs.run_heavy == 'true'
    needs: [compute-ci-level, package]
    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@v7
      - name: Bootc Ubuntu Setup
        uses: bootc-dev/actions/bootc-ubuntu-setup@main
        with:
          libvirt: true

      - name: Setup env
        run: |
          echo "BOOTC_base=quay.io/centos-bootc/centos-bootc:stream10" >> $GITHUB_ENV

      - name: Download package artifacts
        uses: actions/download-artifact@v8
        with:
          name: packages-centos-10
          path: target/packages/

      - name: Build and run container export test
        run: |
          BOOTC_SKIP_PACKAGE=1 just test-container-export

      - name: Archive test logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: container-export-test-${{ env.ARCH }}
          path: target/anaconda-test/*.log

  # Sentinel job for required checks - configure this job name in repository settings.
  # Accepts 'skipped' as success so that merge_group-only jobs don't block PRs.
  required-checks:
    if: always()
    needs: [compute-ci-level, cargo-deny, validate, install-tests, docs, package, test-integration, test-upgrade, test-baseconfigs, test-container-export]
    runs-on: ubuntu-latest
    steps:
      - name: Check all jobs
        env:
          NEEDS: ${{ toJson(needs) }}
        run: |
          FAILED=$(echo "$NEEDS" | jq -r 'to_entries[] | select(.value.result | IN("success","skipped") | not) | .key')
          if [ -n "$FAILED" ]; then
            echo "The following jobs did not succeed: $FAILED"
            exit 1
          fi