ci: Implement 4-way hash sharding for integration tests

.github/workflows/main.yml

@@ -14,60 +14,64 @@ env:
 
 
 jobs:
-  build-and-test:
+  build:
     runs-on: ubuntu-24.04
 
     steps:
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y just pkg-config go-md2man libvirt-daemon libvirt-clients qemu-kvm qemu-system qemu-utils virtiofsd
+      - uses: actions/checkout@v4
 
-      - name: Install podman for heredoc support
-        run: |
-          set -eux
-          echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
-          sudo apt update
-          sudo apt install -y crun/testing podman/testing just
+      - uses: ./.github/actions/bootc-ubuntu-setup
+        with:
+          libvirt: 'true'
 
-      - name: Enable KVM group perms
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm
-          ls -l /dev/kvm
+      - name: Install additional dependencies
+        run: sudo apt install -y go-md2man
 
-      - uses: actions/checkout@v4
+      - name: Extract image lists from Justfile
+        run: |
+          echo "PRIMARY_IMAGE=$(just --evaluate PRIMARY_IMAGE)" >> $GITHUB_ENV
+          echo "ALL_BASE_IMAGES=$(just --evaluate ALL_BASE_IMAGES)" >> $GITHUB_ENV
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
-
-      - uses: taiki-e/install-action@nextest
 
-      - name: Cache build artifacts
-        uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@nextest
 
       - name: Build
         run: just validate && just build
-        
+
       - name: Run unit tests
         run: just unit
-
-      - name: Run integration tests
-        run: just test-integration
 
-      - name: Upload junit XML
-        if: always()
+      - name: Pull test images
+        run: just pull-test-images
+
+      - name: Create nextest archive
+        run: |
+          cargo nextest archive --release -p integration-tests --archive-file nextest-archive.tar.zst
+        env:
+          BCVK_PATH: ${{ github.workspace }}/target/release/bcvk
+          BCVK_PRIMARY_IMAGE: ${{ env.PRIMARY_IMAGE }}
+          BCVK_ALL_IMAGES: ${{ env.ALL_BASE_IMAGES }}
+
+      - name: Upload nextest archive
         uses: actions/upload-artifact@v4
         with:
-          name: integration-junit-xml
-          path: target/nextest/integration/junit.xml
+          name: nextest-archive
+          path: nextest-archive.tar.zst
           retention-days: 7
 
-      - name: Create archive
+      - name: Upload bcvk binary for tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: bcvk-binary-tests
+          path: target/release/bcvk
+          retention-days: 7
+
+      - name: Create bcvk archive
         run: just archive
-        
-      - name: Upload artifacts
+
+      - name: Upload bcvk binary artifacts
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         uses: actions/upload-artifact@v4
         with:
@@ -76,3 +80,81 @@ jobs:
             target/bcvk-*.tar.gz
             target/bcvk-*.tar.gz.sha256
           retention-days: 7
+
+  integration-tests:
+    runs-on: ubuntu-24.04
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        partition: [1, 2, 3, 4]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/bootc-ubuntu-setup
+        with:
+          libvirt: 'true'
+
+      - name: Extract image lists from Justfile
+        run: |
+          echo "PRIMARY_IMAGE=$(just --evaluate PRIMARY_IMAGE)" >> $GITHUB_ENV
+          echo "ALL_BASE_IMAGES=$(just --evaluate ALL_BASE_IMAGES)" >> $GITHUB_ENV
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Pull test images
+        run: just pull-test-images
+
+      - name: Download nextest archive
+        uses: actions/download-artifact@v4
+        with:
+          name: nextest-archive
+
+      - name: Download bcvk binary
+        uses: actions/download-artifact@v4
+        with:
+          name: bcvk-binary-tests
+          path: target/release
+
+      - name: Make bcvk executable
+        run: chmod +x target/release/bcvk
+
+      - name: Run integration tests (partition ${{ matrix.partition }}/4)
+        run: |
+          # Clean up any leftover containers before starting
+          cargo run --release --bin test-cleanup -p integration-tests 2>/dev/null || true
+
+          # Run the partitioned tests
+          cargo nextest run --archive-file nextest-archive.tar.zst \
+            --profile integration \
+            --partition hash:${{ matrix.partition }}/4
+
+          # Clean up containers after tests complete
+          cargo run --release --bin test-cleanup -p integration-tests 2>/dev/null || true
+        env:
+          BCVK_PATH: ${{ github.workspace }}/target/release/bcvk
+          BCVK_PRIMARY_IMAGE: ${{ env.PRIMARY_IMAGE }}
+          BCVK_ALL_IMAGES: ${{ env.ALL_BASE_IMAGES }}
+
+      - name: Upload junit XML
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-junit-xml-${{ matrix.partition }}
+          path: target/nextest/integration/junit.xml
+          retention-days: 7
+
+  # Sentinel job for required checks - configure this job name in repository settings
+  required-checks:
+    if: always()
+    needs: [build, integration-tests]
+    runs-on: ubuntu-latest
+    steps:
+      - run: exit 1
+        if: >-
+          needs.build.result != 'success' ||
+          needs.integration-tests.result != 'success'

crates/integration-tests/src/tests/libvirt_base_disks.rs

@@ -92,6 +92,41 @@ fn test_base_disk_creation_and_reuse() -> Result<()> {
         "Should mention using base disk"
     );
 
+    // Test base-disks list shows creation timestamp
+    println!("Testing that base-disks list shows creation timestamp...");
+    let bck = get_bck_command()?;
+    let list_output = std::process::Command::new(&bck)
+        .args(["libvirt", "base-disks", "list"])
+        .output()
+        .expect("Failed to run base-disks list");
+
+    let list_stdout = String::from_utf8_lossy(&list_output.stdout);
+    let list_stderr = String::from_utf8_lossy(&list_output.stderr);
+
+    if list_output.status.success() {
+        println!("base-disks list output:\n{}", list_stdout);
+
+        // Should have CREATED column in header
+        assert!(
+            list_stdout.contains("CREATED"),
+            "Should show CREATED column in header"
+        );
+
+        // Should show timestamp values (either a date or "unknown")
+        // Timestamp format is YYYY-MM-DD HH:MM
+        let re = Regex::new(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}|unknown").unwrap();
+        let has_timestamp = re.is_match(&list_stdout);
+        assert!(
+            has_timestamp,
+            "Should show timestamp values in CREATED column"
+        );
+
+        println!("✓ base-disks list shows creation timestamp");
+    } else {
+        println!("base-disks list failed: {}", list_stderr);
+        panic!("Failed to run base-disks list: {}", list_stderr);
+    }
+
     println!("✓ Base disk creation and reuse test passed");
     Ok(())
 }
@@ -138,76 +173,6 @@ fn test_base_disks_list_command() -> Result<()> {
 }
 integration_test!(test_base_disks_list_command);
 
-/// Test base-disks list shows creation timestamp
-fn test_base_disks_list_shows_timestamp() -> Result<()> {
-    let test_image = get_test_image();
-    let bck = get_bck_command()?;
-
-    println!("Testing base-disks list shows creation timestamp");
-
-    let timestamp = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap()
-        .as_secs();
-    let vm_name = format!("test-base-timestamp-{}", timestamp);
-
-    cleanup_domain(&vm_name);
-
-    // Create a VM to ensure we have at least one base disk
-    println!("Creating VM to generate base disk...");
-    let vm_output = run_bcvk(&[
-        "libvirt",
-        "run",
-        "--name",
-        &vm_name,
-        "--filesystem",
-        "ext4",
-        &test_image,
-    ])?;
-
-    if !vm_output.success() {
-        cleanup_domain(&vm_name);
-        panic!("Failed to create VM: {}", vm_output.stderr);
-    }
-
-    // Run base-disks list
-    let output = Command::new(&bck)
-        .args(["libvirt", "base-disks", "list"])
-        .output()
-        .expect("Failed to run base-disks list");
-
-    let stdout = String::from_utf8_lossy(&output.stdout);
-    let stderr = String::from_utf8_lossy(&output.stderr);
-
-    cleanup_domain(&vm_name);
-
-    if output.status.success() {
-        println!("base-disks list output:\n{}", stdout);
-
-        // Should have CREATED column in header
-        assert!(
-            stdout.contains("CREATED"),
-            "Should show CREATED column in header"
-        );
-
-        // Should show timestamp values (either a date or "unknown")
-        // Timestamp format is YYYY-MM-DD HH:MM
-        let re = Regex::new(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}|unknown").unwrap();
-        let has_timestamp = re.is_match(&stdout);
-        assert!(
-            has_timestamp,
-            "Should show timestamp values in CREATED column"
-        );
-
-        println!("✓ base-disks list shows creation timestamp");
-    } else {
-        println!("base-disks list failed: {}", stderr);
-        panic!("Failed to run base-disks list: {}", stderr);
-    }
-    Ok(())
-}
-integration_test!(test_base_disks_list_shows_timestamp);
-
 /// Test base-disks prune command with dry-run
 fn test_base_disks_prune_dry_run() -> Result<()> {
     let bck = get_bck_command()?;