Bundle MPICH with PETSc for ABI compatibility

- Use --download-mpich in PETSc build to bundle MPICH
- Package both lib/ and bin/ directories in tarball
- CI uses bundled libmpi and mpiexec from tarball
- Fixes MPI ABI mismatch between build and test environments

Author: Sébastien Loisel

.github/workflows/CI.yml

@@ -71,11 +71,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install MPI and CUDA runtime
+      - name: Install CUDA runtime
         run: |
           sudo apt-get update
-          # Install system MPICH (same as build environment) and CUDA runtime
-          sudo apt-get install -y mpich libmpich-dev nvidia-cuda-toolkit
+          # Only install CUDA runtime (MPI is bundled in the tarball)
+          sudo apt-get install -y nvidia-cuda-toolkit
 
       - uses: julia-actions/setup-julia@v2
         with:
@@ -84,7 +84,7 @@ jobs:
 
       # Note: Intentionally NOT using julia-actions/cache@v2 here
       # The cache would restore MPI.jl compiled against JLL MPI, conflicting
-      # with the system MPICH that the pre-built PETSc requires
+      # with the bundled MPICH in the pre-built PETSc tarball
 
       - name: Download pre-built PETSc STRUMPACK
         run: |
@@ -94,20 +94,26 @@ jobs:
           mkdir -p /tmp/petsc_strumpack
           tar -xzf /tmp/${PETSC_RELEASE_TAG}.tar.gz -C /tmp/petsc_strumpack
           echo "Done. Library at /tmp/petsc_strumpack/lib/libpetsc.so"
+          echo "Contents of lib/:"
           ls -la /tmp/petsc_strumpack/lib/
+          echo "Contents of bin/:"
+          ls -la /tmp/petsc_strumpack/bin/
         env:
           GH_TOKEN: ${{ github.token }}
 
-      - name: Configure MPI to use system MPICH
+      - name: Configure MPI to use bundled MPICH
         run: |
-          # Configure MPI.jl to use the system MPICH (same as PETSc build environment)
-          # CRITICAL: MPICH uses "libmpich", not "libmpi"
+          # Configure MPI.jl to use the bundled MPICH from the tarball
+          # This ensures ABI compatibility with the pre-built PETSc
           julia --project=. -e "
             using Pkg
             Pkg.add(\"MPIPreferences\")
             using MPIPreferences
-            # Use libmpich (not libmpi!) - this is how MPICH names its library
-            MPIPreferences.use_system_binary(library_names=[\"libmpich\"])
+            # Use bundled libmpi from tarball
+            MPIPreferences.use_system_binary(
+              library_names=[\"libmpi\"],
+              mpiexec=\"/tmp/petsc_strumpack/bin/mpiexec\"
+            )
             # Force MPI to rebuild with new preferences
             Pkg.build(\"MPI\"; verbose=true)
           "
@@ -127,6 +133,7 @@ jobs:
         env:
           JULIA_PETSC_LIBRARY: /tmp/petsc_strumpack/lib/libpetsc.so
           LD_LIBRARY_PATH: /tmp/petsc_strumpack/lib
+          MPIEXEC_PATH: /tmp/petsc_strumpack/bin/mpiexec
         run: |
           # Run tests directly instead of Pkg.test() to preserve LocalPreferences.toml
           # (Pkg.test() creates a fresh temp environment that loses MPI preferences)

.github/workflows/build-petsc-strumpack.yml

@@ -79,13 +79,16 @@ jobs:
           INSTALL_DIR="${{ steps.build-info.outputs.install_dir }}"
           TAG="${{ steps.build-info.outputs.tag }}"
 
-          # Package lib/ directory only (uses system MPI, no bundled mpiexec)
+          # Package lib/ and bin/ directories (bundled MPICH includes mpiexec)
           cd $INSTALL_DIR
           echo "Contents of lib/:"
           ls -lh lib/
           echo ""
-          echo "Creating tarball of lib/..."
-          tar -czvf ${TAG}.tar.gz lib/
+          echo "Contents of bin/:"
+          ls -lh bin/
+          echo ""
+          echo "Creating tarball of lib/ and bin/..."
+          tar -czvf ${TAG}.tar.gz lib/ bin/
           mv ${TAG}.tar.gz $GITHUB_WORKSPACE/
 
           echo "Created: ${TAG}.tar.gz"
@@ -97,26 +100,25 @@ jobs:
           tag_name: ${{ steps.build-info.outputs.tag }}
           name: PETSc ${{ steps.build-info.outputs.petsc_version }} with STRUMPACK${{ inputs.with_cuda && ' + CUDA' || '' }}
           body: |
-            Pre-built PETSc library with STRUMPACK solver.
+            Pre-built PETSc library with STRUMPACK solver and bundled MPICH.
 
             **Build Configuration:**
             - PETSc version: ${{ steps.build-info.outputs.petsc_version }}
             - STRUMPACK: enabled
             - CUDA: ${{ inputs.with_cuda && 'enabled' || 'disabled' }}
             - Platform: Linux x86_64 (Ubuntu)
-            - MPI: System MPICH (requires `apt install mpich libmpich-dev`)
+            - MPI: Bundled MPICH (self-contained, no system MPI required)
 
             **Usage:**
             ```bash
-            # Install system MPICH (required)
-            sudo apt install mpich libmpich-dev
-
             # Download and extract
             wget https://github.com/${{ github.repository }}/releases/download/${{ steps.build-info.outputs.tag }}/${{ steps.build-info.outputs.tag }}.tar.gz
             tar -xzf ${{ steps.build-info.outputs.tag }}.tar.gz
 
-            # Set environment variable before starting Julia
-            export JULIA_PETSC_LIBRARY=/path/to/petsc_strumpack/lib/libpetsc.so
+            # Set environment variables before starting Julia
+            export JULIA_PETSC_LIBRARY=/path/to/lib/libpetsc.so
+            export LD_LIBRARY_PATH=/path/to/lib:$LD_LIBRARY_PATH
+            export MPIEXEC_PATH=/path/to/bin/mpiexec
             ```
           files: ${{ steps.build-info.outputs.tag }}.tar.gz
           draft: false

src/build_petsc.jl

@@ -307,17 +307,16 @@ end
 function _build_petsc_with_strumpack(src_dir::String, install_dir::String, with_debugging::Bool,
                                      with_cuda::Bool, verbose::Bool)
     # Build configuration flags - includes both STRUMPACK and MUMPS
-    # Use system MPI via compiler wrappers (requires: apt install mpich libmpich-dev OR brew install mpich)
+    # Bundle MPICH with PETSc for ABI compatibility (avoids mismatch between build/test environments)
     configure_flags = [
         "--prefix=$install_dir",
-        "--with-cc=mpicc",      # Use MPI C compiler wrapper
-        "--with-cxx=mpicxx",    # Use MPI C++ compiler wrapper
-        "--with-fc=mpif90",     # Use MPI Fortran compiler wrapper (needed for fblaslapack)
+        "--download-mpich",     # Bundle MPICH for guaranteed ABI compatibility
+        "--with-fc=gfortran",   # Fortran needed for fblaslapack
         "--with-debugging=$(with_debugging ? 1 : 0)",
         "--with-shared-libraries=1",
     ]
     if verbose
-        @info "Building with system MPI (using mpicc/mpicxx/mpif90 wrappers)"
+        @info "Building with bundled MPICH (for ABI compatibility)"
     end
 
     # Add common dependencies