Use system MPI for STRUMPACK build (fix 4-rank MPI tests)

- Use MPI wrapper compilers (mpicc, mpicxx, mpif90) instead of bundled MPICH
- Add system MPI detection and clear error message if not found
- Update docs to require system MPICH installation
- Tests now correctly show "aggregated across 4 ranks"

Author: Sébastien Loisel

docs/src/guide/strumpack.md

@@ -29,35 +29,25 @@ Stick with MUMPS when:
 
 ### Prerequisites
 
-**System MPI is required.** The JLL-provided MPI wrappers have broken paths and cannot be used for building PETSc from source.
+**System MPI is required.** The JLL-provided MPI wrappers have broken paths and cannot be used for building PETSc from source. MPI.jl must be configured to use the same system MPI before building.
 
 ```bash
 # macOS
-brew install open-mpi
+brew install mpich
 
 # Ubuntu/Debian
-sudo apt-get install libopenmpi-dev
+sudo apt-get install mpich libmpich-dev
 ```
 
 Then configure MPI.jl to use system MPI:
 
 ```julia
 using MPIPreferences
-
-# On macOS (Open MPI from Homebrew)
-MPIPreferences.use_system_binary(;
-    library_names=["libmpi"],
-    extra_paths=["/opt/homebrew/Cellar/open-mpi/5.0.8/lib"],  # Adjust version as needed
-    mpiexec="/opt/homebrew/bin/mpiexec"
-)
-
-# On Linux (usually auto-detects)
 MPIPreferences.use_system_binary()
-
 # Restart Julia after configuring
 ```
 
-**Important**: MPI.jl must use the **same MPI implementation** as the PETSc build. If PETSc is built with Open MPI, MPI.jl must also use Open MPI. Mixing Open MPI and MPICH causes crashes.
+**Important**: MPI.jl must use the **same MPI implementation** as the PETSc build. Since we build with system MPI (detected from `mpicc` in PATH), MPI.jl must also use system MPI. Mixing different MPI implementations causes crashes.
 
 ### One-Time Build
 
@@ -267,10 +257,10 @@ sudo apt-get install build-essential gfortran cmake
 **Missing MPI**: The build requires system MPI (not the JLL wrappers). Install it and configure MPI.jl:
 ```bash
 # macOS
-brew install open-mpi
+brew install mpich
 
 # Ubuntu/Debian
-sudo apt-get install libopenmpi-dev
+sudo apt-get install mpich libmpich-dev
 ```
 
 ```julia
@@ -292,15 +282,10 @@ using MPI
 MPI.identify_implementation()  # Should match what PETSc was built with
 ```
 
-The STRUMPACK build uses system MPI (usually Open MPI from Homebrew). Configure MPI.jl to match:
+The STRUMPACK build uses system MPI (detected from `mpicc` in PATH). Configure MPI.jl to use the same system MPI:
 ```julia
 using MPIPreferences
-# On macOS with Homebrew Open MPI:
-MPIPreferences.use_system_binary(;
-    library_names=["libmpi"],
-    extra_paths=["/opt/homebrew/Cellar/open-mpi/5.0.8/lib"],
-    mpiexec="/opt/homebrew/bin/mpiexec"
-)
+MPIPreferences.use_system_binary()
 # Then restart Julia and delete compiled caches
 ```
 

src/build_petsc.jl

@@ -2,8 +2,9 @@
 # This module provides functions to download, build, and configure PETSc with
 # both STRUMPACK (for GPU acceleration and MPI scalability) and MUMPS (proven direct solver).
 #
-# IMPORTANT: Requires system MPI (brew install open-mpi or apt-get install libopenmpi-dev)
+# IMPORTANT: Requires system MPI (brew install mpich or apt-get install mpich libmpich-dev)
 # The JLL-provided MPI wrappers have broken paths and cannot be used for building.
+# MPI.jl must also be configured to use system MPI: MPIPreferences.use_system_binary()
 
 using Scratch
 using Downloads
@@ -109,20 +110,21 @@ compression features.
 
 # Prerequisites
 **System MPI is required.** The JLL-provided MPI wrappers have broken paths.
+MPI.jl must be configured to use the same system MPI before building.
 
 ```bash
 # macOS
-brew install open-mpi
+brew install mpich
 
 # Ubuntu/Debian
-sudo apt-get install libopenmpi-dev
+sudo apt-get install mpich libmpich-dev
 ```
 
 Then configure MPI.jl to use system MPI:
 ```julia
 using MPIPreferences
 MPIPreferences.use_system_binary()
-# Restart Julia
+# Restart Julia before building
 ```
 
 # Arguments
@@ -306,18 +308,35 @@ end
 
 function _build_petsc_with_strumpack(src_dir::String, install_dir::String, with_debugging::Bool,
                                      with_cuda::Bool, verbose::Bool)
-    # Build configuration flags - includes both STRUMPACK and MUMPS
-    # Bundle MPICH with PETSc for ABI compatibility (avoids mismatch between build/test environments)
-    # Note: We use MPICH instead of Open MPI because Open MPI 5.x requires PMIx server setup
+    # Check that system MPI is available
+    if !_check_system_mpi()
+        error("""
+        System MPI not found! STRUMPACK build requires system MPI (not Julia's MPI JLL).
+
+        Install system MPI first:
+            macOS:  brew install mpich
+            Linux:  sudo apt-get install mpich libmpich-dev
+
+        Then configure MPI.jl to use system MPI:
+            using MPIPreferences
+            MPIPreferences.use_system_binary()
+            # Restart Julia before building
+        """)
+    end
+
+    # Build configuration flags - uses system MPI via wrapper compilers
+    # System MPI is required for proper mpiexec functionality in containers/CI environments
     configure_flags = [
         "--prefix=$install_dir",
-        "--download-mpich",   # Bundle MPICH for guaranteed ABI compatibility
-        "--with-fc=gfortran",   # Fortran needed for fblaslapack
+        # Use MPI wrapper compilers directly - PETSc will detect MPI from these
+        "--with-cc=mpicc",
+        "--with-cxx=mpicxx",
+        "--with-fc=mpif90",   # Fortran needed for fblaslapack (use MPI wrapper)
         "--with-debugging=$(with_debugging ? 1 : 0)",
         "--with-shared-libraries=1",
     ]
     if verbose
-        @info "Building with bundled MPICH (for ABI compatibility)"
+        @info "Using system MPI via wrapper compilers (mpicc, mpicxx, mpif90)"
     end
 
     # Add common dependencies

test/runtests.jl

@@ -1,12 +1,51 @@
 using Test
 using MPI
 
+# Check if PETSc is loading the expected library when JULIA_PETSC_LIBRARY is set.
+# PETSc.jl uses @static to check the env var at precompile time, so if the cache
+# was compiled without JULIA_PETSC_LIBRARY, it uses PETSc_jll even if the env var
+# is now set.
+#
+# IMPORTANT: Pkg.test() runs with --startup-file=no by default, so startup.jl
+# settings don't apply during precompilation. For STRUMPACK testing, run tests
+# directly: JULIA_PETSC_LIBRARY="..." julia --project=. test/runtests.jl
+function check_petsc_library()
+    lib_env = get(ENV, "JULIA_PETSC_LIBRARY", "")
+    if isempty(lib_env)
+        return  # Using default JLL, no check needed
+    end
+
+    # Load PETSc and check the loaded library
+    @eval using PETSc
+    libs_str = string(PETSc.libs)
+
+    if occursin(lib_env, libs_str) || occursin("petsc_strumpack", libs_str)
+        println("[runtests.jl] PETSc library OK: using custom build")
+    else
+        @warn """
+        PETSc library mismatch!
+
+        JULIA_PETSC_LIBRARY is set but PETSc loaded the default JLL library.
+
+        This happens because PETSc.jl checks JULIA_PETSC_LIBRARY at precompile time.
+        If running via Pkg.test(), the env var isn't visible during precompilation.
+
+        For STRUMPACK testing, run tests directly instead:
+            JULIA_PETSC_LIBRARY="$lib_env" julia --project=. test/runtests.jl
+
+        Tests will continue with the default PETSc library (no STRUMPACK).
+        """ expected=lib_env loaded=libs_str
+    end
+end
+
 # Precompile all dependencies once before any mpiexec to avoid concurrent compilation
 # This significantly reduces hangs due to pidfile contention and MPI initialization conflicts
 try
-    # Force precompilation of all test dependencies
+    # Check PETSc library - warns if JULIA_PETSC_LIBRARY is set but not loaded
+    check_petsc_library()
+
+    # Force precompilation of all test dependencies (PETSc already loaded in check_petsc_library)
     @eval using SafePETSc
-    @eval using PETSc
     @eval using LinearAlgebra
     @eval using SparseArrays
     println("Precompilation complete for test environment")
@@ -28,6 +67,10 @@ function run_mpi_test(test_file::AbstractString; nprocs::Integer=4, expect_succe
     # This ensures LocalPreferences.toml is honored and avoids recompilation under mpiexec
     test_proj = Base.active_project()
     cmd = `$mpiexec_cmd -n $nprocs $(Base.julia_cmd()) --project=$test_proj $test_file`
+    # Pass through JULIA_PETSC_LIBRARY for custom PETSc builds (e.g., STRUMPACK)
+    if haskey(ENV, "JULIA_PETSC_LIBRARY")
+        cmd = addenv(cmd, "JULIA_PETSC_LIBRARY" => ENV["JULIA_PETSC_LIBRARY"])
+    end
     proc = run(ignorestatus(cmd))
     ok = success(proc)
     if ok != expect_success