Skip to content

Commit 66ca717

Browse files
Sébastien LoiselSébastien Loisel
authored andcommitted
Use MPI jail escape for precompilation
Replace SnoopCompile static precompile.jl with PrecompileTools + MPI environment cleanup. The precompilation subprocess clears PMI_*, PMIX_*, OMPI_*, and MPI_* environment variables before MPI.Init(), allowing it to initialize as a fresh single-rank process even when the parent was launched under mpiexec.
1 parent cb0191e commit 66ca717

4 files changed

Lines changed: 132 additions & 218 deletions

File tree

Project.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,19 @@ Blake3Hash = "8f478455-a32d-4928-b0e4-72b19a7d5574"
88
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
99
MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
1010
MUMPS = "55d2b088-9f4e-11e9-26c0-150b02ea6a46"
11+
PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
1112
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
1213

1314
[compat]
1415
Blake3Hash = "0.3"
1516
MPI = "0.20"
1617
MUMPS = "1.5"
18+
PrecompileTools = "1"
1719
julia = "1.10"
1820

1921
[extras]
2022
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
2123
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
22-
SnoopCompile = "aa65fe97-06da-5843-b5b1-d5d13cad87d2"
2324
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2425

2526
[targets]

scripts/generate_precompile.jl

Lines changed: 0 additions & 167 deletions
This file was deleted.

src/LinearAlgebraMPI.jl

Lines changed: 130 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,8 +662,135 @@ end
662662
# Precompilation Workload
663663
# ============================================================================
664664

665-
# Precompile directives generated by SnoopCompile
666-
# Regenerate with: mpiexec -n 1 julia --project=. scripts/generate_precompile.jl
667-
include("precompile.jl")
665+
using PrecompileTools
666+
667+
@setup_workload begin
668+
# Small test data for precompilation (runs with single MPI rank)
669+
n = 8
670+
671+
# Sparse matrix (tridiagonal) - Float64
672+
I_sp = Int[]; J_sp = Int[]; V_sp = Float64[]
673+
for i in 1:n
674+
push!(I_sp, i); push!(J_sp, i); push!(V_sp, 4.0)
675+
if i > 1
676+
push!(I_sp, i); push!(J_sp, i-1); push!(V_sp, -1.0)
677+
push!(I_sp, i-1); push!(J_sp, i); push!(V_sp, -1.0)
678+
end
679+
end
680+
A_sparse_f64 = sparse(I_sp, J_sp, V_sp, n, n)
681+
682+
# Sparse matrix - ComplexF64
683+
A_sparse_c64 = sparse(I_sp, J_sp, ComplexF64.(V_sp), n, n)
684+
685+
# Dense matrix - Float64
686+
A_dense_f64 = Float64[i == j ? 4.0 : (abs(i-j) == 1 ? -1.0 : 0.0) for i in 1:n, j in 1:n]
687+
688+
# Dense matrix - ComplexF64
689+
A_dense_c64 = ComplexF64.(A_dense_f64)
690+
691+
# Vectors
692+
v_f64 = ones(Float64, n)
693+
v_c64 = ones(ComplexF64, n)
694+
695+
# Identity for SPD construction
696+
I_sparse = sparse(1.0 * LinearAlgebra.I, n, n)
697+
698+
@compile_workload begin
699+
# === MPI Jail Escape ===
700+
# When precompiling under mpiexec, the subprocess inherits MPI environment
701+
# variables but isn't part of the MPI job. Clean them to allow MPI.Init()
702+
# to succeed as a fresh single-rank process.
703+
for k in collect(keys(ENV))
704+
if startswith(k, "PMI") || startswith(k, "PMIX") || startswith(k, "OMPI_") || startswith(k, "MPI_")
705+
delete!(ENV, k)
706+
end
707+
end
708+
709+
MPI.Init()
710+
711+
# === VectorMPI operations (Float64) ===
712+
v = VectorMPI(v_f64)
713+
w = VectorMPI(2.0 .* v_f64)
714+
_ = v + w
715+
_ = v - w
716+
_ = 2.0 * v
717+
_ = v * 2.0
718+
_ = norm(v)
719+
_ = dot(v, w)
720+
_ = conj(v)
721+
_ = length(v)
722+
_ = size(v)
723+
724+
# VectorMPI (ComplexF64)
725+
vc = VectorMPI(v_c64)
726+
_ = conj(vc)
727+
_ = norm(vc)
728+
729+
# === SparseMatrixMPI operations (Float64) ===
730+
A = SparseMatrixMPI{Float64}(A_sparse_f64)
731+
B = SparseMatrixMPI{Float64}(A_sparse_f64)
732+
_ = A + B
733+
_ = A - B
734+
_ = 2.0 * A
735+
_ = A * v
736+
_ = A * B
737+
_ = transpose(A)
738+
At = SparseMatrixMPI(transpose(A))
739+
_ = size(A)
740+
_ = nnz(A)
741+
_ = norm(A)
742+
743+
# SparseMatrixMPI (ComplexF64)
744+
Ac = SparseMatrixMPI{ComplexF64}(A_sparse_c64)
745+
_ = Ac * vc
746+
747+
# === MatrixMPI operations (Float64) ===
748+
D = MatrixMPI(A_dense_f64)
749+
_ = 2.0 * D
750+
_ = D * v
751+
_ = transpose(D)
752+
Dt = copy(transpose(D)) # Materialize dense transpose
753+
_ = size(D)
754+
_ = norm(D)
755+
756+
# MatrixMPI (ComplexF64)
757+
Dc = MatrixMPI(A_dense_c64)
758+
_ = Dc * vc
759+
760+
# === Mixed operations ===
761+
_ = A * D # Sparse * Dense
762+
763+
# === Indexing ===
764+
_ = v[1]
765+
_ = A[1, 1]
766+
_ = D[1, 1]
767+
768+
# === Factorization (MUMPS) ===
769+
# Make symmetric positive definite: A + A^T + 10I
770+
At_mat = SparseMatrixMPI(transpose(A))
771+
I_dist = SparseMatrixMPI{Float64}(I_sparse)
772+
A_spd = A + At_mat + I_dist * 10.0
773+
F = LinearAlgebra.ldlt(A_spd)
774+
x = F \ v
775+
finalize!(F)
776+
777+
# LU factorization
778+
F_lu = LinearAlgebra.lu(A)
779+
x = F_lu \ v
780+
finalize!(F_lu)
781+
782+
# === Block operations ===
783+
_ = cat(v, w; dims=1)
784+
_ = blockdiag(A, B)
785+
786+
# === Conversions ===
787+
_ = Vector(v)
788+
_ = Matrix(D)
789+
_ = SparseMatrixCSC(A)
790+
791+
# Clear caches
792+
clear_plan_cache!()
793+
end
794+
end
668795

669796
end # module LinearAlgebraMPI

src/precompile.jl

Lines changed: 0 additions & 47 deletions
This file was deleted.

0 commit comments

Comments
 (0)