From e272dc2bc08833b66e69b3159dd3ef91c4e763c0 Mon Sep 17 00:00:00 2001 From: Jordi Manyer Date: Sun, 6 Jul 2025 22:52:19 +0200 Subject: [PATCH 1/4] Added CartesianDiscreteModels with an arbitrary number of ghost layers --- Project.toml | 2 + src/Geometry.jl | 10 +++-- src/GridapDistributed.jl | 3 ++ src/PArraysExtras.jl | 84 ++++++++++++++++++++++++++++++++++++++++ test/ExtraGhostLayers.jl | 36 +++++++++++++++++ 5 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 src/PArraysExtras.jl create mode 100644 test/ExtraGhostLayers.jl diff --git a/Project.toml b/Project.toml index 1e425e62..4552edab 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.4.8" [deps] BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" +CircularArrays = "7a955b69-7140-5f4e-a0ed-f168c5e2e749" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Gridap = "56d4f2e9-7ea1-5844-9cf6-b9c51ca7ce8e" @@ -17,6 +18,7 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" [compat] BlockArrays = "1" +CircularArrays = "1.4.0" FillArrays = "1" ForwardDiff = "0.10, 1" Gridap = "0.19.1" diff --git a/src/Geometry.jl b/src/Geometry.jl index 3e25f389..56e38d5f 100644 --- a/src/Geometry.jl +++ b/src/Geometry.jl @@ -243,7 +243,11 @@ const DistributedCartesianDiscreteModel{Dc,Dp,A,B,C} = function Geometry.CartesianDiscreteModel( ranks::AbstractArray{<:Integer}, # Distributed array with the rank IDs parts::NTuple{N,<:Integer}, # Number of ranks (parts) in each direction - args...;isperiodic=map(i->false,parts),kwargs...) where N + args...; + ghost = map(i->true,parts), + isperiodic = map(i->false,parts), + kwargs... +) where N desc = CartesianDescriptor(args...;isperiodic=isperiodic,kwargs...) nc = desc.partition @@ -254,10 +258,10 @@ function Geometry.CartesianDiscreteModel( @assert N == length(nc) msg if any(isperiodic) + @notimplementedif ghost != map(i->true,parts) _cartesian_model_with_periodic_bcs(ranks,parts,desc) else - ghost = map(i->true,parts) - upartition = uniform_partition(ranks,parts,nc,ghost,isperiodic) + upartition = _uniform_partition(ranks,parts,nc,ghost,isperiodic) gcids = CartesianIndices(nc) models = map(ranks,upartition) do rank, upartition cmin = gcids[first(upartition)] diff --git a/src/GridapDistributed.jl b/src/GridapDistributed.jl index 2126a71e..ed4fa324 100644 --- a/src/GridapDistributed.jl +++ b/src/GridapDistributed.jl @@ -25,6 +25,7 @@ using FillArrays using BlockArrays using LinearAlgebra using ForwardDiff +using CircularArrays import Gridap.TensorValues: inner, outer, double_contraction, symmetric_part import LinearAlgebra: det, tr, cross, dot, ⋅, diag @@ -43,6 +44,8 @@ export with_ghost, no_ghost export redistribute +include("PArraysExtras.jl") + include("BlockPartitionedArrays.jl") include("Algebra.jl") diff --git a/src/PArraysExtras.jl b/src/PArraysExtras.jl new file mode 100644 index 00000000..3e5264ee --- /dev/null +++ b/src/PArraysExtras.jl @@ -0,0 +1,84 @@ + +# This is copied from PArrays v0.5, which is not yet supported. It is necessary to get +# more than one layers of ghosts. + +function _uniform_partition(rank,np,n,args...) + @assert prod(np) == length(rank) + indices = map(rank) do rank + _block_with_constant_size(rank,np,n,args...) + end + if length(args) == 0 + map(indices) do indices + cache = assembly_cache(indices) + copy!(cache,empty_assembly_cache()) + end + else + assembly_neighbors(indices;symmetric=true) + end + indices +end + +function _block_with_constant_size(rank,np,n,ghost,periodic=map(i->false,ghost)) + N = length(n) + p = CartesianIndices(np)[rank] + own_ranges = map(_local_range,Tuple(p),np,n) + local_ranges = map(_local_range,Tuple(p),np,n,ghost,periodic) + owners = map(Tuple(p), np, n, local_ranges) do p, np, n, lr + myowners = zeros(Int32,length(lr)) + i = 1 + for p in Iterators.cycle(1:np) + plr = _local_range(p, np, n) + while mod(lr[i]-1, n)+1 in plr + myowners[i] = p + (i += 1) > length(myowners) && return myowners + end + end + end + n_local = prod(map(length, local_ranges)) + n_own = prod(map(length, own_ranges)) + n_ghost = n_local - n_own + + ghost_to_global = zeros(Int,n_ghost) + ghost_to_owner = zeros(Int32,n_ghost) + perm = zeros(Int32,n_local) + i_ghost = 0 + i_own = 0 + + cis = CartesianIndices(map(length,local_ranges)) + lis = CircularArray(LinearIndices(n)) + local_cis = CartesianIndices(local_ranges) + owner_lis = LinearIndices(np) + for (i,ci) in enumerate(cis) + flags = map(Tuple(ci), own_ranges, local_ranges) do i, or, lr + i in (or .- first(lr) .+ 1) + end + if !all(flags) + i_ghost += 1 + ghost_to_global[i_ghost] = lis[local_cis[i]] + o = map(getindex,owners,Tuple(ci)) + o_ci = CartesianIndex(o) + ghost_to_owner[i_ghost] = owner_lis[o_ci] + perm[i] = i_ghost + n_own + else + i_own += 1 + perm[i] = i_own + end + end + ghostids = GhostIndices(prod(n),ghost_to_global,ghost_to_owner) + ids = PartitionedArrays.LocalIndicesWithConstantBlockSize(p,np,n,ghostids) + PartitionedArrays.PermutedLocalIndices(ids,perm) +end + +function _local_range(p,np,n,ghost=false,periodic=false) + l, rem = divrem(n, np) + offset = l * (p-1) + if rem >= (np-p+1) + l += 1 + offset += p - (np-rem) - 1 + end + start = 1+offset-ghost + stop = l+offset+ghost + + periodic && return start:stop + return max(1, start):min(n,stop) +end diff --git a/test/ExtraGhostLayers.jl b/test/ExtraGhostLayers.jl new file mode 100644 index 00000000..0b337b26 --- /dev/null +++ b/test/ExtraGhostLayers.jl @@ -0,0 +1,36 @@ + +using Gridap +using GridapDistributed +using PartitionedArrays + + +parts = (2,2) +ranks = with_debug() do distribute + distribute(LinearIndices((prod(parts),))) +end + +nc = (4,4) +model = CartesianDiscreteModel(ranks,parts,(0,1,0,1),nc;ghost=(2,2)) + +num_cells(model) +map(num_cells,local_views(model)) + +reffe = ReferenceFE(lagrangian,Float64,1) +V = FESpace(model,reffe) + +cell_dof_ids = map(get_cell_dof_ids,local_views(V)) + +gids = get_free_dof_ids(V) +own_to_local(gids) + +Ω = Triangulation(model) +map(num_cells,local_views(Ω)) + +dΩ = Measure(Ω,2) + +f = 1.0 +a(u,v) = ∫(∇(u)⋅∇(v))dΩ +l(v) = ∫(f⋅v)dΩ + + + From c84d8e63cc9a9cb410c3b06b4a72a66542e67605 Mon Sep 17 00:00:00 2001 From: Jordi Manyer Date: Sun, 6 Jul 2025 23:05:01 +0200 Subject: [PATCH 2/4] Minor --- test/ExtraGhostLayers.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/ExtraGhostLayers.jl b/test/ExtraGhostLayers.jl index 0b337b26..e050cbe5 100644 --- a/test/ExtraGhostLayers.jl +++ b/test/ExtraGhostLayers.jl @@ -32,5 +32,7 @@ f = 1.0 a(u,v) = ∫(∇(u)⋅∇(v))dΩ l(v) = ∫(f⋅v)dΩ +op = AffineFEOperator(a,l,V,V) + From 5f1b66ac16616deef8173fd1369bdb70cbddf79e Mon Sep 17 00:00:00 2001 From: Jordi Manyer Date: Wed, 10 Sep 2025 11:20:57 +1000 Subject: [PATCH 3/4] Bugfixes --- src/Geometry.jl | 13 +++---------- test/ExtraGhostLayers.jl | 38 -------------------------------------- test/GeometryTests.jl | 3 +++ 3 files changed, 6 insertions(+), 48 deletions(-) delete mode 100644 test/ExtraGhostLayers.jl diff --git a/src/Geometry.jl b/src/Geometry.jl index e1121afe..bbda42b3 100644 --- a/src/Geometry.jl +++ b/src/Geometry.jl @@ -244,19 +244,12 @@ const DistributedCartesianDiscreteModel{Dc,Dp,A,B,C} = function Geometry.CartesianDiscreteModel( ranks::AbstractArray{<:Integer}, # Distributed array with the rank IDs parts::NTuple{N,<:Integer}, # Number of ranks (parts) in each direction - args...; kwargs... + args...; ghost = map(i -> true, parts), kwargs... ) where N - desc = CartesianDescriptor(args...;isperiodic=isperiodic,kwargs...) - nc = desc.partition - msg = """ - A CartesianDiscreteModel needs a Cartesian subdomain partition - of the right dimensions. - """ - @assert N == length(nc) msg desc = CartesianDescriptor(args...;kwargs...) @check N == length(desc.partition) @check prod(parts) == length(ranks) - pdesc = DistributedCartesianDescriptor(ranks,parts,desc) + pdesc = DistributedCartesianDescriptor(ranks,parts,desc,ghost) return CartesianDiscreteModel(pdesc) end @@ -271,7 +264,7 @@ function Geometry.CartesianDiscreteModel(pdesc::DistributedCartesianDescriptor) ranks = pdesc.ranks parts = pdesc.mesh_partition ghost = pdesc.ghost - cell_indices = uniform_partition(ranks,parts,nc,ghost,isperiodic) + cell_indices = _uniform_partition(ranks,parts,nc,ghost,isperiodic) gcids = CartesianIndices(nc) models = map(cell_indices) do cell_indices cmin = gcids[first(cell_indices)] diff --git a/test/ExtraGhostLayers.jl b/test/ExtraGhostLayers.jl deleted file mode 100644 index e050cbe5..00000000 --- a/test/ExtraGhostLayers.jl +++ /dev/null @@ -1,38 +0,0 @@ - -using Gridap -using GridapDistributed -using PartitionedArrays - - -parts = (2,2) -ranks = with_debug() do distribute - distribute(LinearIndices((prod(parts),))) -end - -nc = (4,4) -model = CartesianDiscreteModel(ranks,parts,(0,1,0,1),nc;ghost=(2,2)) - -num_cells(model) -map(num_cells,local_views(model)) - -reffe = ReferenceFE(lagrangian,Float64,1) -V = FESpace(model,reffe) - -cell_dof_ids = map(get_cell_dof_ids,local_views(V)) - -gids = get_free_dof_ids(V) -own_to_local(gids) - -Ω = Triangulation(model) -map(num_cells,local_views(Ω)) - -dΩ = Measure(Ω,2) - -f = 1.0 -a(u,v) = ∫(∇(u)⋅∇(v))dΩ -l(v) = ∫(f⋅v)dΩ - -op = AffineFEOperator(a,l,V,V) - - - diff --git a/test/GeometryTests.jl b/test/GeometryTests.jl index 430ef6c4..2d3b2893 100644 --- a/test/GeometryTests.jl +++ b/test/GeometryTests.jl @@ -109,6 +109,9 @@ function main(distribute,parts) Λs = Skeleton(Ωs) Γs = Boundary(Ωs) + # Multiple ghost layers + model = CartesianDiscreteModel(ranks,parts,domain,cells;ghost=map(i->2,parts)) + end function test_local_part_face_labelings_consistency(lmodel::CartesianDiscreteModel{D},gids,gmodel) where {D} From 9908b60f870b8c121ad450ec3942972022684262 Mon Sep 17 00:00:00 2001 From: Jordi Manyer Date: Wed, 10 Sep 2025 11:22:36 +1000 Subject: [PATCH 4/4] Updated news --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index 05ae2e7d..8eae5a98 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added support for multiple ghost layers on cartesian models. Since PR[#182](https://github.com/gridap/GridapDistributed.jl/pull/182). + ## [0.4.9] - 2025-08-08 ### Added