Skip to content

Commit f9de5de

Browse files
Dimitrios AdamDimitrios Adam
Dimitrios Adam
authored and
Dimitrios Adam
committed
...
1 parent 924f1e4 commit f9de5de

8 files changed

Lines changed: 26165 additions & 111 deletions

File tree

‎src/common/m_boundary_common.fpp‎

Lines changed: 238 additions & 36 deletions
Large diffs are not rendered by default.

‎src/common/m_chemistry.fpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -250,14 +250,9 @@ contains
250250
rho_L = q_prim_qp(1)%sf(x, y, z)
251251
rho_R = q_prim_qp(1)%sf(x + offsets(1), y + offsets(2), z + offsets(3))
252252

253-
if ( x .eq. -1 .or. x .eq. 499 .or. y .eq. -1 .or. y .eq. 499) then
254-
T_L = P_L/rho_L/Rgas_L
255-
T_R = P_R/rho_R/Rgas_R
256253

257-
else
258254
T_L =q_T_sf%sf(x, y, z)! P_L/rho_L/Rgas_L
259255
T_R = q_T_sf%sf(x + offsets(1), y + offsets(2), z + offsets(3))! P_R/rho_R/Rgas_R
260-
end if
261256

262257

263258
rho_cell = 0.5_wp*(rho_L + rho_R)

‎src/common/m_mpi_common.fpp‎

Lines changed: 121 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ contains
5959
6060
if (qbmm .and. .not. polytropic) then
6161
v_size = sys_size + 2*nb*4
62+
else if (chemistry) then
63+
v_size = sys_size + 1
6264
else
6365
v_size = sys_size
6466
end if
@@ -124,15 +126,11 @@ contains
124126
125127
!! @param q_cons_vf Conservative variables
126128
!! @param ib_markers track if a cell is within the immersed boundary
127-
!! @param levelset closest distance from every cell to the IB
128-
!! @param levelset_norm normalized vector from every cell to the closest point to the IB
129129
!! @param beta Eulerian void fraction from lagrangian bubbles
130-
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, levelset, levelset_norm, beta)
130+
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, beta)
131131
132132
type(scalar_field), dimension(sys_size), intent(in) :: q_cons_vf
133133
type(integer_field), optional, intent(in) :: ib_markers
134-
type(levelset_field), optional, intent(IN) :: levelset
135-
type(levelset_norm_field), optional, intent(IN) :: levelset_norm
136134
type(scalar_field), intent(in), optional :: beta
137135
138136
integer, dimension(num_dims) :: sizes_glb, sizes_loc
@@ -203,73 +201,13 @@ contains
203201
end if
204202
#endif
205203
204+
#ifndef MFC_PRE_PROCESS
206205
if (present(ib_markers)) then
207-
208-
#ifdef MFC_PRE_PROCESS
209-
MPI_IO_IB_DATA%var%sf => ib_markers%sf
210-
MPI_IO_levelset_DATA%var%sf => levelset%sf
211-
MPI_IO_levelsetnorm_DATA%var%sf => levelset_norm%sf
212-
#else
213206
MPI_IO_IB_DATA%var%sf => ib_markers%sf(0:m, 0:n, 0:p)
214207
215-
#ifndef MFC_POST_PROCESS
216-
MPI_IO_levelset_DATA%var%sf => levelset%sf(0:m, 0:n, 0:p, 1:num_ibs)
217-
MPI_IO_levelsetnorm_DATA%var%sf => levelset_norm%sf(0:m, 0:n, 0:p, 1:num_ibs, 1:3)
218-
#endif
219-
220-
#endif
221208
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, start_idx, &
222209
MPI_ORDER_FORTRAN, MPI_INTEGER, MPI_IO_IB_DATA%view, ierr)
223210
call MPI_TYPE_COMMIT(MPI_IO_IB_DATA%view, ierr)
224-
225-
#ifndef MFC_POST_PROCESS
226-
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, start_idx, &
227-
MPI_ORDER_FORTRAN, mpi_p, MPI_IO_levelset_DATA%view, ierr)
228-
call MPI_TYPE_CREATE_SUBARRAY(num_dims, sizes_glb, sizes_loc, start_idx, &
229-
MPI_ORDER_FORTRAN, mpi_p, MPI_IO_levelsetnorm_DATA%view, ierr)
230-
231-
call MPI_TYPE_COMMIT(MPI_IO_levelset_DATA%view, ierr)
232-
call MPI_TYPE_COMMIT(MPI_IO_levelsetnorm_DATA%view, ierr)
233-
#endif
234-
end if
235-
236-
#ifndef MFC_POST_PROCESS
237-
if (present(ib_markers)) then
238-
do j = 1, num_ibs
239-
if (patch_ib(j)%c > 0) then
240-
241-
#ifdef MFC_PRE_PROCESS
242-
allocate (MPI_IO_airfoil_IB_DATA%var(1:2*Np))
243-
#endif
244-
245-
airfoil_glb(1) = 3*Np*num_procs
246-
airfoil_loc(1) = 3*Np
247-
airfoil_start(1) = 3*proc_rank*Np
248-
249-
#ifdef MFC_PRE_PROCESS
250-
do i = 1, Np
251-
MPI_IO_airfoil_IB_DATA%var(i)%x = airfoil_grid_l(i)%x
252-
MPI_IO_airfoil_IB_DATA%var(i)%y = airfoil_grid_l(i)%y
253-
end do
254-
#endif
255-
256-
call MPI_TYPE_CREATE_SUBARRAY(1, airfoil_glb, airfoil_loc, airfoil_start, &
257-
MPI_ORDER_FORTRAN, mpi_p, MPI_IO_airfoil_IB_DATA%view(1), ierr)
258-
call MPI_TYPE_COMMIT(MPI_IO_airfoil_IB_DATA%view(1), ierr)
259-
260-
#ifdef MFC_PRE_PROCESS
261-
do i = 1, Np
262-
MPI_IO_airfoil_IB_DATA%var(Np + i)%x = airfoil_grid_u(i)%x
263-
MPI_IO_airfoil_IB_DATA%var(Np + i)%y = airfoil_grid_u(i)%y
264-
end do
265-
#endif
266-
call MPI_TYPE_CREATE_SUBARRAY(1, airfoil_glb, airfoil_loc, airfoil_start, &
267-
MPI_ORDER_FORTRAN, mpi_p, MPI_IO_airfoil_IB_DATA%view(2), ierr)
268-
call MPI_TYPE_COMMIT(MPI_IO_airfoil_IB_DATA%view(2), ierr)
269-
270-
end if
271-
end do
272-
273211
end if
274212
#endif
275213
@@ -705,10 +643,11 @@ contains
705643
mpi_dir, &
706644
pbc_loc, &
707645
nVar, &
708-
pb_in, mv_in)
646+
pb_in, mv_in, q_T_sf)
709647
710648
type(scalar_field), dimension(1:), intent(inout) :: q_comm
711649
real(stp), optional, dimension(idwbuff(1)%beg:, idwbuff(2)%beg:, idwbuff(3)%beg:, 1:, 1:), intent(inout) :: pb_in, mv_in
650+
type(scalar_field), optional, intent(inout) :: q_T_sf
712651
integer, intent(in) :: mpi_dir, pbc_loc, nVar
713652
714653
integer :: i, j, k, l, r, q !< Generic loop iterators
@@ -738,6 +677,13 @@ contains
738677
buff_size*v_size*(m + 2*buff_size + 1)*(p + 1), &
739678
buff_size*v_size*(m + 2*buff_size + 1)*(n + 2*buff_size + 1) &
740679
/)
680+
else if (chemistry) then
681+
v_size = nVar + 1
682+
buffer_counts = (/ &
683+
buff_size*v_size*(n + 1)*(p + 1), &
684+
buff_size*v_size*(m + 2*buff_size + 1)*(p + 1), &
685+
buff_size*v_size*(m + 2*buff_size + 1)*(n + 2*buff_size + 1) &
686+
/)
741687
else
742688
v_size = nVar
743689
buffer_counts = (/ &
@@ -796,6 +742,21 @@ contains
796742
end do
797743
$:END_GPU_PARALLEL_LOOP()
798744
745+
if (chemistry) then
746+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
747+
do l = 0, p
748+
do k = 0, n
749+
do j = 0, buff_size - 1
750+
! (v_size - 1) puts T at the end of the cell's buffer chunk
751+
r = (v_size - 1) + v_size* &
752+
(j + buff_size*(k + (n + 1)*l))
753+
buff_send(r) = real(q_T_sf%sf(j + pack_offset, k, l), kind=wp)
754+
end do
755+
end do
756+
end do
757+
$:END_GPU_PARALLEL_LOOP()
758+
end if
759+
799760
if (qbmm_comm) then
800761
$:GPU_PARALLEL_LOOP(collapse=4,private='[r]')
801762
do l = 0, p
@@ -845,6 +806,21 @@ contains
845806
end do
846807
$:END_GPU_PARALLEL_LOOP()
847808

809+
if (chemistry) then
810+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
811+
do l = 0, p
812+
do k = 0, buff_size - 1
813+
do j = -buff_size, m + buff_size
814+
r = nVar + v_size* &
815+
((j + buff_size) + (m + 2*buff_size + 1)* &
816+
(k + buff_size*l))
817+
buff_send(r) = real(q_T_sf%sf(j, k + pack_offset, l), kind=wp)
818+
end do
819+
end do
820+
end do
821+
$:END_GPU_PARALLEL_LOOP()
822+
end if
823+
848824
if (qbmm_comm) then
849825
$:GPU_PARALLEL_LOOP(collapse=5,private='[r]')
850826
do i = nVar + 1, nVar + 4
@@ -896,6 +872,21 @@ contains
896872
end do
897873
$:END_GPU_PARALLEL_LOOP()
898874

875+
if (chemistry) then
876+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
877+
do l = 0, buff_size - 1
878+
do k = -buff_size, n + buff_size
879+
do j = -buff_size, m + buff_size
880+
r = nVar + v_size* &
881+
((j + buff_size) + (m + 2*buff_size + 1)* &
882+
((k + buff_size) + (n + 2*buff_size + 1)*l))
883+
buff_send(r) = real(q_T_sf%sf(j, k, l + pack_offset), kind=wp)
884+
end do
885+
end do
886+
end do
887+
$:END_GPU_PARALLEL_LOOP()
888+
end if
889+
899890
if (qbmm_comm) then
900891
$:GPU_PARALLEL_LOOP(collapse=5,private='[r]')
901892
do i = nVar + 1, nVar + 4
@@ -1004,6 +995,26 @@ contains
1004995
end do
1005996
$:END_GPU_PARALLEL_LOOP()
1006997

998+
if (chemistry) then
999+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
1000+
do l = 0, p
1001+
do k = 0, n
1002+
do j = -buff_size, -1
1003+
r = nVar + v_size* &
1004+
(j + buff_size*((k + 1) + (n + 1)*l))
1005+
q_T_sf%sf(j + unpack_offset, k, l) = real(buff_recv(r), kind=stp)
1006+
#if defined(__INTEL_COMPILER)
1007+
if (ieee_is_nan(q_T_sf%sf(j, k, l))) then
1008+
print *, "Error", j, k, l, i
1009+
error stop "NaN(s) in recv"
1010+
end if
1011+
#endif
1012+
end do
1013+
end do
1014+
end do
1015+
$:END_GPU_PARALLEL_LOOP()
1016+
end if
1017+
10071018
if (qbmm_comm) then
10081019
$:GPU_PARALLEL_LOOP(collapse=5,private='[r]')
10091020
do l = 0, p
@@ -1059,6 +1070,27 @@ contains
10591070
end do
10601071
$:END_GPU_PARALLEL_LOOP()
10611072

1073+
if (chemistry) then
1074+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
1075+
do l = 0, p
1076+
do k = -buff_size, -1
1077+
do j = -buff_size, m + buff_size
1078+
r = nVar + v_size* &
1079+
((j + buff_size) + (m + 2*buff_size + 1)* &
1080+
((k + buff_size) + buff_size*l))
1081+
q_T_sf%sf(j, k + unpack_offset, l) = real(buff_recv(r), kind=stp)
1082+
#if defined(__INTEL_COMPILER)
1083+
if (ieee_is_nan(q_T_sf%sf(j, k, l))) then
1084+
print *, "Error", j, k, l, i
1085+
error stop "NaN(s) in recv"
1086+
end if
1087+
#endif
1088+
end do
1089+
end do
1090+
end do
1091+
$:END_GPU_PARALLEL_LOOP()
1092+
end if
1093+
10621094
if (qbmm_comm) then
10631095
$:GPU_PARALLEL_LOOP(collapse=5,private='[r]')
10641096
do i = nVar + 1, nVar + 4
@@ -1118,6 +1150,28 @@ contains
11181150
end do
11191151
$:END_GPU_PARALLEL_LOOP()
11201152

1153+
if (chemistry) then
1154+
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
1155+
do l = -buff_size, -1
1156+
do k = -buff_size, n + buff_size
1157+
do j = -buff_size, m + buff_size
1158+
r = nVar + v_size* &
1159+
((j + buff_size) + (m + 2*buff_size + 1)* &
1160+
((k + buff_size) + (n + 2*buff_size + 1)* &
1161+
(l + buff_size)))
1162+
q_T_sf%sf(j, k, l + unpack_offset) = real(buff_recv(r), kind=stp)
1163+
#if defined(__INTEL_COMPILER)
1164+
if (ieee_is_nan(q_T_sf%sf(j, k, l))) then
1165+
print *, "Error", j, k, l, i
1166+
error stop "NaN(s) in recv"
1167+
end if
1168+
#endif
1169+
end do
1170+
end do
1171+
end do
1172+
$:END_GPU_PARALLEL_LOOP()
1173+
end if
1174+
11211175
if (qbmm_comm) then
11221176
$:GPU_PARALLEL_LOOP(collapse=5,private='[r]')
11231177
do i = nVar + 1, nVar + 4

‎src/pre_process/m_initial_condition.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ contains
9393
end do
9494

9595
if (chemistry) then
96-
allocate (q_T_sf%sf(0:m, 0:n, 0:p))
96+
allocate (q_T_sf%sf(idwbuff(1)%beg:idwbuff(1)%end, idwbuff(2)%beg:idwbuff(2)%end, idwbuff(3)%beg:idwbuff(3)%end))
9797
end if
9898

9999
! Allocating the patch identities bookkeeping variable

‎src/simulation/m_rhs.fpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,7 @@ contains
724724
call nvtxEndRange
725725
726726
call nvtxStartRange("RHS-COMMUNICATION")
727-
call s_populate_variables_buffers(bc_type, q_prim_qp%vf, pb_in, mv_in)
727+
call s_populate_variables_buffers(bc_type, q_prim_qp%vf, pb_in , mv_in, q_T_sf)
728728
call nvtxEndRange
729729
end if
730730
@@ -954,7 +954,7 @@ contains
954954
! RHS for diffusion
955955
if (chemistry .and. chem_params%diffusion) then
956956
call nvtxStartRange("RHS-CHEM-DIFFUSION")
957-
call s_compute_chemistry_diffusion_flux(id, q_prim_qp%vf, flux_src_n(id)%vf, irx, iry, irz,q_T_sf)
957+
call s_compute_chemistry_diffusion_flux(id, q_prim_qp%vf, flux_src_n(id)%vf, irx, iry, irz, q_T_sf)
958958
call nvtxEndRange
959959
end if
960960

0 commit comments

Comments
 (0)