Skip to content

Commit 21e1e5d

Browse files
sbryngelsonclaude
sbryngelson
and
claude
committed
Fix wp -> stp in pre_process I/O and remove dead Lustre block
Pre-process MPI I/O strides should use storage precision (stp) not working precision (wp) since field arrays use stp. Also removes the dead file_per_process sequential-read path that contained known copy-paste bugs (n_MOK/p_MOK both using m_glb_read). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 2f6dd04 commit 21e1e5d

3 files changed

Lines changed: 61 additions & 154 deletions

File tree

‎src/pre_process/m_data_output.fpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ contains
549549
m_MOK = int(m_glb_save, MPI_OFFSET_KIND)
550550
n_MOK = int(n_glb_save, MPI_OFFSET_KIND)
551551
p_MOK = int(p_glb_save, MPI_OFFSET_KIND)
552-
WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND)
552+
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
553553
MOK = int(1._wp, MPI_OFFSET_KIND)
554554
str_MOK = int(name_len, MPI_OFFSET_KIND)
555555
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)
@@ -615,7 +615,7 @@ contains
615615
m_MOK = int(m_glb + 1, MPI_OFFSET_KIND)
616616
n_MOK = int(n_glb + 1, MPI_OFFSET_KIND)
617617
p_MOK = int(p_glb + 1, MPI_OFFSET_KIND)
618-
WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND)
618+
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
619619
MOK = int(1._wp, MPI_OFFSET_KIND)
620620
str_MOK = int(name_len, MPI_OFFSET_KIND)
621621
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)

‎src/pre_process/m_start_up.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,7 @@ contains
660660
m_MOK = int(m_glb + 1, MPI_OFFSET_KIND)
661661
n_MOK = int(n_glb + 1, MPI_OFFSET_KIND)
662662
p_MOK = int(p_glb + 1, MPI_OFFSET_KIND)
663-
WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND)
663+
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
664664
MOK = int(1._wp, MPI_OFFSET_KIND)
665665
str_MOK = int(name_len, MPI_OFFSET_KIND)
666666
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)

‎src/simulation/m_start_up.fpp‎

Lines changed: 58 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -532,134 +532,55 @@ contains
532532
end if
533533
end if
534534

535-
if (file_per_process) then
536-
if (cfl_dt) then
537-
call s_int_to_str(n_start, t_step_start_string)
538-
write (file_loc, '(I0,A1,I7.7,A)') n_start, '_', proc_rank, '.dat'
539-
else
540-
call s_int_to_str(t_step_start, t_step_start_string)
541-
write (file_loc, '(I0,A1,I7.7,A)') t_step_start, '_', proc_rank, '.dat'
542-
end if
543-
file_loc = trim(case_dir)//'/restart_data/lustre_'//trim(t_step_start_string)//trim(mpiiofs)//trim(file_loc)
544-
inquire (FILE=trim(file_loc), EXIST=file_exist)
545-
546-
if (file_exist) then
547-
call MPI_FILE_OPEN(MPI_COMM_SELF, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr)
548-
549-
! Initialize MPI data I/O
550-
if (down_sample) then
551-
call s_initialize_mpi_data_ds(q_cons_vf)
552-
else
553-
if (ib) then
554-
call s_initialize_mpi_data(q_cons_vf, ib_markers)
555-
else
556-
call s_initialize_mpi_data(q_cons_vf)
557-
end if
558-
end if
559-
560-
if (down_sample) then
561-
! Size of local arrays
562-
data_size = (m_ds + 3)*(n_ds + 3)*(p_ds + 3)
563-
m_glb_read = m_glb_ds + 1
564-
n_glb_read = n_glb_ds + 1
565-
p_glb_read = p_glb_ds + 1
566-
else
567-
! Size of local arrays
568-
data_size = (m + 1)*(n + 1)*(p + 1)
569-
m_glb_read = m_glb + 1
570-
n_glb_read = n_glb + 1
571-
p_glb_read = p_glb + 1
572-
end if
573-
574-
! Resize some integers so MPI can read even the biggest file
575-
m_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND)
576-
n_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND)
577-
p_MOK = int(m_glb_read + 1, MPI_OFFSET_KIND)
578-
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
579-
MOK = int(1._wp, MPI_OFFSET_KIND)
580-
str_MOK = int(name_len, MPI_OFFSET_KIND)
581-
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)
582-
583-
! Read the data for each variable
584-
if (bubbles_euler .or. elasticity) then
585-
do i = 1, sys_size!adv_idx%end
586-
var_MOK = int(i, MPI_OFFSET_KIND)
587-
588-
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
589-
mpi_io_p, status, ierr)
590-
end do
591-
!Read pb and mv for non-polytropic qbmm
592-
if (qbmm .and. .not. polytropic) then
593-
do i = sys_size + 1, sys_size + 2*nb*nnode
594-
var_MOK = int(i, MPI_OFFSET_KIND)
595-
596-
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
597-
mpi_io_p, status, ierr)
598-
end do
599-
end if
600-
else
601-
if (down_sample) then
602-
do i = 1, sys_size
603-
var_MOK = int(i, MPI_OFFSET_KIND)
604-
605-
call MPI_FILE_READ(ifile, q_cons_temp(i)%sf, data_size*mpi_io_type, &
606-
mpi_io_p, status, ierr)
607-
end do
608-
else
609-
do i = 1, sys_size
610-
var_MOK = int(i, MPI_OFFSET_KIND)
611-
612-
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
613-
mpi_io_p, status, ierr)
614-
end do
615-
end if
616-
end if
617-
618-
call s_mpi_barrier()
619-
620-
call MPI_FILE_CLOSE(ifile, ierr)
621-
622-
else
623-
call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.')
624-
end if
535+
! Open the file to read conservative variables
536+
if (cfl_dt) then
537+
write (file_loc, '(I0,A)') n_start, '.dat'
625538
else
626-
! Open the file to read conservative variables
627-
if (cfl_dt) then
628-
write (file_loc, '(I0,A)') n_start, '.dat'
629-
else
630-
write (file_loc, '(I0,A)') t_step_start, '.dat'
631-
end if
632-
file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc)
633-
inquire (FILE=trim(file_loc), EXIST=file_exist)
539+
write (file_loc, '(I0,A)') t_step_start, '.dat'
540+
end if
541+
file_loc = trim(case_dir)//'/restart_data'//trim(mpiiofs)//trim(file_loc)
542+
inquire (FILE=trim(file_loc), EXIST=file_exist)
634543

635-
if (file_exist) then
636-
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr)
544+
if (file_exist) then
545+
call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, MPI_MODE_RDONLY, mpi_info_int, ifile, ierr)
637546

638-
! Initialize MPI data I/O
547+
! Initialize MPI data I/O
639548

640-
if (ib) then
641-
call s_initialize_mpi_data(q_cons_vf, ib_markers)
642-
else
549+
if (ib) then
550+
call s_initialize_mpi_data(q_cons_vf, ib_markers)
551+
else
643552

644-
call s_initialize_mpi_data(q_cons_vf)
553+
call s_initialize_mpi_data(q_cons_vf)
645554

646-
end if
555+
end if
647556

648-
! Size of local arrays
649-
data_size = (m + 1)*(n + 1)*(p + 1)
650-
651-
! Resize some integers so MPI can read even the biggest file
652-
m_MOK = int(m_glb + 1, MPI_OFFSET_KIND)
653-
n_MOK = int(n_glb + 1, MPI_OFFSET_KIND)
654-
p_MOK = int(p_glb + 1, MPI_OFFSET_KIND)
655-
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
656-
MOK = int(1._wp, MPI_OFFSET_KIND)
657-
str_MOK = int(name_len, MPI_OFFSET_KIND)
658-
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)
659-
660-
! Read the data for each variable
661-
if (bubbles_euler .or. elasticity) then
662-
do i = 1, sys_size !adv_idx%end
557+
! Size of local arrays
558+
data_size = (m + 1)*(n + 1)*(p + 1)
559+
560+
! Resize some integers so MPI can read even the biggest file
561+
m_MOK = int(m_glb + 1, MPI_OFFSET_KIND)
562+
n_MOK = int(n_glb + 1, MPI_OFFSET_KIND)
563+
p_MOK = int(p_glb + 1, MPI_OFFSET_KIND)
564+
WP_MOK = int(storage_size(0._stp)/8, MPI_OFFSET_KIND)
565+
MOK = int(1._wp, MPI_OFFSET_KIND)
566+
str_MOK = int(name_len, MPI_OFFSET_KIND)
567+
NVARS_MOK = int(sys_size, MPI_OFFSET_KIND)
568+
569+
! Read the data for each variable
570+
if (bubbles_euler .or. elasticity) then
571+
do i = 1, sys_size !adv_idx%end
572+
var_MOK = int(i, MPI_OFFSET_KIND)
573+
! Initial displacement to skip at beginning of file
574+
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)
575+
576+
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), &
577+
'native', mpi_info_int, ierr)
578+
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
579+
mpi_io_p, status, ierr)
580+
end do
581+
!Read pb and mv for non-polytropic qbmm
582+
if (qbmm .and. .not. polytropic) then
583+
do i = sys_size + 1, sys_size + 2*nb*nnode
663584
var_MOK = int(i, MPI_OFFSET_KIND)
664585
! Initial displacement to skip at beginning of file
665586
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)
@@ -669,41 +590,27 @@ contains
669590
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
670591
mpi_io_p, status, ierr)
671592
end do
672-
!Read pb and mv for non-polytropic qbmm
673-
if (qbmm .and. .not. polytropic) then
674-
do i = sys_size + 1, sys_size + 2*nb*nnode
675-
var_MOK = int(i, MPI_OFFSET_KIND)
676-
! Initial displacement to skip at beginning of file
677-
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)
678-
679-
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), &
680-
'native', mpi_info_int, ierr)
681-
call MPI_FILE_READ(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
682-
mpi_io_p, status, ierr)
683-
end do
684-
end if
685-
else
686-
do i = 1, sys_size
687-
var_MOK = int(i, MPI_OFFSET_KIND)
688-
689-
! Initial displacement to skip at beginning of file
690-
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)
691-
692-
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), &
693-
'native', mpi_info_int, ierr)
694-
call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
695-
mpi_io_p, status, ierr)
696-
end do
697593
end if
594+
else
595+
do i = 1, sys_size
596+
var_MOK = int(i, MPI_OFFSET_KIND)
698597

699-
call s_mpi_barrier()
700-
701-
call MPI_FILE_CLOSE(ifile, ierr)
598+
! Initial displacement to skip at beginning of file
599+
disp = m_MOK*max(MOK, n_MOK)*max(MOK, p_MOK)*WP_MOK*(var_MOK - 1)
702600

703-
else
704-
call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.')
601+
call MPI_FILE_SET_VIEW(ifile, disp, mpi_io_p, MPI_IO_DATA%view(i), &
602+
'native', mpi_info_int, ierr)
603+
call MPI_FILE_READ_ALL(ifile, MPI_IO_DATA%var(i)%sf, data_size*mpi_io_type, &
604+
mpi_io_p, status, ierr)
605+
end do
705606
end if
706607

608+
call s_mpi_barrier()
609+
610+
call MPI_FILE_CLOSE(ifile, ierr)
611+
612+
else
613+
call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting.')
707614
end if
708615

709616
deallocate (x_cb_glb, y_cb_glb, z_cb_glb)

0 commit comments

Comments
 (0)