Skip to content

Commit dccf4b9

Browse files
kestonsmith-noaakestonsmith-noaa
authored
Communication is added after boundary conditions are applied in PDLIB_JACOBI_GAUSS_SEIDEL_BLOCK to enable bit for bit reproducibility across different numbers of MPI processes. (NOAA-EMC#1538)
1 parent 57f0705 commit dccf4b9

28 files changed

Lines changed: 2074 additions & 1 deletion

‎model/bin/ww3_from_ftp.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ cp -r data_regtests/ww3_tp2.14/input/toy/toy_coupled_field.nc.OASACM regtests/ww
105105
cp -r data_regtests/ww3_tp2.14/input/toy/toy_coupled_field.nc.OASACM regtests/ww3_tp2.14/input/toy/toy_coupled_field.nc.OASACM6
106106
cp -r data_regtests/ww3_tp2.14/input/toy/*.nc regtests/ww3_tp2.14/input/toy/
107107
cp -r data_regtests/ww3_tp2.17/input/* regtests/ww3_tp2.17/input/
108+
cp -r data_regtests/ww3_tp2.17/input_ice/* regtests/ww3_tp2.17/input_ice/
108109
cp -r data_regtests/ww3_tp2.19/input/* regtests/ww3_tp2.19/input/
109110
cp -r data_regtests/ww3_tp2.21/input/* regtests/ww3_tp2.21/input/
110111
cp -r data_regtests/mww3_test_09/input/* regtests/mww3_test_09/input/

‎model/src/w3profsmd_pdlib.F90‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5785,6 +5785,10 @@ SUBROUTINE PDLIB_JACOBI_GAUSS_SEIDEL_BLOCK(IMOD, FACX, FACY, DTG, VGX, VGY, LCAL
57855785
ENDIF
57865786
END IF
57875787
CALL APPLY_BOUNDARY_CONDITION(IMOD)
5788+
!
5789+
! for reproducability state must be communicated after BC application
5790+
!
5791+
CALL PDLIB_exchange2DREAL_zero(VA)
57885792
call print_memcheck(memunit, 'memcheck_____:'//' WW3_PROP SECTION 6')
57895793
!
57905794
#ifdef W3_DEBUGSOLVERCOH

‎regtests/bin/matrix.base‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2019,13 +2019,18 @@
20192019
if [ "$pdlib" = 'y' ] && [ "$dist" = 'y' ]
20202020
then
20212021
echo ' ' >> matrix.body
2022+
2023+
20222024
echo "$rtst -s MPI -s NO_PDLIB -w work_a -g a -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20232025
echo "$rtst -s MPI -s NO_PDLIB -w work_ma -m grdset_a -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20242026
echo "$rtst -s MPI -s PDLIB -w work_b -g b -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20252027
echo "$rtst -s MPI -s PDLIB -w work_c -g c -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20262028
echo "$rtst -s MPI -s PDLIB -w work_pdlib -g pdlib -f -p $mpi -n $np $ww3 ww3_tp2.6" >> matrix.body
20272029
echo "$rtst -s MPI -s PDLIB -w work_mb -m grdset_b -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20282030
echo "$rtst -s MPI -s PDLIB -w work_mc -m grdset_c -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
2031+
echo "$rtst -s MPI -s PDLIB -i input_ice -w work_ice -g d -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
2032+
echo "$rtst -s MPI -s PDLIB -i input_ice -w work_iceB -g d -f -p $mpi -n $(( $np -1 )) $ww3 ww3_tp2.17" >> matrix.body
2033+
echo "./bin/test.comp ww3_tp2.17 work_ice work_iceB" >> matrix.body
20292034
if [ "$rstrt_b4b" = 'y' ]
20302035
then
20312036
echo "mkdir -p ww3_tp2.17/work_ma1" >> matrix.body
@@ -2036,6 +2041,11 @@
20362041
echo "cp ww3_tp2.17/work_mc/restart001.inlc ww3_tp2.17/work_mc1/restart.inlc" >> matrix.body
20372042
echo "$rtst -s MPI -s PDLIB -w work_mc1 -m grdset_c1 -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
20382043
echo "./bin/test.comp ww3_tp2.17 work_mc work_mc1" >> matrix.body
2044+
2045+
echo "mkdir -p ww3_tp2.17/work_ice_restart" >> matrix.body
2046+
echo "cp ww3_tp2.17/work_ice/20151214.120000.restart.ww3 ww3_tp2.17/work_ice_restart/restart.ww3" >> matrix.body
2047+
echo "$rtst -s MPI -s PDLIB -i input_ice_restart -w work_ice_restart -g d -f -p $mpi -n $np $ww3 ww3_tp2.17" >> matrix.body
2048+
echo "./bin/test.comp ww3_tp2.17 work_ice work_ice_restart" >> matrix.body
20392049
fi
20402050
fi
20412051

‎regtests/ww3_tp2.17/info‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,31 @@
4848
# Ali Abdolali, Aron Roland, Jessica Meixner, May 2018 #
4949
# Last Mod : May 2018 #
5050
# #
51-
#############################################################################
51+
#
52+
#---------------------------------------------------------------------------#
53+
#
54+
#---------------------------------------------------------------------------#
55+
# Addendum Keston Smith, Jessica Meixner, November 2025
56+
# A test is added with ice forcing in addition to currents, water level, #
57+
# and boundary forcing. This test exhibits failure to restart reproduce bit #
58+
# for bit when both boundary forcing and water level are present. #
59+
#
60+
# The full: ice, wind, water level, currents and boundary forcing can be run
61+
# with the following script:
62+
#
63+
#mkdir ww3_tp2.17/work_ice_restart24
64+
#
65+
# ./bin/run_cmake_test -b slurm -o all -S -T -s MPI -s PDLIB -i input_ice -w work_ice -g d -f -p srun -n 24 ../model ww3_tp2.17
66+
#
67+
# cp ww3_tp2.17/work_ice/20151214.120000.restart.ww3 ww3_tp2.17/work_ice_restart24/restart.ww3
68+
#
69+
# ./bin/run_cmake_test -b slurm -o all -S -T -s MPI -s PDLIB -i input_ice_restart -w work_ice_restart24 -g d -f -p srun -n 24 ../model ww3_tp2.17
70+
# ./bin/run_cmake_test -b slurm -o all -S -T -s MPI -s PDLIB -i input_ice -w work_ice20 -g d -f -p srun -n 20 ../model ww3_tp2.17
71+
#
72+
# Bit for bit reproducability for different numbers of MPI processes #
73+
# is evident in comparison of solutions in work_ice and work_ice20 #
74+
# (cold starts) with 24 and 20 MPI processes respectivly. Restart #
75+
# reproducability can be checked by comparing solution in work_ice/ to #
76+
# work_ice_restart24. #
77+
#---------------------------------------------------------------------------#
78+
############################################################################

‎regtests/ww3_tp2.17/input_ice/switch_PDLIB‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB SCOTCH LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎regtests/ww3_tp2.17/input_ice/ww3_bounc.inp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
$ -------------------------------------------------------------------- $
2+
$ WAVEWATCH III NetCDF boundary input processing $
3+
$--------------------------------------------------------------------- $
4+
$
5+
$ Boundary option: READ or WRITE
6+
$
7+
WRITE
8+
$
9+
$ Interpolation method: 1: nearest
10+
$ 2: linear interpolation
11+
2
12+
$ Verbose (0, 1, 2)
13+
1
14+
$
15+
$ List of spectra files. These NetCDF files use the WAVEWATCH III
16+
$ format as described in the ww3_ounp.inp file. The files are
17+
$ defined relative to the directory in which the program is run.
18+
$
19+
../input/bound.nc
20+
'STOPSTRING'
21+
$
22+
$ -------------------------------------------------------------------- $
23+
$ End of input file $
24+
$ -------------------------------------------------------------------- $
25+

‎regtests/ww3_tp2.17/input_ice/ww3_grib.inp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
$ -------------------------------------------------------------------- $
2+
$ WAVEWATCH-III gridded output input file
3+
$ ----------------------------------------
4+
20151214 000000 3600 9
5+
N
6+
WND CUR ICE HS T01 T02 DIR FP DP PHS PTP PDIR UST
7+
$
8+
20151214 000000 7 11 255 0 0
9+
$
10+
$ end of input file

0 commit comments

Comments
 (0)