Skip to content

Commit 033e390

Browse files
hplin-ucarhplin-ucar
committed
Merge branch 'development' into hplin/tracer_data
2 parents 0fbbbda + 8c0095a commit 033e390

19 files changed

Lines changed: 175 additions & 41 deletions

‎.gitmodules‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[submodule "ccpp-framework"]
22
path = ccpp_framework
33
url = https://github.com/NCAR/ccpp-framework
4-
fxtag = 2025-10-15-dev
4+
fxtag = sima_2026-01-20
55
fxrequired = AlwaysRequired
66
fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
77
[submodule "history"]
@@ -20,7 +20,7 @@
2020
[submodule "ncar-physics"]
2121
path = src/physics/ncar_ccpp
2222
url = https://github.com/ESCOMP/atmospheric_physics
23-
fxtag = b47246dedfa7c377ac2bfb060e2d465beb931f2a
23+
fxtag = 87e76a6ce90767d1d32259ff3f11dec9c9a03b03
2424
fxrequired = AlwaysRequired
2525
fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
2626
[submodule "rrtmgp-data"]
@@ -32,7 +32,7 @@
3232
[submodule "ccs_config"]
3333
path = ccs_config
3434
url = https://github.com/ESMCI/ccs_config_cesm.git
35-
fxtag = ccs_config_cesm1.0.65
35+
fxtag = ccs_config_cesm1.0.72
3636
fxrequired = ToplevelRequired
3737
fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
3838
[submodule "cdeps"]

‎ccpp_framework‎

Submodule ccpp_framework updated 108 files

‎ccs_config‎

‎cime_config/cam_autogen.py‎

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,68 @@ def _update_genccpp_dir(utility_files, genccpp_dir):
357357
# end if
358358
# end for
359359

360+
###############################################################################
361+
def _set_rrtmgp_dependencies(dependency_files, gpu_flag):
362+
###############################################################################
363+
"""
364+
Modify the list of files that physics schemes depend on
365+
(as provided by the CCPP) so that the correct RRTMGP
366+
dependencies exist for either CPUs or GPUs.
367+
368+
This function returns a modified list with either the
369+
CPU or GPU RRTMGP dependencies, but not both.
370+
371+
>>> in_files = ['/some/path/file.F90','/some/rte-kernels/rrtmgp.F90', \
372+
'/some/rte-kernels/accel/rrtmgp.F90', \
373+
'/some/rte-kernels/mo_rte_solver_kernels.F90', \
374+
'/some/rte-kernels/accel/mo_rte_solver_kernels.F90', \
375+
'/some/other/path/file.F90', \
376+
'/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90', \
377+
'/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90', ]
378+
>>> _set_rrtmgp_dependencies(in_files, False)
379+
['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
380+
'/some/rte-kernels/accel/rrtmgp.F90', '/some/rte-kernels/mo_rte_solver_kernels.F90', \
381+
'/some/other/path/file.F90', '/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90']
382+
383+
>>> _set_rrtmgp_dependencies(in_files, True)
384+
['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
385+
'/some/rte-kernels/accel/rrtmgp.F90', \
386+
'/some/rte-kernels/accel/mo_rte_solver_kernels.F90', '/some/other/path/file.F90', \
387+
'/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90']
388+
"""
389+
390+
# Create new list of CCPP-dependent files:
391+
new_dependency_files = []
392+
393+
# List of directory strings that indicate
394+
# RRTMGP dependencies:
395+
rrtmgp_subdirs = ['rte-kernels', 'rrtmgp-kernels']
396+
397+
# List of RRTMGP files that differ between CPUs and GPUs:
398+
rrtmgp_files = ['mo_rte_solver_kernels.F90',
399+
'mo_optical_props_kernels.F90',
400+
'mo_gas_optics_rrtmgp_kernels.F90']
401+
402+
403+
for file_path in dependency_files:
404+
if any(subdir in file_path for subdir in rrtmgp_subdirs):
405+
if any(rfile in file_path for rfile in rrtmgp_files):
406+
if (gpu_flag and 'accel' in file_path):
407+
#If GPU-enabled and is "accelerated", include it:
408+
new_dependency_files.append(file_path)
409+
elif (not gpu_flag and 'accel' not in file_path):
410+
#If CPU-only and not "accelerated", include it:
411+
new_dependency_files.append(file_path)
412+
else:
413+
#Not a hardware-dependent file, so include it:
414+
new_dependency_files.append(file_path)
415+
else:
416+
#Not an RRTMGP file, so include it:
417+
new_dependency_files.append(file_path)
418+
419+
# Return newly-modified dependency files:
420+
return new_dependency_files
421+
360422
###############################################################################
361423
def generate_registry(data_search, build_cache, atm_root, bldroot,
362424
source_mods_dir, dycore, gen_fort_indent):
@@ -420,7 +482,8 @@ def generate_registry(data_search, build_cache, atm_root, bldroot,
420482
###############################################################################
421483
def generate_physics_suites(build_cache, preproc_defs, host_name,
422484
phys_suites_str, atm_root, bldroot,
423-
reg_dir, reg_files, source_mods_dir, force):
485+
reg_dir, reg_files, source_mods_dir,
486+
gpu_flag, force):
424487
###############################################################################
425488
"""
426489
Generate the source for the configured physics suites,
@@ -664,8 +727,6 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
664727
build_cache.update_ccpp(sdfs, scheme_files, host_files, xml_files,
665728
scheme_nl_meta_files, nl_groups, create_nl_file,
666729
preproc_cache_str, kind_types)
667-
##XXgoldyXX: v Temporary fix: Copy CCPP Framework source code into
668-
##XXgoldyXX: v generated code directory
669730
request = DatatableReport("utility_files")
670731
ufiles_str = datatable_report(cap_output_file, request, ";")
671732
utility_files = ufiles_str.split(';')
@@ -674,8 +735,15 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
674735
dep_str = datatable_report(cap_output_file, request, ";")
675736
if len(dep_str) > 0:
676737
dependency_files = dep_str.split(';')
738+
# If using RRTMGP in the physics suite, then modify
739+
# the provided dependency files list to use the correct
740+
# CPU or GPU RRTMGP dependencies:
741+
if any("rrtmgp_" in scheme_name for scheme_name in scheme_names):
742+
dependency_files = _set_rrtmgp_dependencies(dependency_files,
743+
gpu_flag)
744+
745+
# Copy dependencies files into CCPP build directory
677746
_update_genccpp_dir(dependency_files, genccpp_dir)
678-
##XXgoldyXX: ^ Temporary fix:
679747
# End if
680748

681749
return [physics_blddir, genccpp_dir], do_gen_ccpp, cap_output_file, \
@@ -695,7 +763,7 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
695763
and/or script).
696764
"""
697765

698-
#Add new directory to build path:
766+
# Add new directory to build path:
699767
init_dir = os.path.join(bldroot, "phys_init")
700768
# Use this for cache check
701769
gen_init_file = os.path.join(_REG_GEN_DIR, "write_init_files.py")
@@ -706,11 +774,11 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
706774
if force_ccpp or force_init:
707775
do_gen_init = True
708776
else:
709-
#If not, then check cache to see if actual
710-
#"write_init_files.py" was modified:
777+
# If not, then check cache to see if actual
778+
# "write_init_files.py" was modified:
711779
do_gen_init = build_cache.init_write_mismatch(gen_init_file)
712780
else:
713-
#If no directory exists, then one will need
781+
# If no directory exists, then one will need
714782
# to create new routines:
715783
os.mkdir(init_dir)
716784
do_gen_init = True

‎cime_config/cam_config.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,9 @@ def __init__(self, case, case_log):
186186
# Save case variables needed for code auto-generation:
187187
self.__atm_root = case.get_value("COMP_ROOT_DIR_ATM")
188188
self.__caseroot = case.get_value("CASEROOT")
189-
self.__bldroot = os.path.join(exeroot, "atm", "obj")
189+
self.__bldroot = os.path.join(exeroot, "atm", "obj")
190190
self.__atm_name = case.get_value("COMP_ATM")
191+
self.__gpu_flag = case.get_value("OPENACC_GPU_OFFLOAD") #Returns a Boolean
191192

192193
# Save CPP definitions as a list:
193194
self.__cppdefs = [x for x in case.get_value("CAM_CPPDEFS").split() if x]
@@ -879,7 +880,7 @@ def generate_cam_src(self, gen_fort_indent):
879880
self.__atm_name, phys_suites,
880881
self.__atm_root, self.__bldroot,
881882
reg_dir, reg_files, source_mods_dir,
882-
force_ccpp)
883+
self.__gpu_flag, force_ccpp)
883884
phys_dirs, force_init, _, nml_fils, capgen_db, scheme_names = retvals
884885

885886
# Add namelist definition files to dictionary:

‎cime_config/testdefs/testlist_cam.xml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,15 @@
115115
<option name="comment">Snapshot test for RRTMGP</option>
116116
</options>
117117
</test>
118+
<test compset="FPHYStest" grid="ne3pg3_ne3pg3_mg37" name="SMS_Ln2" testmods="cam/outfrq_rrtmgp_derecho_gpu">
119+
<machines>
120+
<machine name="derecho" compiler="nvhpc" category="aux_sima"/>
121+
</machines>
122+
<options>
123+
<option name="wallclock">00:10:00</option>
124+
<option name="comment">GPU snapshot test for RRTMGP</option>
125+
</options>
126+
</test>
118127

119128
<!-- Derecho dycore tests -->
120129
<test compset="FKESSLER" grid="mpasa480_mpasa480" name="SMS_Ln9" testmods="cam/outfrq_kessler_mpas_derecho">

‎cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/shell_commands‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
./xmlchange NTASKS=128
2+
./xmlchange NTHRDS=1
3+
./xmlchange ROOTPE='0'
4+
./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
5+
./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
6+
./xmlchange TIMER_DETAIL='6'
7+
./xmlchange TIMER_LEVEL='999'
8+
./xmlchange GPU_TYPE=a100
9+
./xmlchange OPENACC_GPU_OFFLOAD=TRUE
10+
./xmlchange OVERSUBSCRIBE_GPU=TRUE
11+
./xmlchange NGPUS_PER_NODE=4
12+
./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites rrtmgp"
13+
./xmlchange RUN_STARTDATE=1979-01-01

‎cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cam‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
! these are CPU FHISTC_LTso snapshots
2+
ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_before_c20251013.nc'
3+
ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_after_c20251013.nc'
4+
5+
! tolerances for testing (currently have high tolerance due to CPU snapshots)
6+
ncdata_check_err = .true.
7+
min_difference = 1e-08
8+
9+
! vertical levels in snapshot
10+
pver = 58
11+
12+
! Do radiation on every timestep we're testing
13+
irad_always=3
14+
15+
! diagnostic output
16+
hist_output_frequency;h1: 1*nsteps
17+
hist_precision;h1: REAL64
18+
hist_add_inst_fields;h1: HR
19+
! Cloud output
20+
hist_add_inst_fields;h1: TOT_CLD_VISTAU,TOT_ICLD_VISTAU,ICE_ICLD_VISTAU,LIQ_ICLD_VISTAU
21+
! Longwave diagnostic output
22+
hist_add_inst_fields;h1: QRL,QRLC,FLNT,FLNTC,FLUT,FLUTC,LWCF,FLN200,FLN200C,FLNR,FLNS,FLNSC,FLDS,FLDSC,FUL,FDL,FULC,FDLC
23+
! Shortwave diagnostic fields
24+
hist_add_inst_fields;h1: SOLIN,QRS,QRSC,FSNT,FSNTC,FSNTOA,FSNTOAC,SWCF,FSUTOA,FSN200,FSN200C,FSNR,SOLL,SOLS,SOLLD,SOLSD
25+
hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FUS,FDS,FUSC,FDSC

‎cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cpl‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
! Set fixed orbital parameters
2+
orb_mode='fixed_parameters'
3+
orb_eccen = 0.
4+
orb_obliq = 0.
5+
orb_mvelp = 0.

‎src/control/cam_comp.F90‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,7 @@ subroutine cam_register_constituents(cam_runtime_opts)
657657
default_value=0._kind_phys, &
658658
vertical_dim="vertical_layer_dimension", &
659659
advected=.true., &
660+
diag_name="Q", &
660661
errcode=errflg, errmsg=errmsg)
661662

662663
if (errflg /= 0) then

0 commit comments

Comments
 (0)