Skip to content

Commit 8b239f1

Browse files
mgdudamgduda
committed
Enable the use of mpas_halo to exchange group halos in the atmosphere core
Following what was done for the setup and use of group halo exchange routines from the mpas_dmpar module in the atmosphere core, this commit adds code in the atm_build_halo_groups() routine to use the new mpas_halo module to set up halo exchange groups and to set the routine used to exchange halo groups; similarly, in the atm_destroy_halo_groups() routine, mpas_halo routines may now be used. To control whether the atmosphere core will use the mpas_dmpar routines or the mpas_halo routines for building and communicating halo groups, the config_halo_exch_method namelist option may now be set to either 'mpas_dmpar' or to 'mpas_halo'. The default is to still use the mpas_dmpar routines.
1 parent bd97fed commit 8b239f1

2 files changed

Lines changed: 177 additions & 1 deletion

File tree

‎src/core_atmosphere/Registry.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,7 @@
382382
<nml_option name="config_halo_exch_method" type="character" default_value="mpas_dmpar"
383383
units="-"
384384
description="Method to use for exchanging halos"
385-
possible_values="`mpas_dmpar'"/>
385+
possible_values="`mpas_dmpar', `mpas_halo'"/>
386386
</nml_record>
387387

388388
<!-- **************************************************************************************** -->

‎src/core_atmosphere/mpas_atm_core.F‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,6 +1436,9 @@ end subroutine mpas_atm_run_compatibility
14361436
!-----------------------------------------------------------------------
14371437
subroutine atm_build_halo_groups(domain, ierr)
14381438

1439+
use mpas_halo, only : mpas_halo_init, mpas_halo_exch_group_create, mpas_halo_exch_group_add_field, &
1440+
mpas_halo_exch_group_complete, mpas_halo_exch_group_full_halo_exch
1441+
14391442
type (domain_type), intent(inout) :: domain
14401443
integer, intent(inout) :: ierr
14411444

@@ -1560,6 +1563,139 @@ subroutine atm_build_halo_groups(domain, ierr)
15601563
!
15611564
exchange_halo_group => mpas_dmpar_exch_group_full_halo_exch
15621565

1566+
else if (trim(config_halo_exch_method) == 'mpas_halo') then
1567+
1568+
call mpas_log_write('')
1569+
call mpas_log_write('*** Using ''mpas_halo'' routines for exchanging halos')
1570+
call mpas_log_write('')
1571+
1572+
call mpas_halo_init(domain)
1573+
1574+
!
1575+
! Set up halo exchange groups used during atmosphere core initialization
1576+
!
1577+
call mpas_halo_exch_group_create(domain, 'initialization:u')
1578+
call mpas_halo_exch_group_add_field(domain, 'initialization:u', 'u', timeLevel=1, haloLayers=(/1,2,3/))
1579+
call mpas_halo_exch_group_complete(domain, 'initialization:u')
1580+
1581+
call mpas_halo_exch_group_create(domain, 'initialization:pv_edge,ru,rw')
1582+
call mpas_halo_exch_group_add_field(domain, 'initialization:pv_edge,ru,rw', 'pv_edge', timeLevel=1, haloLayers=(/1,2,3/))
1583+
call mpas_halo_exch_group_add_field(domain, 'initialization:pv_edge,ru,rw', 'ru', timeLevel=1, haloLayers=(/1,2,3/))
1584+
call mpas_halo_exch_group_add_field(domain, 'initialization:pv_edge,ru,rw', 'rw', timeLevel=1, haloLayers=(/1,2/))
1585+
call mpas_halo_exch_group_complete(domain, 'initialization:pv_edge,ru,rw')
1586+
1587+
!
1588+
! Set up halo exchange groups used by dynamics
1589+
!
1590+
call mpas_halo_exch_group_create(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p')
1591+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p', 'theta_m', &
1592+
timeLevel=1, haloLayers=(/1,2/))
1593+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p', 'scalars', &
1594+
timeLevel=1, haloLayers=(/1,2/))
1595+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p', 'pressure_p', &
1596+
timeLevel=1, haloLayers=(/1,2/))
1597+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p', 'rtheta_p', &
1598+
timeLevel=1, haloLayers=(/1,2/))
1599+
call mpas_halo_exch_group_complete(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p')
1600+
1601+
call mpas_halo_exch_group_create(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp')
1602+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp', 'rw_p', &
1603+
timeLevel=1, haloLayers=(/1/))
1604+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp', 'ru_p', &
1605+
timeLevel=1, haloLayers=(/2/))
1606+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp', 'rho_pp', &
1607+
timeLevel=1, haloLayers=(/1,2/))
1608+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp', 'rtheta_pp', &
1609+
timeLevel=1, haloLayers=(/2/))
1610+
call mpas_halo_exch_group_complete(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp')
1611+
1612+
call mpas_halo_exch_group_create(domain, 'dynamics:w,pv_edge,rho_edge')
1613+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge', 'w', timeLevel=2, haloLayers=(/1,2/))
1614+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge', 'pv_edge', timeLevel=1, haloLayers=(/1,2/))
1615+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge', 'rho_edge', timeLevel=1, haloLayers=(/1,2/))
1616+
call mpas_halo_exch_group_complete(domain, 'dynamics:w,pv_edge,rho_edge')
1617+
1618+
call mpas_halo_exch_group_create(domain, 'dynamics:w,pv_edge,rho_edge,scalars')
1619+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge,scalars', 'w', &
1620+
timeLevel=2, haloLayers=(/1,2/))
1621+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge,scalars', 'pv_edge', &
1622+
timeLevel=1, haloLayers=(/1,2/))
1623+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge,scalars', 'rho_edge', &
1624+
timeLevel=1, haloLayers=(/1,2/))
1625+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w,pv_edge,rho_edge,scalars', 'scalars', &
1626+
timeLevel=2, haloLayers=(/1,2/))
1627+
call mpas_halo_exch_group_complete(domain, 'dynamics:w,pv_edge,rho_edge,scalars')
1628+
1629+
call mpas_halo_exch_group_create(domain, 'dynamics:theta_m,pressure_p,rtheta_p')
1630+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,pressure_p,rtheta_p', 'theta_m', &
1631+
timeLevel=2, haloLayers=(/1,2/))
1632+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,pressure_p,rtheta_p', 'pressure_p', &
1633+
timeLevel=1, haloLayers=(/1,2/))
1634+
call mpas_halo_exch_group_add_field(domain, 'dynamics:theta_m,pressure_p,rtheta_p', 'rtheta_p', &
1635+
timeLevel=1, haloLayers=(/1,2/))
1636+
call mpas_halo_exch_group_complete(domain, 'dynamics:theta_m,pressure_p,rtheta_p')
1637+
1638+
1639+
call mpas_halo_exch_group_create(domain, 'dynamics:exner')
1640+
call mpas_halo_exch_group_add_field(domain, 'dynamics:exner', 'exner', timeLevel=1, haloLayers=(/1,2/))
1641+
call mpas_halo_exch_group_complete(domain, 'dynamics:exner')
1642+
1643+
call mpas_halo_exch_group_create(domain, 'dynamics:tend_u')
1644+
call mpas_halo_exch_group_add_field(domain, 'dynamics:tend_u', 'tend_u', timeLevel=1, haloLayers=(/1/))
1645+
call mpas_halo_exch_group_complete(domain, 'dynamics:tend_u')
1646+
1647+
call mpas_halo_exch_group_create(domain, 'dynamics:rho_pp')
1648+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rho_pp', 'rho_pp', timeLevel=1, haloLayers=(/1/))
1649+
call mpas_halo_exch_group_complete(domain, 'dynamics:rho_pp')
1650+
1651+
call mpas_halo_exch_group_create(domain, 'dynamics:rtheta_pp')
1652+
call mpas_halo_exch_group_add_field(domain, 'dynamics:rtheta_pp', 'rtheta_pp', timeLevel=1, haloLayers=(/1/))
1653+
call mpas_halo_exch_group_complete(domain, 'dynamics:rtheta_pp')
1654+
1655+
call mpas_halo_exch_group_create(domain, 'dynamics:u_123')
1656+
call mpas_halo_exch_group_add_field(domain, 'dynamics:u_123', 'u', timeLevel=2, haloLayers=(/1,2,3/))
1657+
call mpas_halo_exch_group_complete(domain, 'dynamics:u_123')
1658+
1659+
call mpas_halo_exch_group_create(domain, 'dynamics:u_3')
1660+
call mpas_halo_exch_group_add_field(domain, 'dynamics:u_3', 'u', timeLevel=2, haloLayers=(/3/))
1661+
call mpas_halo_exch_group_complete(domain, 'dynamics:u_3')
1662+
1663+
call mpas_halo_exch_group_create(domain, 'dynamics:scalars')
1664+
call mpas_halo_exch_group_add_field(domain, 'dynamics:scalars', 'scalars', timeLevel=2, haloLayers=(/1,2/))
1665+
call mpas_halo_exch_group_complete(domain, 'dynamics:scalars')
1666+
1667+
call mpas_halo_exch_group_create(domain, 'dynamics:scalars_old')
1668+
call mpas_halo_exch_group_add_field(domain, 'dynamics:scalars_old', 'scalars', timeLevel=1, haloLayers=(/1,2/))
1669+
call mpas_halo_exch_group_complete(domain, 'dynamics:scalars_old')
1670+
1671+
call mpas_halo_exch_group_create(domain, 'dynamics:w')
1672+
call mpas_halo_exch_group_add_field(domain, 'dynamics:w', 'w', timeLevel=2, haloLayers=(/1,2/))
1673+
call mpas_halo_exch_group_complete(domain, 'dynamics:w')
1674+
1675+
call mpas_halo_exch_group_create(domain, 'dynamics:scale')
1676+
call mpas_halo_exch_group_add_field(domain, 'dynamics:scale', 'scale', timeLevel=1, haloLayers=(/1,2/))
1677+
call mpas_halo_exch_group_complete(domain, 'dynamics:scale')
1678+
1679+
#ifdef DO_PHYSICS
1680+
!
1681+
! Set up halo exchange groups used by physics
1682+
!
1683+
call mpas_halo_exch_group_create(domain, 'physics:blten')
1684+
call mpas_halo_exch_group_add_field(domain, 'physics:blten', 'rublten', timeLevel=1, haloLayers=(/1,2/))
1685+
call mpas_halo_exch_group_add_field(domain, 'physics:blten', 'rvblten', timeLevel=1, haloLayers=(/1,2/))
1686+
call mpas_halo_exch_group_complete(domain, 'physics:blten')
1687+
1688+
call mpas_halo_exch_group_create(domain, 'physics:cuten')
1689+
call mpas_halo_exch_group_add_field(domain, 'physics:cuten', 'rucuten', timeLevel=1, haloLayers=(/1,2/))
1690+
call mpas_halo_exch_group_add_field(domain, 'physics:cuten', 'rvcuten', timeLevel=1, haloLayers=(/1,2/))
1691+
call mpas_halo_exch_group_complete(domain, 'physics:cuten')
1692+
#endif
1693+
1694+
!
1695+
! Set routine to exchange a halo group
1696+
!
1697+
exchange_halo_group => mpas_halo_exch_group_full_halo_exch
1698+
15631699
else
15641700

15651701
!
@@ -1594,6 +1730,8 @@ end subroutine atm_build_halo_groups
15941730
!-----------------------------------------------------------------------
15951731
subroutine atm_destroy_halo_groups(domain, ierr)
15961732

1733+
use mpas_halo, only : mpas_halo_exch_group_destroy, mpas_halo_finalize
1734+
15971735
type (domain_type), intent(inout) :: domain
15981736
integer, intent(inout) :: ierr
15991737

@@ -1633,6 +1771,44 @@ subroutine atm_destroy_halo_groups(domain, ierr)
16331771
call mpas_dmpar_exch_group_destroy(domain, 'physics:cuten')
16341772
#endif
16351773

1774+
else if (trim(config_halo_exch_method) == 'mpas_halo') then
1775+
1776+
!
1777+
! Destroy halo exchange groups used only during initialization
1778+
!
1779+
call mpas_halo_exch_group_destroy(domain, 'initialization:u')
1780+
call mpas_halo_exch_group_destroy(domain, 'initialization:pv_edge,ru,rw')
1781+
1782+
!
1783+
! Destroy halo exchange groups used by dynamics
1784+
!
1785+
call mpas_halo_exch_group_destroy(domain, 'dynamics:theta_m,scalars,pressure_p,rtheta_p')
1786+
call mpas_halo_exch_group_destroy(domain, 'dynamics:rw_p,ru_p,rho_pp,rtheta_pp')
1787+
call mpas_halo_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge')
1788+
call mpas_halo_exch_group_destroy(domain, 'dynamics:w,pv_edge,rho_edge,scalars')
1789+
call mpas_halo_exch_group_destroy(domain, 'dynamics:theta_m,pressure_p,rtheta_p')
1790+
1791+
call mpas_halo_exch_group_destroy(domain, 'dynamics:exner')
1792+
call mpas_halo_exch_group_destroy(domain, 'dynamics:tend_u')
1793+
call mpas_halo_exch_group_destroy(domain, 'dynamics:rho_pp')
1794+
call mpas_halo_exch_group_destroy(domain, 'dynamics:rtheta_pp')
1795+
call mpas_halo_exch_group_destroy(domain, 'dynamics:u_123')
1796+
call mpas_halo_exch_group_destroy(domain, 'dynamics:u_3')
1797+
call mpas_halo_exch_group_destroy(domain, 'dynamics:scalars')
1798+
call mpas_halo_exch_group_destroy(domain, 'dynamics:scalars_old')
1799+
call mpas_halo_exch_group_destroy(domain, 'dynamics:w')
1800+
call mpas_halo_exch_group_destroy(domain, 'dynamics:scale')
1801+
1802+
#ifdef DO_PHYSICS
1803+
!
1804+
! Destroy halo exchange groups used by physics
1805+
!
1806+
call mpas_halo_exch_group_destroy(domain, 'physics:blten')
1807+
call mpas_halo_exch_group_destroy(domain, 'physics:cuten')
1808+
#endif
1809+
1810+
call mpas_halo_finalize(domain)
1811+
16361812
else
16371813

16381814
!

0 commit comments

Comments
 (0)