Merge branch 'bishtgautam/lnd/add-mpi-barrier' (PR #5741)

A namelist option (mpi_sync_nstep_freq) is added to allow ELM to call an MPI_Barrier. By default, the MPI_Barrier is not called. [BFB]
E3SM-Project · Jul 3, 2023 · f1fe0cd · f1fe0cd
2 parents 9e17b77 + a9ad253
commit f1fe0cd
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 2 deletions.
diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml
@@ -2020,5 +2020,15 @@ explain the SZA dependence of direct irradiance.
 Toggle to turn on the internal mixing of dust-snow. 
 </entry>
 
+<!-- ========================================================================================  -->
+<!-- Namelist option for MPI syncing                                                           -->
+<!-- ========================================================================================  -->
+
+<entry id="mpi_sync_nstep_freq" type="integer(6)" category="history"
+       group="elm_inparm" value="0">
+The number of steps after which a MPI_Barrier is called. By default, a MPI_Barrier will
+not be called.
+<default>Default: 0</default>
+</entry>
 
 </namelist_definition>
diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90
@@ -225,7 +225,8 @@ subroutine control_init( )
     namelist /elm_inparm/  &
          clump_pproc, wrtdia, &
          create_crop_landunit, nsegspc, co2_ppmv, override_nsrest, &
-         albice, more_vertlayers, subgridflag, irrigate, tw_irr, extra_gw_irr, firrig_data, all_active
+         albice, more_vertlayers, subgridflag, irrigate, tw_irr, extra_gw_irr, firrig_data, all_active, &
+         mpi_sync_nstep_freq
     ! Urban options
 
     namelist /elm_inparm/  &
@@ -318,7 +319,7 @@ subroutine control_init( )
 
     namelist /elm_inparm/ &
          snow_shape, snicar_atm_type, use_dust_snow_internal_mixing 
-    
+
     ! ----------------------------------------------------------------------
     ! Default values
     ! ----------------------------------------------------------------------
@@ -938,6 +939,8 @@ subroutine control_spmd()
     call mpi_bcast (snicar_atm_type, len(snicar_atm_type), MPI_CHARACTER, 0, mpicom, ier)
     call mpi_bcast (use_dust_snow_internal_mixing, 1, MPI_LOGICAL, 0, mpicom, ier)
 
+    call mpi_bcast (mpi_sync_nstep_freq, 1, MPI_INTEGER, 0, mpicom, ier)
+
   end subroutine control_spmd
 
   !------------------------------------------------------------------------
@@ -1195,6 +1198,7 @@ subroutine control_print ()
     ! land river two way coupling
     write(iulog,*) '    use_lnd_rof_two_way    = ', use_lnd_rof_two_way
     write(iulog,*) '    lnd_rof_coupling_nstep = ', lnd_rof_coupling_nstep
+    write(iulog,*) '    mpi_sync_nstep_freq    = ', mpi_sync_nstep_freq
 
   end subroutine control_print
 

diff --git a/components/elm/src/main/elm_driver.F90 b/components/elm/src/main/elm_driver.F90
@@ -15,6 +15,7 @@ module elm_driver
   use elm_varctl             , only : wrtdia, iulog, create_glacier_mec_landunit, use_fates, use_betr, use_extrasnowlayers
   use elm_varctl             , only : use_cn, use_lch4, use_voc, use_noio, use_c13, use_c14
   use elm_varctl             , only : use_erosion, use_fates_sp
+  use elm_varctl             , only : mpi_sync_nstep_freq
   use clm_time_manager       , only : get_step_size, get_curr_date, get_ref_date, get_nstep, is_beg_curr_day, get_curr_time_string
   use clm_time_manager       , only : get_curr_calday, get_days_per_year
   use elm_varpar             , only : nlevsno, nlevgrnd, crop_prog
@@ -251,6 +252,15 @@ subroutine elm_drv(doalb, nextsw_cday, declinp1, declin, rstwr, nlend, rdate)
     dayspyr_mod = get_days_per_year()
     jday_mod = get_curr_calday()
 
+    if (mpi_sync_nstep_freq > 0) then
+       if (mod(nstep_mod,mpi_sync_nstep_freq) == 0) then
+          call MPI_Barrier(mpicom, ier)
+          if (masterproc) then
+             write(iulog,*)'                       A MPI_Barrier is added in this timestep.'
+          end if
+       end if
+    end if
+
     if (do_budgets) then
        call WaterBudget_Reset()
 

diff --git a/components/elm/src/main/elm_varctl.F90 b/components/elm/src/main/elm_varctl.F90
@@ -522,6 +522,10 @@ module elm_varctl
    character(len=256), public :: snicar_atm_type = 'default'
    logical, public :: use_dust_snow_internal_mixing = .false.
 
+   !----------------------------------------------------------
+   ! MPI syncing
+   !----------------------------------------------------------
+   integer, public :: mpi_sync_nstep_freq = 0
 
 contains