Merge branch 'update/post_beta02/cam6_4_019' into 'ew-develop' (PR #31)

Bring in new tag from upstream for GPU-enabled CLUBB

.gitmodules

@@ -96,7 +96,7 @@
 	url = https://github.com/larson-group/clubb_release
         fxrequired = AlwaysRequired
         fxsparse = ../.clubb_sparse_checkout
-        fxtag = clubb_4ncar_20231115_5406350
+        fxtag = clubb_4ncar_20240605_73d60f6_gpufixes_posinf
         fxDONOTUSEurl = https://github.com/larson-group/clubb_release
 
 [submodule "cism"]
@@ -151,7 +151,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 [submodule "cdeps"]
 path = components/cdeps
 url = https://github.com/ESCOMP/CDEPS.git
-fxtag = cdeps1.0.43
+fxtag = cdeps1.0.45
 fxrequired = ToplevelRequired
 fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git
 
@@ -174,7 +174,7 @@ path = components/cice
 url = https://github.com/ESCOMP/CESM_CICE
 fxtag = cesm_cice6_5_0_10
 fxrequired = ToplevelRequired
-fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
+fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE
 
 [submodule "clm"]
 path = components/clm

.lib/git-fleximod/git_fleximod/cli.py

@@ -2,7 +2,7 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.8.2"
+__version__ = "0.8.4"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -195,18 +195,19 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
 
         result,n,l,t = submod.status()
-        testfails += t
-        localmods += l
-        needsupdate += n
         if toplevel or not submod.toplevel():
             print(wrapper.fill(result))
-        subdir = os.path.join(root_dir, submod.path)
-        if os.path.exists(os.path.join(subdir, ".gitmodules")):
-            submod = GitModules(logger, confpath=subdir)
-            t,l,n = submodules_status(submod, subdir, depth=depth+1)
             testfails += t
             localmods += l
             needsupdate += n
+        subdir = os.path.join(root_dir, submod.path)
+        if os.path.exists(os.path.join(subdir, ".gitmodules")):
+            gsubmod = GitModules(logger, confpath=subdir)
+            t,l,n = submodules_status(gsubmod, subdir, depth=depth+1)
+            if toplevel or not submod.toplevel():
+                testfails += t
+                localmods += l
+                needsupdate += n
 
     return testfails, localmods, needsupdate
 

.lib/git-fleximod/git_fleximod/submodule.py

@@ -92,7 +92,7 @@ def status(self):
                 needsupdate = True
             else:
                 result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}"
-                testfails = True
+                testfails = False
         else:
             with utils.pushd(smpath):
                 git = GitInterface(smpath, self.logger)
@@ -103,15 +103,23 @@ def status(self):
                     needsupdate = True
                     return result, needsupdate, localmods, testfails                    
                 rurl = git.git_operation("ls-remote","--get-url").rstrip()
-                line = git.git_operation("log", "--pretty=format:\"%h %d").partition('\n')[0]
+                line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0]
                 parts = line.split()
                 ahash = parts[0][1:]
+                atag = None
                 if len(parts) > 3:
-                    atag = parts[3][:-1]
-                else:
-                    atag = None
+                    idx = 0
+                    while idx < len(parts)-1:
+                        idx = idx+1
+                        if parts[idx] == 'tag:':
+                            atag = parts[idx+1]
+                            while atag.endswith(')') or atag.endswith(',') or atag.endswith("\""):
+                                atag = atag[:-1]
+                            if atag == self.fxtag:
+                                break
+
 
-                #print(f"line is {line} ahash is {ahash} atag is {atag}")
+                #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
                 #                atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 # ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
 
@@ -122,9 +130,11 @@ def status(self):
                 if self.fxtag and atag == self.fxtag:
                     result = f"  {self.name:>20} at tag {self.fxtag}"
                     recurse = True
-                elif self.fxtag and ahash[: len(self.fxtag)] == self.fxtag:
+                    testfails = False
+                elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)):
                     result = f"  {self.name:>20} at hash {ahash}"
                     recurse = True
+                    testfails = False
                 elif atag == ahash:
                     result = f"  {self.name:>20} at hash {ahash}"
                     recurse = True
@@ -133,14 +143,17 @@ def status(self):
                     testfails = True
                     needsupdate = True
                 else:
-                    result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}"
-                    testfails = True
+                    if atag:
+                        result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}"
+                    else:
+                        result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}"
+                    testfails = False
 
                 status = git.git_operation("status", "--ignore-submodules", "-uno")
                 if "nothing to commit" not in status:
                     localmods = True
                     result = "M" + textwrap.indent(status, "                      ")
-
+#        print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}")
         return result, needsupdate, localmods, testfails
 
 

.lib/git-fleximod/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.8.2"
+version = "0.8.4"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]

.lib/git-fleximod/tbump.toml

@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.8.2"
+current = "0.8.4"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

bld/configure

@@ -641,6 +641,10 @@ if ($dyn_pkg eq 'fv3' and $spmd eq 'OFF') {
     die "configure: FATAL: the fv3 dycore requires at least 6 tasks SPMD must not be switched off.$eol";
 }
 
+if ($dyn_pkg eq 'se' and $smp eq 'ON') {
+    die "CAM configure: ERROR: The SE dycore does not currently work with threading on. $eol";
+}
+
 if ($print>=2) { print "Dynamics package: $dyn_pkg$eol"; }
 
 $cfg_ref->set('analytic_ic', (defined $opts{'analytic_ic'}) ? $opts{'analytic_ic'} : 0);

bld/namelist_files/namelist_defaults_cam.xml

@@ -601,7 +601,7 @@
 <mam4_mode4_file rad="rrtmgp">atm/cam/physprops/mam4_mode4_rrtmg_c130628.nc</mam4_mode4_file>
 
 <mam4_mode1_file rad="rrtmgp" ver="strat">atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_sig1.6_dgnh.48_c140304.nc</mam4_mode1_file>
-<mam4_mode3_file rad="rrtmgp" ver="strat">atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_sig1.2_dgnl.40_c150219.nc</mam4_mode3_file>
+<mam4_mode3_file rad="rrtmgp" ver="strat" phys="cam6">atm/cam/physprops/mam4_mode3_rrtmg_aeronetdust_sig1.2_dgnl.40_c150219.nc</mam4_mode3_file>
 
 <mam5_mode1_file rad="rrtmgp" ver="strat">atm/cam/physprops/mam4_mode1_rrtmg_aeronetdust_sig1.6_dgnh.48_c140304.nc</mam5_mode1_file>
 <mam5_mode2_file rad="rrtmgp" ver="strat">atm/cam/physprops/mam4_mode2_rrtmg_aitkendust_c141106.nc</mam5_mode2_file>
@@ -2280,6 +2280,7 @@
 <do_hb_above_clubb                                >.false.  </do_hb_above_clubb>
 <do_hb_above_clubb phys="cam6"                    >.true.   </do_hb_above_clubb>
 <do_hb_above_clubb phys="cam7"                    >.true.   </do_hb_above_clubb>
+
 <!-- SILHS options -->
 <clubb_do_icesuper                      silhs="1" > .true.  </clubb_do_icesuper>
 <clubb_C2rt                             silhs="1" > 0.2     </clubb_C2rt>
@@ -2305,6 +2306,7 @@
 <clubb_skw_max_mag                      silhs="1" > 10.0    </clubb_skw_max_mag>
 <clubb_up2_sfc_coef                     silhs="1" > 4.0     </clubb_up2_sfc_coef>
 <clubb_C_wp2_splat                      silhs="1" > 0.0     </clubb_C_wp2_splat>
+<clubb_bv_efold                         silhs="1" > 5.0     </clubb_bv_efold>
 
 <clubb_l_brunt_vaisala_freq_moist       silhs="1" > .true.  </clubb_l_brunt_vaisala_freq_moist>
 <clubb_l_call_pdf_closure_twice         silhs="1" > .false. </clubb_l_call_pdf_closure_twice>
@@ -2323,6 +2325,8 @@
 <clubb_l_vert_avg_closure               silhs="1" > .false. </clubb_l_vert_avg_closure>
 <clubb_l_diag_Lscale_from_tau           silhs="1" > .false. </clubb_l_diag_Lscale_from_tau>
 <clubb_l_damp_wp2_using_em              silhs="1" > .false. </clubb_l_damp_wp2_using_em>
+<clubb_wpxp_Ri_exp                      silhs="1" > 0.5     </clubb_wpxp_Ri_exp>
+<clubb_z_displace                       silhs="1" > 25.00   </clubb_z_displace>
 
 
 <!-- CLUBB+MF options -->

cime_config/testdefs/testlist_cam.xml

@@ -1480,6 +1480,24 @@
 <!--                    (unsupported)                         -->
 <!--              @@@@@@@@@@@@@@@@@@@@@@@@@@@                 -->
 
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_default">
+    <machines>
+      <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
+      <machine name="derecho" compiler="nvhpc" category="aux_cam"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+    </options>
+  </test>   
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_pcols760">
+    <machines>
+      <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
+      <machine name="derecho" compiler="nvhpc" category="prealpha"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:30:00</option>
+    </options>
+  </test>
   <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERP_D_Ln9" testmods="cam/outfrq9s" supported="false">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_pumas"/>
@@ -1542,51 +1560,6 @@
       <machine name="derecho" compiler="intel" category="camchem"/>
     </machines>
   </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg2_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_nondefault">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9" testmods="cam/outfrq9s_mg3_pcols1536">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
-  <test compset="F2000dev" grid="f09_f09_mg17" name="ERP_Ln9_G4" testmods="cam/outfrq9s_mg3_default">
-    <machines>
-      <machine name="casper" compiler="pgi-gpu"   category="casper_gpu"/>
-      <machine name="casper" compiler="nvhpc-gpu" category="casper_gpu"/>
-    </machines>
-    <options>
-      <option name="wallclock">00:59:00</option>
-    </options>
-  </test>
   <test compset="QPC4" grid="f19_f19_mg17" name="ERP_Ln9" testmods="cam/outfrq9s">
     <machines>
       <machine name="derecho" compiler="intel" category="test_release"/>

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_default/shell_commands

@@ -1,4 +1,4 @@
-./xmlchange NTASKS=36
+./xmlchange NTASKS=128
 ./xmlchange NTHRDS=1
 ./xmlchange ROOTPE='0'
 ./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`

cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols1536/shell_commands → cime_config/testdefs/testmods_dirs/cam/outfrq9s_mg3_pcols760/shell_commands

@@ -1,8 +1,8 @@
-./xmlchange NTASKS=36
+./xmlchange NTASKS=64
 ./xmlchange NTHRDS=1
 ./xmlchange ROOTPE='0'
 ./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
 ./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
-./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 1536' --append
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -pcols 760 ' --append
 ./xmlchange TIMER_DETAIL='6'
 ./xmlchange TIMER_LEVEL='999'

cime_config/usermods_dirs/eworg_opt_clubb/user_nl_cam

@@ -0,0 +1,3 @@
+clubb_l_diag_lscale_from_tau = .true.
+clubb_penta_solve_method = 2
+clubb_tridiag_solve_method = 2