From 03e12bdfc0053c9d5d6cc89e1889a243316eefb3 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Thu, 9 Jan 2025 10:10:24 -0700 Subject: [PATCH] Fix CAM7 regression tests (#348) Tag name (required for release branches): sima0_00_003 Originator(s): peverwhee Description (include the issue title, and the keyword ['closes', 'fixes', 'resolves'] followed by the issue number): Updates framework tag to include bugfix for groups - closes #335 Updates atmospheric_physics external to use apply_constituent_tendencies Updates git-fleixmod to 0.9.4 Describe any changes made to build system: n/a Describe any changes made to the namelist: n/a List any changes to the defaults for the input datasets (e.g. boundary datasets): n/a List all files eliminated and why: n/a List all files added and what they do: n/a List all existing files that have been modified, and describe the changes: (Helpful git command: `git diff --name-status development...`) M .gitmodules - update ccpp-framework and atmospheric_physics tags M .lib/git-fleixmod/* - update git-fleixmod to 0.9.4 M test/existing-test-failures.txt - only MPAS tests still fail If there are new failures (compared to the `test/existing-test-failures.txt` file), have them OK'd by the gatekeeper, note them here, and add them to the file. If there are baseline differences, include the test and the reason for the diff. What is the nature of the change? Roundoff? derecho/intel/aux_sima: All PASS except MPAS; new baselines for CAM7 (test never passed before) derecho/gnu/aux_sima: All PASS except MPAS; new baselines for CAM7 (test never passed before) If this changes climate describe any run(s) done to evaluate the new climate in enough detail that it(they) could be reproduced: CAM-SIMA date used for the baseline comparison tests if different than latest: --- .gitmodules | 4 +- .lib/git-fleximod/git_fleximod/cli.py | 38 ++++--- .../git-fleximod/git_fleximod/git_fleximod.py | 11 +- .../git-fleximod/git_fleximod/gitinterface.py | 7 +- .lib/git-fleximod/git_fleximod/submodule.py | 103 ++++++++++-------- .lib/git-fleximod/git_fleximod/utils.py | 10 +- .lib/git-fleximod/poetry.lock | 6 +- .lib/git-fleximod/pyproject.toml | 2 +- .lib/git-fleximod/tbump.toml | 2 +- .lib/git-fleximod/tests/conftest.py | 4 +- ccpp_framework | 2 +- src/physics/ncar_ccpp | 2 +- test/existing-test-failures.txt | 5 - 13 files changed, 105 insertions(+), 91 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0fdc3af0..ae8c3a76 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ccpp-framework"] path = ccpp_framework url = https://github.com/NCAR/ccpp-framework - fxtag = 2024-11-07-dev + fxtag = 2025-01-06-dev fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/ESCOMP/atmospheric_physics - fxtag = 491e56247815ef23bfd8dba65d1e3c3b78ba164a + fxtag = 74e905b7a0ee5b2d2bfc3e3dd942eb9963398373 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "ccs_config"] diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index b6f728f8..131466b9 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -2,27 +2,31 @@ import argparse from git_fleximod import utils -__version__ = "0.8.4" +__version__ = "0.9.4" def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree which contains a file called filename """ - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - + try: + root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"], + output_to_caller=True ).rstrip() + except: + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None + return Path(root) def get_parser(): description = """ diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index 50e0ef83..2c2601fa 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master # set the repository remote logger.info("Setting remote origin in {}/{}".format(root_dir, path)) - status = sprepo_git.git_operation("remote", "-v") - if url not in status: + _, remotelist = sprepo_git.git_operation("remote", "-v") + if url not in remotelist: sprepo_git.git_operation("remote", "add", "origin", url) topgit = os.path.join(gitroot, ".git") @@ -181,6 +181,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): url = gitmodules.get(name, "url") assert path and url, f"Malformed .gitmodules file {path} {url}" tag = gitmodules.get(name, "fxtag") + if not tag: + tag = gitmodules.get(name, "hash") fxurl = gitmodules.get(name, "fxDONOTUSEurl") fxsparse = gitmodules.get(name, "fxsparse") fxrequired = gitmodules.get(name, "fxrequired") @@ -213,7 +215,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): def git_toplevelroot(root_dir, logger): rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") return superroot def submodules_update(gitmodules, root_dir, requiredlist, force): @@ -250,7 +252,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): newrequiredlist = ["AlwaysRequired"] if optional: newrequiredlist.append("AlwaysOptional") - submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) def local_mods_output(): @@ -342,7 +343,7 @@ def main(): excludelist=excludelist, ) if not gitmodules.sections(): - sys.exit("No submodule components found") + sys.exit(f"No submodule components found, root_dir={root_dir}") retval = 0 if action == "update": submodules_update(gitmodules, root_dir, fxrequired, force) diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index 58312014..fb20883c 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs): command = self._git_command(operation, *newargs) if isinstance(command, list): try: - return utils.execute_subprocess(command, output_to_caller=True) + status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) + return status, output.rstrip() except Exception as e: sys.exit(e) else: - return command + return 0, command def config_get_value(self, section, name): if self._use_module: @@ -81,6 +82,8 @@ def config_get_value(self, section, name): def config_set_value(self, section, name, value): if self._use_module: with self.repo.config_writer() as writer: + if "." in section: + section = section.replace("."," \"")+'"' writer.set_value(section, name, value) writer.release() # Ensure changes are saved else: diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index 70a3018a..75d9dd4e 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -60,8 +60,9 @@ def status(self): if not os.path.exists(os.path.join(smpath, ".git")): rootgit = GitInterface(self.root_dir, self.logger) # submodule commands use path, not name - tags = rootgit.git_operation("ls-remote", "--tags", self.url) - result = rootgit.git_operation("submodule","status",smpath).split() + status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) + status, result = rootgit.git_operation("submodule","status",smpath) + result = result.split() if result: ahash = result[0][1:] @@ -80,9 +81,9 @@ def status(self): result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" needsupdate = True elif self.fxtag: - ahash = rootgit.git_operation( + status, ahash = rootgit.git_operation( "submodule", "status", "{}".format(self.path) - ).rstrip() + ) ahash = ahash[1 : len(self.fxtag) + 1] if self.fxtag == ahash: result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" @@ -96,14 +97,15 @@ def status(self): else: with utils.pushd(smpath): git = GitInterface(smpath, self.logger) - remote = git.git_operation("remote").rstrip() + status, remote = git.git_operation("remote") if remote == '': result = f"e {self.name:>20} has no associated remote" testfails = True needsupdate = True return result, needsupdate, localmods, testfails - rurl = git.git_operation("ls-remote","--get-url").rstrip() - line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0] + status, rurl = git.git_operation("ls-remote","--get-url") + status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") + line = lines.partition('\n')[0] parts = line.split() ahash = parts[0][1:] atag = None @@ -120,7 +122,7 @@ def status(self): #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") - # atag = git.git_operation("describe", "--tags", "--always").rstrip() + # atag = git.git_operation("describe", "--tags", "--always") # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] recurse = False @@ -149,10 +151,10 @@ def status(self): result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" testfails = False - status = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in status: + status, output = git.git_operation("status", "--ignore-submodules", "-uno") + if "nothing to commit" not in output: localmods = True - result = "M" + textwrap.indent(status, " ") + result = "M" + textwrap.indent(output, " ") # print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") return result, needsupdate, localmods, testfails @@ -171,10 +173,11 @@ def _add_remote(self, git): Returns: str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. """ - remotes = git.git_operation("remote", "-v").splitlines() + status, remotes = git.git_operation("remote", "-v") + remotes = remotes.splitlines() upstream = None if remotes: - upstream = git.git_operation("ls-remote", "--get-url").rstrip() + status, upstream = git.git_operation("ls-remote", "--get-url") newremote = "newremote.00" tmpurl = self.url.replace("git@github.com:", "https://github.com/") line = next((s for s in remotes if self.url in s or tmpurl in s), None) @@ -183,7 +186,7 @@ def _add_remote(self, git): return newremote else: i = 0 - while "newremote" in remotes: + while newremote in remotes: i = i + 1 newremote = f"newremote.{i:02d}" else: @@ -214,12 +217,19 @@ def sparse_checkout(self): """ self.logger.info("Called sparse_checkout for {}".format(self.name)) rgit = GitInterface(self.root_dir, self.logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") if superroot: gitroot = superroot.strip() else: - gitroot = self.root_dir.strip() - assert os.path.isdir(os.path.join(gitroot, ".git")) + gitroot = self.root_dir + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + while os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline().rstrip() + if line.startswith("gitdir: "): + rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:])) + assert os.path.isdir(rootdotgit) # first create the module directory if not os.path.isdir(os.path.join(self.root_dir, self.path)): os.makedirs(os.path.join(self.root_dir, self.path)) @@ -244,8 +254,8 @@ def sparse_checkout(self): # set the repository remote self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) - status = sprepo_git.git_operation("remote", "-v") - if self.url not in status: + status, remotes = sprepo_git.git_operation("remote", "-v") + if self.url not in remotes: sprepo_git.git_operation("remote", "add", "origin", self.url) topgit = os.path.join(gitroot, ".git") @@ -256,46 +266,46 @@ def sparse_checkout(self): os.path.join(self.root_dir, f.read().split()[1]), start=os.path.join(self.root_dir, self.path), ) - topgit = os.path.join(gitpath, "modules") + rootdotgit = os.path.join(gitpath, "modules", self.name) else: - topgit = os.path.relpath( - os.path.join(self.root_dir, ".git", "modules"), + rootdotgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules", self.name), start=os.path.join(self.root_dir, self.path), ) - with utils.pushd(sprep_repo): - if not os.path.isdir(topgit): - os.makedirs(topgit) - topgit += os.sep + self.name - if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(topgit,".git")): - shutil.rmtree(os.path.join(topgit,".git")) - shutil.move(".git", topgit) + if os.path.isdir(os.path.join(rootdotgit,".git")): + shutil.rmtree(os.path.join(rootdotgit,".git")) + shutil.move(".git", rootdotgit) with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(topgit)) - # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) - gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + f.write("gitdir: " + os.path.relpath(rootdotgit)) + infodir = os.path.join(rootdotgit, "info") + if not os.path.isdir(infodir): + os.makedirs(infodir) + gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) if os.path.isfile(gitsparse): self.logger.warning( - "submodule {} is already initialized {}".format(self.name, topgit) + "submodule {} is already initialized {}".format(self.name, rootdotgit) ) return with utils.pushd(sprep_repo): if os.path.isfile(self.fxsparse): + shutil.copy(self.fxsparse, gitsparse) # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") - sprepo_git.git_operation("checkout", self.fxtag) - - print(f"Successfully checked out {self.name:>20} at {self.fxtag}") - rgit.config_set_value(f'submodule "{self.name}"', "active", "true") - rgit.config_set_value(f'submodule "{self.name}"', "url", self.url) - rgit.config_set_value(f'submodule "{self.name}"', "path", self.path) + status,_ = sprepo_git.git_operation("checkout", self.fxtag) + if status: + print(f"Error checking out {self.name:>20} at {self.fxtag}") + else: + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + rgit.config_set_value('submodule.' + self.name, "active", "true") + rgit.config_set_value('submodule.' + self.name, "url", self.url) + rgit.config_set_value('submodule.' + self.name, "path", self.path) def update(self): """ @@ -342,7 +352,7 @@ def update(self): git.git_operation("clone", self.url, self.path) smgit = GitInterface(repodir, self.logger) if not tag: - tag = smgit.git_operation("describe", "--tags", "--always").rstrip() + status, tag = smgit.git_operation("describe", "--tags", "--always") smgit.git_operation("checkout", tag) # Now need to move the .git dir to the submodule location rootdotgit = os.path.join(self.root_dir, ".git") @@ -350,7 +360,7 @@ def update(self): with open(rootdotgit) as f: line = f.readline() if line.startswith("gitdir: "): - rootdotgit = line[8:].rstrip() + rootdotgit = line[8:] newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) if os.path.exists(newpath): @@ -393,15 +403,16 @@ def update(self): git = GitInterface(submoddir, self.logger) # first make sure the url is correct newremote = self._add_remote(git) - tags = git.git_operation("tag", "-l") + status, tags = git.git_operation("tag", "-l") fxtag = self.fxtag if fxtag and fxtag not in tags: git.git_operation("fetch", newremote, "--tags") - atag = git.git_operation("describe", "--tags", "--always").rstrip() + status, atag = git.git_operation("describe", "--tags", "--always") if fxtag and fxtag != atag: try: - git.git_operation("checkout", fxtag) - print(f"{self.name:>20} updated to {fxtag}") + status, _ = git.git_operation("checkout", fxtag) + if not status: + print(f"{self.name:>20} updated to {fxtag}") except Exception as error: print(error) diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py index 1a2d5ccf..c4f43d52 100644 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ b/.lib/git-fleximod/git_fleximod/utils.py @@ -307,12 +307,12 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False) # simple status check. If returning, it is the callers # responsibility determine if an error occurred and handle it # appropriately. + msg_context = ( + "Process did not run successfully; " + "returned status {0}".format(error.returncode) + ) + msg = failed_command_msg(msg_context, commands, output=error.output) if not return_to_caller: - msg_context = ( - "Process did not run successfully; " - "returned status {0}".format(error.returncode) - ) - msg = failed_command_msg(msg_context, commands, output=error.output) logging.error(error) logging.error(msg) log_process_output(error.output) diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock index 3a74effc..ac82fb0d 100644 --- a/.lib/git-fleximod/poetry.lock +++ b/.lib/git-fleximod/poetry.lock @@ -30,13 +30,13 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index 850e57d5..65924ff9 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.8.4" +version = "0.9.4" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index bd82c557..be0b799d 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.8.4" +current = "0.9.4" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 81edbe71..1dd1b86f 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -32,7 +32,7 @@ def logger(): "submodule_name": "test_optional", "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag None, expected tag is MPIserial_2.4.0 (optional)", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.1, expected tag is MPIserial_2.4.0 (optional)", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] @@ -46,7 +46,7 @@ def logger(): "submodule_name": "test_alwaysoptional", "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag None, expected tag is e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.1, expected tag is e5cf35c", "status4" : "test_alwaysoptional at hash e5cf35c", "gitmodules_content": """ [submodule "test_alwaysoptional"] diff --git a/ccpp_framework b/ccpp_framework index 27d0486f..fca3a9e3 160000 --- a/ccpp_framework +++ b/ccpp_framework @@ -1 +1 @@ -Subproject commit 27d0486f61e86d54921edc82139884e5f9c67700 +Subproject commit fca3a9e3df101bd07a228631e5509ec4a93a9906 diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 45c37530..491e5624 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 45c37530ead4f5b93621e3d979e972ea91217923 +Subproject commit 491e56247815ef23bfd8dba65d1e3c3b78ba164a diff --git a/test/existing-test-failures.txt b/test/existing-test-failures.txt index 05e0a145..975b8681 100644 --- a/test/existing-test-failures.txt +++ b/test/existing-test-failures.txt @@ -1,8 +1,3 @@ SMS_Ln2.mpasa480_mpasa480.FKESSLER.derecho_intel.cam-outfrq_kessler_mpas_derecho (Overall: FAIL) SMS_Ln2.mpasa480_mpasa480.FKESSLER.derecho_gnu.cam-outfrq_kessler_mpas_derecho (Overall: FAIL) - will fail until MPAS is fully integrated - -SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_intel.cam-outfrq_se_cslam_analy_ic (Overall: PEND) details: -SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_gnu.cam-outfrq_se_cslam_analy_ic (Overall: PEND) details: - - build failure due to dadadj_apply_qv_tendency removal, needs to be updated to use constituent tendency updater atmospheric_physics#179 - - also expected runtime failure CAM-SIMA#335