diff --git a/.gitmodules b/.gitmodules index 0fdc3af0..ae8c3a76 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ccpp-framework"] path = ccpp_framework url = https://github.com/NCAR/ccpp-framework - fxtag = 2024-11-07-dev + fxtag = 2025-01-06-dev fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/ESCOMP/atmospheric_physics - fxtag = 491e56247815ef23bfd8dba65d1e3c3b78ba164a + fxtag = 74e905b7a0ee5b2d2bfc3e3dd942eb9963398373 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "ccs_config"] diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index b6f728f8..131466b9 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -2,27 +2,31 @@ import argparse from git_fleximod import utils -__version__ = "0.8.4" +__version__ = "0.9.4" def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree which contains a file called filename """ - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - + try: + root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"], + output_to_caller=True ).rstrip() + except: + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None + return Path(root) def get_parser(): description = """ diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index 50e0ef83..2c2601fa 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master # set the repository remote logger.info("Setting remote origin in {}/{}".format(root_dir, path)) - status = sprepo_git.git_operation("remote", "-v") - if url not in status: + _, remotelist = sprepo_git.git_operation("remote", "-v") + if url not in remotelist: sprepo_git.git_operation("remote", "add", "origin", url) topgit = os.path.join(gitroot, ".git") @@ -181,6 +181,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): url = gitmodules.get(name, "url") assert path and url, f"Malformed .gitmodules file {path} {url}" tag = gitmodules.get(name, "fxtag") + if not tag: + tag = gitmodules.get(name, "hash") fxurl = gitmodules.get(name, "fxDONOTUSEurl") fxsparse = gitmodules.get(name, "fxsparse") fxrequired = gitmodules.get(name, "fxrequired") @@ -213,7 +215,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): def git_toplevelroot(root_dir, logger): rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") return superroot def submodules_update(gitmodules, root_dir, requiredlist, force): @@ -250,7 +252,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): newrequiredlist = ["AlwaysRequired"] if optional: newrequiredlist.append("AlwaysOptional") - submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) def local_mods_output(): @@ -342,7 +343,7 @@ def main(): excludelist=excludelist, ) if not gitmodules.sections(): - sys.exit("No submodule components found") + sys.exit(f"No submodule components found, root_dir={root_dir}") retval = 0 if action == "update": submodules_update(gitmodules, root_dir, fxrequired, force) diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index 58312014..fb20883c 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs): command = self._git_command(operation, *newargs) if isinstance(command, list): try: - return utils.execute_subprocess(command, output_to_caller=True) + status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) + return status, output.rstrip() except Exception as e: sys.exit(e) else: - return command + return 0, command def config_get_value(self, section, name): if self._use_module: @@ -81,6 +82,8 @@ def config_get_value(self, section, name): def config_set_value(self, section, name, value): if self._use_module: with self.repo.config_writer() as writer: + if "." in section: + section = section.replace("."," \"")+'"' writer.set_value(section, name, value) writer.release() # Ensure changes are saved else: diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index 70a3018a..75d9dd4e 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -60,8 +60,9 @@ def status(self): if not os.path.exists(os.path.join(smpath, ".git")): rootgit = GitInterface(self.root_dir, self.logger) # submodule commands use path, not name - tags = rootgit.git_operation("ls-remote", "--tags", self.url) - result = rootgit.git_operation("submodule","status",smpath).split() + status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) + status, result = rootgit.git_operation("submodule","status",smpath) + result = result.split() if result: ahash = result[0][1:] @@ -80,9 +81,9 @@ def status(self): result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" needsupdate = True elif self.fxtag: - ahash = rootgit.git_operation( + status, ahash = rootgit.git_operation( "submodule", "status", "{}".format(self.path) - ).rstrip() + ) ahash = ahash[1 : len(self.fxtag) + 1] if self.fxtag == ahash: result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" @@ -96,14 +97,15 @@ def status(self): else: with utils.pushd(smpath): git = GitInterface(smpath, self.logger) - remote = git.git_operation("remote").rstrip() + status, remote = git.git_operation("remote") if remote == '': result = f"e {self.name:>20} has no associated remote" testfails = True needsupdate = True return result, needsupdate, localmods, testfails - rurl = git.git_operation("ls-remote","--get-url").rstrip() - line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0] + status, rurl = git.git_operation("ls-remote","--get-url") + status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") + line = lines.partition('\n')[0] parts = line.split() ahash = parts[0][1:] atag = None @@ -120,7 +122,7 @@ def status(self): #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") - # atag = git.git_operation("describe", "--tags", "--always").rstrip() + # atag = git.git_operation("describe", "--tags", "--always") # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] recurse = False @@ -149,10 +151,10 @@ def status(self): result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" testfails = False - status = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in status: + status, output = git.git_operation("status", "--ignore-submodules", "-uno") + if "nothing to commit" not in output: localmods = True - result = "M" + textwrap.indent(status, " ") + result = "M" + textwrap.indent(output, " ") # print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") return result, needsupdate, localmods, testfails @@ -171,10 +173,11 @@ def _add_remote(self, git): Returns: str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. """ - remotes = git.git_operation("remote", "-v").splitlines() + status, remotes = git.git_operation("remote", "-v") + remotes = remotes.splitlines() upstream = None if remotes: - upstream = git.git_operation("ls-remote", "--get-url").rstrip() + status, upstream = git.git_operation("ls-remote", "--get-url") newremote = "newremote.00" tmpurl = self.url.replace("git@github.com:", "https://github.com/") line = next((s for s in remotes if self.url in s or tmpurl in s), None) @@ -183,7 +186,7 @@ def _add_remote(self, git): return newremote else: i = 0 - while "newremote" in remotes: + while newremote in remotes: i = i + 1 newremote = f"newremote.{i:02d}" else: @@ -214,12 +217,19 @@ def sparse_checkout(self): """ self.logger.info("Called sparse_checkout for {}".format(self.name)) rgit = GitInterface(self.root_dir, self.logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") if superroot: gitroot = superroot.strip() else: - gitroot = self.root_dir.strip() - assert os.path.isdir(os.path.join(gitroot, ".git")) + gitroot = self.root_dir + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + while os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline().rstrip() + if line.startswith("gitdir: "): + rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:])) + assert os.path.isdir(rootdotgit) # first create the module directory if not os.path.isdir(os.path.join(self.root_dir, self.path)): os.makedirs(os.path.join(self.root_dir, self.path)) @@ -244,8 +254,8 @@ def sparse_checkout(self): # set the repository remote self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) - status = sprepo_git.git_operation("remote", "-v") - if self.url not in status: + status, remotes = sprepo_git.git_operation("remote", "-v") + if self.url not in remotes: sprepo_git.git_operation("remote", "add", "origin", self.url) topgit = os.path.join(gitroot, ".git") @@ -256,46 +266,46 @@ def sparse_checkout(self): os.path.join(self.root_dir, f.read().split()[1]), start=os.path.join(self.root_dir, self.path), ) - topgit = os.path.join(gitpath, "modules") + rootdotgit = os.path.join(gitpath, "modules", self.name) else: - topgit = os.path.relpath( - os.path.join(self.root_dir, ".git", "modules"), + rootdotgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules", self.name), start=os.path.join(self.root_dir, self.path), ) - with utils.pushd(sprep_repo): - if not os.path.isdir(topgit): - os.makedirs(topgit) - topgit += os.sep + self.name - if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(topgit,".git")): - shutil.rmtree(os.path.join(topgit,".git")) - shutil.move(".git", topgit) + if os.path.isdir(os.path.join(rootdotgit,".git")): + shutil.rmtree(os.path.join(rootdotgit,".git")) + shutil.move(".git", rootdotgit) with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(topgit)) - # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) - gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + f.write("gitdir: " + os.path.relpath(rootdotgit)) + infodir = os.path.join(rootdotgit, "info") + if not os.path.isdir(infodir): + os.makedirs(infodir) + gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) if os.path.isfile(gitsparse): self.logger.warning( - "submodule {} is already initialized {}".format(self.name, topgit) + "submodule {} is already initialized {}".format(self.name, rootdotgit) ) return with utils.pushd(sprep_repo): if os.path.isfile(self.fxsparse): + shutil.copy(self.fxsparse, gitsparse) # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") - sprepo_git.git_operation("checkout", self.fxtag) - - print(f"Successfully checked out {self.name:>20} at {self.fxtag}") - rgit.config_set_value(f'submodule "{self.name}"', "active", "true") - rgit.config_set_value(f'submodule "{self.name}"', "url", self.url) - rgit.config_set_value(f'submodule "{self.name}"', "path", self.path) + status,_ = sprepo_git.git_operation("checkout", self.fxtag) + if status: + print(f"Error checking out {self.name:>20} at {self.fxtag}") + else: + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + rgit.config_set_value('submodule.' + self.name, "active", "true") + rgit.config_set_value('submodule.' + self.name, "url", self.url) + rgit.config_set_value('submodule.' + self.name, "path", self.path) def update(self): """ @@ -342,7 +352,7 @@ def update(self): git.git_operation("clone", self.url, self.path) smgit = GitInterface(repodir, self.logger) if not tag: - tag = smgit.git_operation("describe", "--tags", "--always").rstrip() + status, tag = smgit.git_operation("describe", "--tags", "--always") smgit.git_operation("checkout", tag) # Now need to move the .git dir to the submodule location rootdotgit = os.path.join(self.root_dir, ".git") @@ -350,7 +360,7 @@ def update(self): with open(rootdotgit) as f: line = f.readline() if line.startswith("gitdir: "): - rootdotgit = line[8:].rstrip() + rootdotgit = line[8:] newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) if os.path.exists(newpath): @@ -393,15 +403,16 @@ def update(self): git = GitInterface(submoddir, self.logger) # first make sure the url is correct newremote = self._add_remote(git) - tags = git.git_operation("tag", "-l") + status, tags = git.git_operation("tag", "-l") fxtag = self.fxtag if fxtag and fxtag not in tags: git.git_operation("fetch", newremote, "--tags") - atag = git.git_operation("describe", "--tags", "--always").rstrip() + status, atag = git.git_operation("describe", "--tags", "--always") if fxtag and fxtag != atag: try: - git.git_operation("checkout", fxtag) - print(f"{self.name:>20} updated to {fxtag}") + status, _ = git.git_operation("checkout", fxtag) + if not status: + print(f"{self.name:>20} updated to {fxtag}") except Exception as error: print(error) diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py index 1a2d5ccf..c4f43d52 100644 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ b/.lib/git-fleximod/git_fleximod/utils.py @@ -307,12 +307,12 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False) # simple status check. If returning, it is the callers # responsibility determine if an error occurred and handle it # appropriately. + msg_context = ( + "Process did not run successfully; " + "returned status {0}".format(error.returncode) + ) + msg = failed_command_msg(msg_context, commands, output=error.output) if not return_to_caller: - msg_context = ( - "Process did not run successfully; " - "returned status {0}".format(error.returncode) - ) - msg = failed_command_msg(msg_context, commands, output=error.output) logging.error(error) logging.error(msg) log_process_output(error.output) diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock index 3a74effc..ac82fb0d 100644 --- a/.lib/git-fleximod/poetry.lock +++ b/.lib/git-fleximod/poetry.lock @@ -30,13 +30,13 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index 850e57d5..65924ff9 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.8.4" +version = "0.9.4" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index bd82c557..be0b799d 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.8.4" +current = "0.9.4" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 81edbe71..1dd1b86f 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -32,7 +32,7 @@ def logger(): "submodule_name": "test_optional", "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag None, expected tag is MPIserial_2.4.0 (optional)", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.1, expected tag is MPIserial_2.4.0 (optional)", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] @@ -46,7 +46,7 @@ def logger(): "submodule_name": "test_alwaysoptional", "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag None, expected tag is e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.1, expected tag is e5cf35c", "status4" : "test_alwaysoptional at hash e5cf35c", "gitmodules_content": """ [submodule "test_alwaysoptional"] diff --git a/ccpp_framework b/ccpp_framework index 27d0486f..fca3a9e3 160000 --- a/ccpp_framework +++ b/ccpp_framework @@ -1 +1 @@ -Subproject commit 27d0486f61e86d54921edc82139884e5f9c67700 +Subproject commit fca3a9e3df101bd07a228631e5509ec4a93a9906 diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 45c37530..491e5624 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 45c37530ead4f5b93621e3d979e972ea91217923 +Subproject commit 491e56247815ef23bfd8dba65d1e3c3b78ba164a diff --git a/test/existing-test-failures.txt b/test/existing-test-failures.txt index 05e0a145..975b8681 100644 --- a/test/existing-test-failures.txt +++ b/test/existing-test-failures.txt @@ -1,8 +1,3 @@ SMS_Ln2.mpasa480_mpasa480.FKESSLER.derecho_intel.cam-outfrq_kessler_mpas_derecho (Overall: FAIL) SMS_Ln2.mpasa480_mpasa480.FKESSLER.derecho_gnu.cam-outfrq_kessler_mpas_derecho (Overall: FAIL) - will fail until MPAS is fully integrated - -SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_intel.cam-outfrq_se_cslam_analy_ic (Overall: PEND) details: -SMS_Ln9.ne5pg3_ne5pg3_mg37.FCAM7.derecho_gnu.cam-outfrq_se_cslam_analy_ic (Overall: PEND) details: - - build failure due to dadadj_apply_qv_tendency removal, needs to be updated to use constituent tendency updater atmospheric_physics#179 - - also expected runtime failure CAM-SIMA#335