Skip to content

Commit

Permalink
update git-fleximod; update framework tag and atmos phys tag; update …
Browse files Browse the repository at this point in the history
…existing test failures
  • Loading branch information
peverwhee committed Jan 8, 2025
1 parent 87f6fad commit a8ffa04
Show file tree
Hide file tree
Showing 13 changed files with 105 additions and 91 deletions.
4 changes: 2 additions & 2 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[submodule "ccpp-framework"]
path = ccpp_framework
url = https://github.com/NCAR/ccpp-framework
fxtag = 2024-11-07-dev
fxtag = 2025-01-06-dev
fxrequired = AlwaysRequired
fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
[submodule "history"]
Expand All @@ -20,7 +20,7 @@
[submodule "ncar-physics"]
path = src/physics/ncar_ccpp
url = https://github.com/ESCOMP/atmospheric_physics
fxtag = 491e56247815ef23bfd8dba65d1e3c3b78ba164a
fxtag = 74e905b7a0ee5b2d2bfc3e3dd942eb9963398373
fxrequired = AlwaysRequired
fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
[submodule "ccs_config"]
Expand Down
38 changes: 21 additions & 17 deletions .lib/git-fleximod/git_fleximod/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,31 @@
import argparse
from git_fleximod import utils

__version__ = "0.8.4"
__version__ = "0.9.4"

def find_root_dir(filename=".gitmodules"):
""" finds the highest directory in tree
which contains a file called filename """
d = Path.cwd()
root = Path(d.root)
dirlist = []
dl = d
while dl != root:
dirlist.append(dl)
dl = dl.parent
dirlist.append(root)
dirlist.reverse()

for dl in dirlist:
attempt = dl / filename
if attempt.is_file():
return str(dl)
return None

try:
root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
output_to_caller=True ).rstrip()
except:
d = Path.cwd()
root = Path(d.root)
dirlist = []
dl = d
while dl != root:
dirlist.append(dl)
dl = dl.parent
dirlist.append(root)
dirlist.reverse()

for dl in dirlist:
attempt = dl / filename
if attempt.is_file():
return str(dl)
return None
return Path(root)

def get_parser():
description = """
Expand Down
11 changes: 6 additions & 5 deletions .lib/git-fleximod/git_fleximod/git_fleximod.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
# set the repository remote

logger.info("Setting remote origin in {}/{}".format(root_dir, path))
status = sprepo_git.git_operation("remote", "-v")
if url not in status:
_, remotelist = sprepo_git.git_operation("remote", "-v")
if url not in remotelist:
sprepo_git.git_operation("remote", "add", "origin", url)

topgit = os.path.join(gitroot, ".git")
Expand Down Expand Up @@ -181,6 +181,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
url = gitmodules.get(name, "url")
assert path and url, f"Malformed .gitmodules file {path} {url}"
tag = gitmodules.get(name, "fxtag")
if not tag:
tag = gitmodules.get(name, "hash")
fxurl = gitmodules.get(name, "fxDONOTUSEurl")
fxsparse = gitmodules.get(name, "fxsparse")
fxrequired = gitmodules.get(name, "fxrequired")
Expand Down Expand Up @@ -213,7 +215,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):

def git_toplevelroot(root_dir, logger):
rgit = GitInterface(root_dir, logger)
superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
_, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
return superroot

def submodules_update(gitmodules, root_dir, requiredlist, force):
Expand Down Expand Up @@ -250,7 +252,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
newrequiredlist = ["AlwaysRequired"]
if optional:
newrequiredlist.append("AlwaysOptional")

submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)

def local_mods_output():
Expand Down Expand Up @@ -342,7 +343,7 @@ def main():
excludelist=excludelist,
)
if not gitmodules.sections():
sys.exit("No submodule components found")
sys.exit(f"No submodule components found, root_dir={root_dir}")
retval = 0
if action == "update":
submodules_update(gitmodules, root_dir, fxrequired, force)
Expand Down
7 changes: 5 additions & 2 deletions .lib/git-fleximod/git_fleximod/gitinterface.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs):
command = self._git_command(operation, *newargs)
if isinstance(command, list):
try:
return utils.execute_subprocess(command, output_to_caller=True)
status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True)
return status, output.rstrip()
except Exception as e:
sys.exit(e)
else:
return command
return 0, command

def config_get_value(self, section, name):
if self._use_module:
Expand All @@ -81,6 +82,8 @@ def config_get_value(self, section, name):
def config_set_value(self, section, name, value):
if self._use_module:
with self.repo.config_writer() as writer:
if "." in section:
section = section.replace("."," \"")+'"'
writer.set_value(section, name, value)
writer.release() # Ensure changes are saved
else:
Expand Down
103 changes: 57 additions & 46 deletions .lib/git-fleximod/git_fleximod/submodule.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@ def status(self):
if not os.path.exists(os.path.join(smpath, ".git")):
rootgit = GitInterface(self.root_dir, self.logger)
# submodule commands use path, not name
tags = rootgit.git_operation("ls-remote", "--tags", self.url)
result = rootgit.git_operation("submodule","status",smpath).split()
status, tags = rootgit.git_operation("ls-remote", "--tags", self.url)
status, result = rootgit.git_operation("submodule","status",smpath)
result = result.split()

if result:
ahash = result[0][1:]
Expand All @@ -80,9 +81,9 @@ def status(self):
result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}"
needsupdate = True
elif self.fxtag:
ahash = rootgit.git_operation(
status, ahash = rootgit.git_operation(
"submodule", "status", "{}".format(self.path)
).rstrip()
)
ahash = ahash[1 : len(self.fxtag) + 1]
if self.fxtag == ahash:
result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}"
Expand All @@ -96,14 +97,15 @@ def status(self):
else:
with utils.pushd(smpath):
git = GitInterface(smpath, self.logger)
remote = git.git_operation("remote").rstrip()
status, remote = git.git_operation("remote")
if remote == '':
result = f"e {self.name:>20} has no associated remote"
testfails = True
needsupdate = True
return result, needsupdate, localmods, testfails
rurl = git.git_operation("ls-remote","--get-url").rstrip()
line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0]
status, rurl = git.git_operation("ls-remote","--get-url")
status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"")
line = lines.partition('\n')[0]
parts = line.split()
ahash = parts[0][1:]
atag = None
Expand All @@ -120,7 +122,7 @@ def status(self):


#print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
# atag = git.git_operation("describe", "--tags", "--always").rstrip()
# atag = git.git_operation("describe", "--tags", "--always")
# ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0]

recurse = False
Expand Down Expand Up @@ -149,10 +151,10 @@ def status(self):
result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}"
testfails = False

status = git.git_operation("status", "--ignore-submodules", "-uno")
if "nothing to commit" not in status:
status, output = git.git_operation("status", "--ignore-submodules", "-uno")
if "nothing to commit" not in output:
localmods = True
result = "M" + textwrap.indent(status, " ")
result = "M" + textwrap.indent(output, " ")
# print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}")
return result, needsupdate, localmods, testfails

Expand All @@ -171,10 +173,11 @@ def _add_remote(self, git):
Returns:
str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL.
"""
remotes = git.git_operation("remote", "-v").splitlines()
status, remotes = git.git_operation("remote", "-v")
remotes = remotes.splitlines()
upstream = None
if remotes:
upstream = git.git_operation("ls-remote", "--get-url").rstrip()
status, upstream = git.git_operation("ls-remote", "--get-url")
newremote = "newremote.00"
tmpurl = self.url.replace("git@github.com:", "https://github.com/")
line = next((s for s in remotes if self.url in s or tmpurl in s), None)
Expand All @@ -183,7 +186,7 @@ def _add_remote(self, git):
return newremote
else:
i = 0
while "newremote" in remotes:
while newremote in remotes:
i = i + 1
newremote = f"newremote.{i:02d}"
else:
Expand Down Expand Up @@ -214,12 +217,19 @@ def sparse_checkout(self):
"""
self.logger.info("Called sparse_checkout for {}".format(self.name))
rgit = GitInterface(self.root_dir, self.logger)
superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
if superroot:
gitroot = superroot.strip()
else:
gitroot = self.root_dir.strip()
assert os.path.isdir(os.path.join(gitroot, ".git"))
gitroot = self.root_dir
# Now need to move the .git dir to the submodule location
rootdotgit = os.path.join(self.root_dir, ".git")
while os.path.isfile(rootdotgit):
with open(rootdotgit) as f:
line = f.readline().rstrip()
if line.startswith("gitdir: "):
rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:]))
assert os.path.isdir(rootdotgit)
# first create the module directory
if not os.path.isdir(os.path.join(self.root_dir, self.path)):
os.makedirs(os.path.join(self.root_dir, self.path))
Expand All @@ -244,8 +254,8 @@ def sparse_checkout(self):
# set the repository remote

self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path))
status = sprepo_git.git_operation("remote", "-v")
if self.url not in status:
status, remotes = sprepo_git.git_operation("remote", "-v")
if self.url not in remotes:
sprepo_git.git_operation("remote", "add", "origin", self.url)

topgit = os.path.join(gitroot, ".git")
Expand All @@ -256,46 +266,46 @@ def sparse_checkout(self):
os.path.join(self.root_dir, f.read().split()[1]),
start=os.path.join(self.root_dir, self.path),
)
topgit = os.path.join(gitpath, "modules")
rootdotgit = os.path.join(gitpath, "modules", self.name)
else:
topgit = os.path.relpath(
os.path.join(self.root_dir, ".git", "modules"),
rootdotgit = os.path.relpath(
os.path.join(self.root_dir, ".git", "modules", self.name),
start=os.path.join(self.root_dir, self.path),
)

with utils.pushd(sprep_repo):
if not os.path.isdir(topgit):
os.makedirs(topgit)
topgit += os.sep + self.name

if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")):
with utils.pushd(sprep_repo):
if os.path.isdir(os.path.join(topgit,".git")):
shutil.rmtree(os.path.join(topgit,".git"))
shutil.move(".git", topgit)
if os.path.isdir(os.path.join(rootdotgit,".git")):
shutil.rmtree(os.path.join(rootdotgit,".git"))
shutil.move(".git", rootdotgit)
with open(".git", "w") as f:
f.write("gitdir: " + os.path.relpath(topgit))
# assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
f.write("gitdir: " + os.path.relpath(rootdotgit))
infodir = os.path.join(rootdotgit, "info")
if not os.path.isdir(infodir):
os.makedirs(infodir)
gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout"))
if os.path.isfile(gitsparse):
self.logger.warning(
"submodule {} is already initialized {}".format(self.name, topgit)
"submodule {} is already initialized {}".format(self.name, rootdotgit)
)
return

with utils.pushd(sprep_repo):
if os.path.isfile(self.fxsparse):

shutil.copy(self.fxsparse, gitsparse)


# Finally checkout the repo
sprepo_git.git_operation("fetch", "origin", "--tags")
sprepo_git.git_operation("checkout", self.fxtag)

print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
rgit.config_set_value(f'submodule "{self.name}"', "active", "true")
rgit.config_set_value(f'submodule "{self.name}"', "url", self.url)
rgit.config_set_value(f'submodule "{self.name}"', "path", self.path)
status,_ = sprepo_git.git_operation("checkout", self.fxtag)
if status:
print(f"Error checking out {self.name:>20} at {self.fxtag}")
else:
print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
rgit.config_set_value('submodule.' + self.name, "active", "true")
rgit.config_set_value('submodule.' + self.name, "url", self.url)
rgit.config_set_value('submodule.' + self.name, "path", self.path)

def update(self):
"""
Expand Down Expand Up @@ -342,15 +352,15 @@ def update(self):
git.git_operation("clone", self.url, self.path)
smgit = GitInterface(repodir, self.logger)
if not tag:
tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
status, tag = smgit.git_operation("describe", "--tags", "--always")
smgit.git_operation("checkout", tag)
# Now need to move the .git dir to the submodule location
rootdotgit = os.path.join(self.root_dir, ".git")
if os.path.isfile(rootdotgit):
with open(rootdotgit) as f:
line = f.readline()
if line.startswith("gitdir: "):
rootdotgit = line[8:].rstrip()
rootdotgit = line[8:]

newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name))
if os.path.exists(newpath):
Expand Down Expand Up @@ -393,15 +403,16 @@ def update(self):
git = GitInterface(submoddir, self.logger)
# first make sure the url is correct
newremote = self._add_remote(git)
tags = git.git_operation("tag", "-l")
status, tags = git.git_operation("tag", "-l")
fxtag = self.fxtag
if fxtag and fxtag not in tags:
git.git_operation("fetch", newremote, "--tags")
atag = git.git_operation("describe", "--tags", "--always").rstrip()
status, atag = git.git_operation("describe", "--tags", "--always")
if fxtag and fxtag != atag:
try:
git.git_operation("checkout", fxtag)
print(f"{self.name:>20} updated to {fxtag}")
status, _ = git.git_operation("checkout", fxtag)
if not status:
print(f"{self.name:>20} updated to {fxtag}")
except Exception as error:
print(error)

Expand Down
10 changes: 5 additions & 5 deletions .lib/git-fleximod/git_fleximod/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,12 +307,12 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False)
# simple status check. If returning, it is the callers
# responsibility determine if an error occurred and handle it
# appropriately.
msg_context = (
"Process did not run successfully; "
"returned status {0}".format(error.returncode)
)
msg = failed_command_msg(msg_context, commands, output=error.output)
if not return_to_caller:
msg_context = (
"Process did not run successfully; "
"returned status {0}".format(error.returncode)
)
msg = failed_command_msg(msg_context, commands, output=error.output)
logging.error(error)
logging.error(msg)
log_process_output(error.output)
Expand Down
Loading

0 comments on commit a8ffa04

Please sign in to comment.