From f093a6ab2e0b101d01f8e2fdb51b94432fa69c68 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 2 Jan 2024 10:57:43 -0700
Subject: [PATCH 001/131] start with basics

---
 git_setup           | 91 +++++++++++++++++++++++++++++++++++++++++++++
 git_sparse_checkout | 69 ++++++++++++++++++++++++++++++++++
 show-tags           | 20 ++++++++++
 3 files changed, 180 insertions(+)
 create mode 100755 git_setup
 create mode 100755 git_sparse_checkout
 create mode 100755 show-tags

diff --git a/git_setup b/git_setup
new file mode 100755
index 000000000..52901897d
--- /dev/null
+++ b/git_setup
@@ -0,0 +1,91 @@
+#!/bin/bash
+# Script to fetch submodules for CAM
+# if argument -internal-only is supplied then only checkout CAM internal submodules, otherwise checkout everything needed for
+# cam as a top level repository
+#
+set -e
+script="install"
+function usage {
+    echo -e "\nusage: $script [-i/--internal-only]\n"
+}
+# Set default arguments
+internal_only=0
+
+# Process arguments
+while [ "$1" != "" ];
+do
+    case $1 in
+
+        # Only checkout CISM internal submodules
+        -i | -internal-only | --internal-only)
+            internal_only=1
+            ;;
+
+        *)
+            echo "$script: illegal option $1"
+            usage
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+########################
+
+# Start with sparse checkouts
+if ! test -f bash-scripts/bin/git_sparse_checkout; then
+  echo "Getting git_sparse_checkout script"
+  git clone -n https://github.com/frgomes/bash-scripts --depth 1
+  pushd bash-scripts 1>/dev/null
+  git checkout HEAD -- bin/git_sparse_checkout
+  popd 1>/dev/null
+fi
+
+
+
+
+
+
+./bash-scripts/bin/git_sparse_checkout  https://github.com/larson-group/clubb_release clubb_4ncar_20221129_59cb19f_branch src/physics/clubb -- src/CLUBB_core/ src/SILHS/
+git restore src/physics/clubb
+
+./bash-scripts/bin/git_sparse_checkout https://github.com/CFMIP/COSPv2.0 master src/physics/cosp2/src -- src/
+git restore src/physics/cosp2/src
+
+./bash-scripts/bin/git_sparse_checkout https://github.com/MPAS-Dev/MPAS-Model.git develop src/dynamics/mpas/dycore -- src/external/ src/operators/ src/tools/ \
+                      src/core_atmosphere/ src/framework/
+git restore src/dynamics/mpas/dycore
+
+submodules=('chem_proc' 'src/physics/carma/base' 'src/physics/pumas' 'src/physics/pumas-frozen' 'src/physics/ali_arms' 'src/atmos_phys' 'src/dynamics/fv3/atmos_cubed_sphere' 'src/hemco')
+for mod in "${submodules[@]}"
+do
+   echo "Initializing $mod"
+   git submodule update --init $mod
+done
+
+if [ ${internal_only} -eq 1 ]
+  then
+    exit 0
+fi
+
+
+submodules=('ccs_config' 'components/cice5' 'components/cice' 'components/cmeps' \
+           'components/cdeps' 'components/cpl7' 'share' 'libraries/mct' \
+           'libraries/parallelio' 'cime' 'libraries/FMS' 'components/mosart' \
+           'components/rtm')
+
+for mod in "${submodules[@]}"
+do
+   echo "Initializing $mod"
+   git submodule update --init --recursive $mod
+done
+
+sparse_submodules=('components/clm' 'components/cism')
+for mod in "${sparse_submodules[@]}"
+do
+   echo "Initializing $mod"
+   git submodule update --init $mod
+   pushd $mod 1>/dev/null
+   bash ./install -internal-only
+   popd 1>/dev/null
+done
diff --git a/git_sparse_checkout b/git_sparse_checkout
new file mode 100755
index 000000000..ec80069c2
--- /dev/null
+++ b/git_sparse_checkout
@@ -0,0 +1,69 @@
+#!/bin/bash -eu
+
+# see also: git_origin and git_clone
+
+function git_sparse_checkout {
+    local self=$(readlink -f "${BASH_SOURCE[0]}")
+    local app=$(basename $self)
+    local usage=\
+"USAGE: ${app} repository-URL [branch] [project-directory] [[--] [list-of-files-or-directories]]
+  where:
+    'repository-URL' is a valid URL pointing to a Git repository.
+    'branch' is a branch, atag or a commit id. Default: master.
+    'project-directory' is a folder to be created and populated. Default: the project name.
+    'list-of-files-or-directories' is a list of file names or directories separated by spaces.
+  Examples:
+    ${app} http://github.com/frgomes/bash-scripts -- README.md
+    ${app} http://github.com/frgomes/bash-scripts develop -- README.md bin/
+    ${app} http://github.com/frgomes/bash-scripts develop tmpdir -- README.md bin/ docs/"
+
+    # obtain repository-URL, e.g.: http://github.com/frgomes/bash-scripts
+    [[ $# != 0 ]] || (echo "${usage}" 1>&2 ; return 1)
+    local arg=${1}
+    [[ "${arg}" != "--" ]] || (echo "${usage}" 1>&2 ; return 1)
+    local url="${arg}"
+    [[ $# == 0 ]] || shift
+
+    # obtain branch, which the default is master for historical reasons
+    if [[ "${arg}" != "--" ]] ;then arg="${1:-master}" ;fi
+    if [[ "${arg}" == "--" ]] ;then
+      local tag=master
+    else
+      local tag="${arg}"
+      [[ $# == 0 ]] || shift
+    fi
+
+    # obtain the project directory, which defaults to the repository name
+    local prj=$(echo "$url" | sed 's:/:\n:g' | tail -1)
+
+    if [[ "${arg}" != "--" ]] ;then arg="${1:-.}" ;fi
+    if [[ "${arg}" == "--" || "${arg}" == "." ]] ;then
+      local dir=$(readlink -f "./${prj}")
+    else
+      local dir=$(readlink -f "${arg}")
+      [[ $# == 0 ]] || shift
+    fi
+
+    if [[ "${arg}" == "--" ]] ;then [[ $# == 0 ]] || shift; fi
+    if [[ "${1:-}" == "--" ]] ;then [[ $# == 0 ]] || shift; fi
+
+    # Note: any remaining arguments after these above are considered as a
+    # list of files or directories to be downloaded. Names of directories
+    # must be followed by a slash /.
+
+    local sparse=true
+    local opts='--depth=1'
+
+    # now perform the sparse checkout
+    mkdir -p "${dir}"
+    git -C "${dir}" init
+    git -C "${dir}" config core.sparseCheckout ${sparse}
+    for path in $* ;do
+        echo "${path}" >> ${dir}/.git/info/sparse-checkout
+    done
+    git -C "${dir}" remote add origin ${url}
+    git -C "${dir}" fetch ${opts} origin ${tag}
+    git -C "${dir}" checkout ${tag}
+}
+
+git_sparse_checkout $@
diff --git a/show-tags b/show-tags
new file mode 100755
index 000000000..56c7d9c5e
--- /dev/null
+++ b/show-tags
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+this_dir=$(pwd)
+printf "\nSubmodule status\n"
+printf "(currently checked out commit for each submodule)\n"
+printf "(when the submodule is initialized and a tag exists, the commit is shown as: 'most recent tag-commits since tag-commit hash')\n"
+printf "(when the submodule is not initialized, only the checked out commit is shown)\n\n"
+grep path .gitmodules | sed 's/.*= //' | while read x
+do
+  cd "$this_dir"
+  printf "$x\n   - current commit: "
+  if [ "$(ls -A $x)" ] ; then
+    cd "$x"
+    git describe --tags --always
+  else
+    git submodule status $x | sed 's/^-//' | awk '{ print $1 }'
+  fi
+done
+printf "\n"
+

From f530e88acee6e959e9345b35a7e00b71d81641bf Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 3 Jan 2024 17:20:30 -0700
Subject: [PATCH 002/131] module based method, sparse working

---
 git_setup               |  37 ++---
 git_setup.py            | 104 +++++++++++++
 git_sparse_checkout     |  10 +-
 modules/__init__.py     |   0
 modules/lstripreader.py |  44 ++++++
 modules/utils.py        | 325 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 501 insertions(+), 19 deletions(-)
 create mode 100644 git_setup.py
 create mode 100644 modules/__init__.py
 create mode 100644 modules/lstripreader.py
 create mode 100644 modules/utils.py

diff --git a/git_setup b/git_setup
index 52901897d..a5b27b13f 100755
--- a/git_setup
+++ b/git_setup
@@ -1,10 +1,10 @@
 #!/bin/bash
-# Script to fetch submodules for CAM
-# if argument -internal-only is supplied then only checkout CAM internal submodules, otherwise checkout everything needed for
-# cam as a top level repository
+# Script to fetch submodules for esm applications 
+# if argument -internal-only is supplied then only checkout internal submodules, otherwise checkout everything needed for
+# a top level repository
 #
 set -e
-script="install"
+script=$0
 function usage {
     echo -e "\nusage: $script [-i/--internal-only]\n"
 }
@@ -31,28 +31,31 @@ do
 done
 
 ########################
+declare -A submods
+submod_list=$(git config -f .gitmodules --list)
+
+while IFS= read -r line;
+do
+    readarray -d. -t strarr <<< "$line" 
+    if [[ "strarr[3]" == "path" ]]; then
+        echo "path is $strarr[4]"
+    fi
+done <<< "$submod_list"
+exit 0
 
 # Start with sparse checkouts
-if ! test -f bash-scripts/bin/git_sparse_checkout; then
+if ! test -f bin/git_sparse_checkout; then
   echo "Getting git_sparse_checkout script"
-  git clone -n https://github.com/frgomes/bash-scripts --depth 1
-  pushd bash-scripts 1>/dev/null
-  git checkout HEAD -- bin/git_sparse_checkout
-  popd 1>/dev/null
+  git clone https://github.com/jedwards4b/gitscripts --depth 1 bin
 fi
 
-
-
-
-
-
-./bash-scripts/bin/git_sparse_checkout  https://github.com/larson-group/clubb_release clubb_4ncar_20221129_59cb19f_branch src/physics/clubb -- src/CLUBB_core/ src/SILHS/
+./bin/git_sparse_checkout  https://github.com/larson-group/clubb_release clubb_4ncar_20221129_59cb19f_branch src/physics/clubb -- src/CLUBB_core/ src/SILHS/
 git restore src/physics/clubb
 
-./bash-scripts/bin/git_sparse_checkout https://github.com/CFMIP/COSPv2.0 master src/physics/cosp2/src -- src/
+./bin/git_sparse_checkout https://github.com/CFMIP/COSPv2.0 master src/physics/cosp2/src -- src/
 git restore src/physics/cosp2/src
 
-./bash-scripts/bin/git_sparse_checkout https://github.com/MPAS-Dev/MPAS-Model.git develop src/dynamics/mpas/dycore -- src/external/ src/operators/ src/tools/ \
+./bin/git_sparse_checkout https://github.com/MPAS-Dev/MPAS-Model.git develop src/dynamics/mpas/dycore -- src/external/ src/operators/ src/tools/ \
                       src/core_atmosphere/ src/framework/
 git restore src/dynamics/mpas/dycore
 
diff --git a/git_setup.py b/git_setup.py
new file mode 100644
index 000000000..5d70486d6
--- /dev/null
+++ b/git_setup.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+import os
+import shutil
+import logging
+from modules import utils
+from configparser import ConfigParser
+from modules.lstripreader import LstripReader
+
+def parse_submodules_desc_section(section, section_items):
+    """Create a dict for this submodule description"""
+    desc = {}
+    esmrequired_options = ("T:T", "I:T", "I:F", "T:F")
+    for item in section_items:
+        name = item[0].strip().lower()
+        desc[name] = item[1].strip()
+    if not "esmrequired" in desc:
+        desc["esmrequired"] = "I:T"
+    
+    if desc["esmrequired"] not in esmrequired_options:
+        val = desc["esmrequired"]
+        utils.fatal_error(f"esmrequired set to {val} which is not a supported option {esmrequired_options}")
+    return desc
+
+
+def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
+    # first create the module directory
+    if not os.path.isdir(path):
+        os.makedirs(path)
+    # Check first if the module is already defined
+    # and the sparse-checkout file exists
+    cmd = ("git", "rev-parse", "--show-toplevel")
+    topdir = utils.execute_subprocess(cmd, output_to_caller=True).rstrip()
+    topgit = os.path.join(topdir, ".git", "modules")
+    gitsparse = os.path.join(topgit, name, "info","sparse-checkout")
+    if os.path.isfile(gitsparse):
+        logging.warning(f"submodule {name} is already initialized")
+        return
+    
+    #initialize a new git repo and set the sparse checkout flag
+    cmd = ("git", "-C", path, "init")
+    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    cmd = ("git", "-C", path, "config", "core.sparseCheckout","true") 
+    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    # set the repository remote
+    cmd = ("git", "-C", path, "remote", "add", "origin", url) 
+    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    
+    if not os.path.isdir(topgit):
+        os.makedirs(topgit)
+    topgit = os.path.join(topgit,name)
+    
+    shutil.move(os.path.join(path, ".git"), topgit)
+
+    shutil.copy(os.path.join(path,sparsefile), gitsparse)
+        
+    with open(os.path.join(path, ".git"), "w") as f:
+        f.write("gitdir: " + os.path.relpath(topgit, path))
+
+    #Finally checkout the repo
+    cmd = ("git", "-C", path, "fetch", "--depth=1", "origin", "--tags")
+    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    cmd = ("git", "-C", path, "checkout", tag)
+    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    print(f"Successfully checked out {name}")
+    
+def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
+    root_dir = os.path.abspath(root_dir)
+
+    msg = 'In directory : {0}'.format(root_dir)
+    logging.info(msg)
+
+    file_path = os.path.join(root_dir, file_name)
+    if not os.path.exists(file_name):
+        msg = ('ERROR: submodules description file, "{0}", does not '
+               'exist in dir:\n    {1}'.format(file_name, root_dir))
+        utils.fatal_error(msg)
+    config = ConfigParser()
+    config.read_file(LstripReader(file_path), source=file_name)
+    for section in config.sections():
+        name = section[11:-1]
+        submodule_desc = parse_submodules_desc_section(section,config.items(section))
+        if submodule_desc["esmrequired"] not in esmrequired:
+            if "T:F" in esmrequired or submodule_desc["esmrequired"].startswith("I:"):
+                print(f"Skipping optional component {section}")
+                # TODO change to logging
+                #                logging.info(f"Skipping optional component {section}")
+            continue
+        if "esmsparse" in submodule_desc:
+            if "esmtag" in submodule_desc:
+                tag = submodule_desc["esmtag"]
+            else:
+                tag = "master"
+            submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
+                                      submodule_desc["esmsparse"], tag)
+
+
+
+
+
+esmrequired = ("I:T", "T:T")
+root_dir = os.getcwd()
+gitmodules = read_gitmodules_file(root_dir, esmrequired)
+
+    
diff --git a/git_sparse_checkout b/git_sparse_checkout
index ec80069c2..9b52bb29a 100755
--- a/git_sparse_checkout
+++ b/git_sparse_checkout
@@ -55,13 +55,19 @@ function git_sparse_checkout {
     local opts='--depth=1'
 
     # now perform the sparse checkout
+    
     mkdir -p "${dir}"
     git -C "${dir}" init
     git -C "${dir}" config core.sparseCheckout ${sparse}
+    git -C "${dir}" remote add origin ${url}
+    super=$(git rev-parse --show-toplevel)
+    pathtodotgit=$(realpath -m --relative-to=${dir} ${super})/.git
+    relpathfromsuper=$(realpath -m --relative-to=${super} ${dir})
+    mv ${dir}/.git  ${super}/.git/modules/${relpathfromsuper}
+    echo "gitdir: ${pathtodotgit}/modules/${relpathfromsuper}" > ${dir}/.git
     for path in $* ;do
-        echo "${path}" >> ${dir}/.git/info/sparse-checkout
+        echo "${path}" >> ${super}/.git/modules/${relpathfromsuper}/info/sparse-checkout
     done
-    git -C "${dir}" remote add origin ${url}
     git -C "${dir}" fetch ${opts} origin ${tag}
     git -C "${dir}" checkout ${tag}
 }
diff --git a/modules/__init__.py b/modules/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/modules/lstripreader.py b/modules/lstripreader.py
new file mode 100644
index 000000000..530abd297
--- /dev/null
+++ b/modules/lstripreader.py
@@ -0,0 +1,44 @@
+
+class LstripReader(object):
+    "LstripReader formats .gitmodules files to be acceptable for configparser"
+    def __init__(self, filename):
+        with open(filename, 'r') as infile:
+            lines = infile.readlines()
+        self._lines = list()
+        self._num_lines = len(lines)
+        self._index = 0
+        for line in lines:
+            self._lines.append(line.lstrip())
+
+    def readlines(self):
+        """Return all the lines from this object's file"""
+        return self._lines
+
+    def readline(self, size=-1):
+        """Format and return the next line or raise StopIteration"""
+        try:
+            line = self.next()
+        except StopIteration:
+            line = ''
+
+        if (size > 0) and (len(line) < size):
+            return line[0:size]
+
+        return line
+
+    def __iter__(self):
+        """Begin an iteration"""
+        self._index = 0
+        return self
+
+    def next(self):
+        """Return the next line or raise StopIteration"""
+        if self._index >= self._num_lines:
+            raise StopIteration
+
+        self._index = self._index + 1
+        return self._lines[self._index - 1]
+
+    def __next__(self):
+        return self.next()
+
diff --git a/modules/utils.py b/modules/utils.py
new file mode 100644
index 000000000..827153879
--- /dev/null
+++ b/modules/utils.py
@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""
+Common public utilities for manic package
+
+"""
+
+import logging
+import os
+import subprocess
+import sys
+from threading import Timer
+
+LOCAL_PATH_INDICATOR = '.'
+# ---------------------------------------------------------------------
+#
+# screen and logging output and functions to massage text for output
+#
+# ---------------------------------------------------------------------
+
+
+def log_process_output(output):
+    """Log each line of process output at debug level so it can be
+    filtered if necessary. By default, output is a single string, and
+    logging.debug(output) will only put log info heading on the first
+    line. This makes it hard to filter with grep.
+
+    """
+    output = output.split('\n')
+    for line in output:
+        logging.debug(line)
+
+
+def printlog(msg, **kwargs):
+    """Wrapper script around print to ensure that everything printed to
+    the screen also gets logged.
+
+    """
+    logging.info(msg)
+    if kwargs:
+        print(msg, **kwargs)
+    else:
+        print(msg)
+    sys.stdout.flush()
+
+
+def last_n_lines(the_string, n_lines, truncation_message=None):
+    """Returns the last n lines of the given string
+
+    Args:
+        the_string: str
+        n_lines: int
+        truncation_message: str, optional
+
+    Returns a string containing the last n lines of the_string
+
+    If truncation_message is provided, the returned string begins with
+    the given message if and only if the string is greater than n lines
+    to begin with.
+    """
+
+    lines = the_string.splitlines(True)
+    if len(lines) <= n_lines:
+        return_val = the_string
+    else:
+        lines_subset = lines[-n_lines:]
+        str_truncated = ''.join(lines_subset)
+        if truncation_message:
+            str_truncated = truncation_message + '\n' + str_truncated
+        return_val = str_truncated
+
+    return return_val
+
+
+def indent_string(the_string, indent_level):
+    """Indents the given string by a given number of spaces
+
+    Args:
+       the_string: str
+       indent_level: int
+
+    Returns a new string that is the same as the_string, except that
+    each line is indented by 'indent_level' spaces.
+
+    In python3, this can be done with textwrap.indent.
+    """
+
+    lines = the_string.splitlines(True)
+    padding = ' ' * indent_level
+    lines_indented = [padding + line for line in lines]
+    return ''.join(lines_indented)
+
+# ---------------------------------------------------------------------
+#
+# error handling
+#
+# ---------------------------------------------------------------------
+
+
+def fatal_error(message):
+    """
+    Error output function
+    """
+    logging.error(message)
+    raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message))
+
+
+# ---------------------------------------------------------------------
+#
+# Data conversion / manipulation
+#
+# ---------------------------------------------------------------------
+def str_to_bool(bool_str):
+    """Convert a sting representation of as boolean into a true boolean.
+
+    Conversion should be case insensitive.
+    """
+    value = None
+    str_lower = bool_str.lower()
+    if str_lower in ('true', 't'):
+        value = True
+    elif str_lower in ('false', 'f'):
+        value = False
+    if value is None:
+        msg = ('ERROR: invalid boolean string value "{0}". '
+               'Must be "true" or "false"'.format(bool_str))
+        fatal_error(msg)
+    return value
+
+
+REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@']
+
+
+def is_remote_url(url):
+    """check if the user provided a local file path instead of a
+       remote. If so, it must be expanded to an absolute
+       path.
+
+    """
+    remote_url = False
+    for prefix in REMOTE_PREFIXES:
+        if url.startswith(prefix):
+            remote_url = True
+    return remote_url
+
+
+def split_remote_url(url):
+    """check if the user provided a local file path or a
+       remote. If remote, try to strip off protocol info.
+
+    """
+    remote_url = is_remote_url(url)
+    if not remote_url:
+        return url
+
+    for prefix in REMOTE_PREFIXES:
+        url = url.replace(prefix, '')
+
+    if '@' in url:
+        url = url.split('@')[1]
+
+    if ':' in url:
+        url = url.split(':')[1]
+
+    return url
+
+
+def expand_local_url(url, field):
+    """check if the user provided a local file path instead of a
+    remote. If so, it must be expanded to an absolute
+    path.
+
+    Note: local paths of LOCAL_PATH_INDICATOR have special meaning and
+    represent local copy only, don't work with the remotes.
+
+    """
+    remote_url = is_remote_url(url)
+    if not remote_url:
+        if url.strip() == LOCAL_PATH_INDICATOR:
+            pass
+        else:
+            url = os.path.expandvars(url)
+            url = os.path.expanduser(url)
+            if not os.path.isabs(url):
+                msg = ('WARNING: Externals description for "{0}" contains a '
+                       'url that is not remote and does not expand to an '
+                       'absolute path. Version control operations may '
+                       'fail.\n\nurl={1}'.format(field, url))
+                printlog(msg)
+            else:
+                url = os.path.normpath(url)
+    return url
+
+
+# ---------------------------------------------------------------------
+#
+# subprocess
+#
+# ---------------------------------------------------------------------
+
+# Give the user a helpful message if we detect that a command seems to
+# be hanging.
+_HANGING_SEC = 300
+
+
+def _hanging_msg(working_directory, command):
+    print("""
+
+Command '{command}'
+from directory {working_directory}
+has taken {hanging_sec} seconds. It may be hanging.
+
+The command will continue to run, but you may want to abort
+manage_externals with ^C and investigate. A possible cause of hangs is
+when svn or git require authentication to access a private
+repository. On some systems, svn and git requests for authentication
+information will not be displayed to the user. In this case, the program
+will appear to hang. Ensure you can run svn and git manually and access
+all repositories without entering your authentication information.
+
+""".format(command=command,
+           working_directory=working_directory,
+           hanging_sec=_HANGING_SEC))
+
+
+def execute_subprocess(commands, status_to_caller=False,
+                       output_to_caller=False):
+    """Wrapper around subprocess.check_output to handle common
+    exceptions.
+
+    check_output runs a command with arguments and waits
+    for it to complete.
+
+    check_output raises an exception on a nonzero return code.  if
+    status_to_caller is true, execute_subprocess returns the subprocess
+    return code, otherwise execute_subprocess treats non-zero return
+    status as an error and raises an exception.
+
+    """
+    cwd = os.getcwd()
+    msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd)
+    logging.info(msg)
+    commands_str = ' '.join(commands)
+    logging.info(commands_str)
+    return_to_caller = status_to_caller or output_to_caller
+    status = -1
+    output = ''
+    hanging_timer = Timer(_HANGING_SEC, _hanging_msg,
+                          kwargs={"working_directory": cwd,
+                                  "command": commands_str})
+    hanging_timer.start()
+    try:
+        output = subprocess.check_output(commands, stderr=subprocess.STDOUT,
+                                         universal_newlines=True)
+        log_process_output(output)
+        status = 0
+    except OSError as error:
+        msg = failed_command_msg(
+            'Command execution failed. Does the executable exist?',
+            commands)
+        logging.error(error)
+        fatal_error(msg)
+    except ValueError as error:
+        msg = failed_command_msg(
+            'DEV_ERROR: Invalid arguments trying to run subprocess',
+            commands)
+        logging.error(error)
+        fatal_error(msg)
+    except subprocess.CalledProcessError as error:
+        # Only report the error if we are NOT returning to the
+        # caller. If we are returning to the caller, then it may be a
+        # simple status check. If returning, it is the callers
+        # responsibility determine if an error occurred and handle it
+        # appropriately.
+        if not return_to_caller:
+            msg_context = ('Process did not run successfully; '
+                           'returned status {0}'.format(error.returncode))
+            msg = failed_command_msg(msg_context, commands,
+                                     output=error.output)
+            logging.error(error)
+            logging.error(msg)
+            log_process_output(error.output)
+            fatal_error(msg)
+        status = error.returncode
+    finally:
+        hanging_timer.cancel()
+
+    if status_to_caller and output_to_caller:
+        ret_value = (status, output)
+    elif status_to_caller:
+        ret_value = status
+    elif output_to_caller:
+        ret_value = output
+    else:
+        ret_value = None
+
+    return ret_value
+
+
+def failed_command_msg(msg_context, command, output=None):
+    """Template for consistent error messages from subprocess calls.
+
+    If 'output' is given, it should provide the output from the failed
+    command
+    """
+
+    if output:
+        output_truncated = last_n_lines(output, 20,
+                                        truncation_message='[... Output truncated for brevity ...]')
+        errmsg = ('Failed with output:\n' +
+                  indent_string(output_truncated, 4) +
+                  '\nERROR: ')
+    else:
+        errmsg = ''
+
+    command_str = ' '.join(command)
+    errmsg += """In directory
+    {cwd}
+{context}:
+    {command}
+""".format(cwd=os.getcwd(), context=msg_context, command=command_str)
+
+    if output:
+        errmsg += 'See above for output from failed command.\n'
+
+    return errmsg

From e0cf1ff87ad4ff09ad950d39d8777b2df377456e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 09:29:53 -0700
Subject: [PATCH 003/131] now works two ways

---
 git_setup.py            | 24 ++++++++---------
 modules/gitinterface.py | 58 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 12 deletions(-)
 create mode 100644 modules/gitinterface.py

diff --git a/git_setup.py b/git_setup.py
index 5d70486d6..56e3d2e39 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -5,6 +5,7 @@
 from modules import utils
 from configparser import ConfigParser
 from modules.lstripreader import LstripReader
+from modules.gitinterface import GitInterface
 
 def parse_submodules_desc_section(section, section_items):
     """Create a dict for this submodule description"""
@@ -28,8 +29,10 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
         os.makedirs(path)
     # Check first if the module is already defined
     # and the sparse-checkout file exists
-    cmd = ("git", "rev-parse", "--show-toplevel")
-    topdir = utils.execute_subprocess(cmd, output_to_caller=True).rstrip()
+    fullpath = os.path.join(os.getcwd(),path)
+    git = GitInterface(os.getcwd())
+    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
+
     topgit = os.path.join(topdir, ".git", "modules")
     gitsparse = os.path.join(topgit, name, "info","sparse-checkout")
     if os.path.isfile(gitsparse):
@@ -37,13 +40,12 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
         return
     
     #initialize a new git repo and set the sparse checkout flag
-    cmd = ("git", "-C", path, "init")
-    status = utils.execute_subprocess(cmd, status_to_caller=True)
-    cmd = ("git", "-C", path, "config", "core.sparseCheckout","true") 
-    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    sprepo_git = GitInterface(os.path.join(topdir,path))
+    sprepo_git.config_set_value("core", "sparseCheckout", "true")
+
     # set the repository remote
-    cmd = ("git", "-C", path, "remote", "add", "origin", url) 
-    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    cmd = ("remote", "add", "origin", url)
+    sprepo_git.git_operation("remote", "add", "origin", url)
     
     if not os.path.isdir(topgit):
         os.makedirs(topgit)
@@ -57,10 +59,8 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
         f.write("gitdir: " + os.path.relpath(topgit, path))
 
     #Finally checkout the repo
-    cmd = ("git", "-C", path, "fetch", "--depth=1", "origin", "--tags")
-    status = utils.execute_subprocess(cmd, status_to_caller=True)
-    cmd = ("git", "-C", path, "checkout", tag)
-    status = utils.execute_subprocess(cmd, status_to_caller=True)
+    sprepo_git.git_operation( "fetch", "--depth=1", "origin", "--tags")
+    sprepo_git.git_operation( "checkout", tag)
     print(f"Successfully checked out {name}")
     
 def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
diff --git a/modules/gitinterface.py b/modules/gitinterface.py
new file mode 100644
index 000000000..105acd3d2
--- /dev/null
+++ b/modules/gitinterface.py
@@ -0,0 +1,58 @@
+import os
+from modules import utils
+
+class GitInterface:
+    def __init__(self, repo_path):
+        self.repo_path = repo_path
+
+        try:
+            import git
+            self._use_module = True
+            try:
+                self.repo = git.Repo(repo_path)  # Initialize GitPython repo
+            except git.exc.InvalidGitRepositoryError:
+                self.git = git
+                self._init_git_repo()
+        except ImportError:
+            self._use_module = False
+            if not os.path.exists(os.path.join(repo_path,".git")):
+                self._init_git_repo()
+
+    def _git_command(self, operation, *args):
+        if self._use_module:
+            return getattr(self.repo.git, operation)(*args)
+        else:
+            return ["git", "-C",self.repo_path, operation] + list(args)
+
+    def _init_git_repo(self):
+        if self._use_module:
+            self.repo = self.git.Repo.init(self.repo_path)
+        else:
+            command = ("git", "-C", self.repo_path, "init")
+            utils.execute_subprocess(command)
+
+
+    def git_operation(self, operation, *args, **kwargs):
+        command = self._git_command(operation, *args)
+        if isinstance(command, list):
+            return utils.execute_subprocess(command, output_to_caller=True)
+        else:
+            return command
+
+    def config_get_value(self, section, name):
+        if self._use_module:
+            config = self.repo.config_reader()
+            return config.get_value(section, name)
+        else:
+            cmd = ("git","-C",self.repo_path,"config", "--get", f"{section}.{name}")
+            output = utils.execute_subprocess(cmd, output_to_caller=True)
+            return output.strip()
+
+    def config_set_value(self, section, name, value):
+        if self._use_module:
+            with self.repo.config_writer() as writer:
+                writer.set_value(section, name, value)
+            writer.release()  # Ensure changes are saved
+        else:
+            cmd = ("git","-C",self.repo_path,"config", f"{section}.{name}", value)
+            utils.execute_subprocess(cmd, output_to_caller=True)

From f36bc9a68a578667f83b59dbc339d96e6fdc6995 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 14:03:45 -0700
Subject: [PATCH 004/131] complete the tools

---
 git_setup.py            | 22 ++++++++++++++++++++--
 modules/gitinterface.py |  7 ++++++-
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 56e3d2e39..319e68cdf 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -14,6 +14,9 @@ def parse_submodules_desc_section(section, section_items):
     for item in section_items:
         name = item[0].strip().lower()
         desc[name] = item[1].strip()
+        # e3sm needs to have ssh protocol urls, we don't
+        if name == "url" and desc[name].startswith("git@github"):
+            desc[name] = desc[name].replace("git@github.com:","https://github.com/")
     if not "esmrequired" in desc:
         desc["esmrequired"] = "I:T"
     
@@ -29,7 +32,6 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
         os.makedirs(path)
     # Check first if the module is already defined
     # and the sparse-checkout file exists
-    fullpath = os.path.join(os.getcwd(),path)
     git = GitInterface(os.getcwd())
     topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
 
@@ -62,6 +64,17 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
     sprepo_git.git_operation( "fetch", "--depth=1", "origin", "--tags")
     sprepo_git.git_operation( "checkout", tag)
     print(f"Successfully checked out {name}")
+
+def submodule_checkout(name, url, path, tag, esmrequired):
+    git = GitInterface(os.cwd())
+    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
+    repodir = os.path.join(topdir, path)
+    git.git_operation("submodule","update","--init")
+    # Look for a .gitmodules file in the newly checkedout repo
+    if os.path.exists(os.path.join(repodir,".gitmodules")):
+        # recursively handle this checkout
+        read_gitmodules_file(repodir, esmrequired)
+    return
     
 def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
     root_dir = os.path.abspath(root_dir)
@@ -92,7 +105,12 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
                 tag = "master"
             submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
                                       submodule_desc["esmsparse"], tag)
-
+            continue
+        Iesmrequired = []
+        for setting in esmrequired:
+            if setting.startswith("I:"):
+                Iesmrequired.append(setting) 
+        submodule_checkout(name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
 
 
 
diff --git a/modules/gitinterface.py b/modules/gitinterface.py
index 105acd3d2..06203bb9d 100644
--- a/modules/gitinterface.py
+++ b/modules/gitinterface.py
@@ -1,4 +1,5 @@
 import os
+import logging
 from modules import utils
 
 class GitInterface:
@@ -13,12 +14,16 @@ def __init__(self, repo_path):
             except git.exc.InvalidGitRepositoryError:
                 self.git = git
                 self._init_git_repo()
+            msg = "Using GitPython interface to git"
         except ImportError:
             self._use_module = False
             if not os.path.exists(os.path.join(repo_path,".git")):
                 self._init_git_repo()
-
+            msg = "Using shell interface to git"
+        logging.info(msg)
+                
     def _git_command(self, operation, *args):
+        logging.info(operation)
         if self._use_module:
             return getattr(self.repo.git, operation)(*args)
         else:

From 2beceaa56b02c1d607696a3d4259a54338214e35 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 15:45:11 -0700
Subject: [PATCH 005/131] more fixes

---
 git_setup.py            | 13 +++++++------
 modules/gitinterface.py |  1 +
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 319e68cdf..82c044cff 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -66,14 +66,15 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
     print(f"Successfully checked out {name}")
 
 def submodule_checkout(name, url, path, tag, esmrequired):
-    git = GitInterface(os.cwd())
+    git = GitInterface(os.getcwd())
     topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
     repodir = os.path.join(topdir, path)
-    git.git_operation("submodule","update","--init")
+    git.git_operation("submodule","update","--init", name)
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
         read_gitmodules_file(repodir, esmrequired)
+    print(f"Successfully checked out {name}")
     return
     
 def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
@@ -98,11 +99,11 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
                 # TODO change to logging
                 #                logging.info(f"Skipping optional component {section}")
             continue
+        if "esmtag" in submodule_desc:
+            tag = submodule_desc["esmtag"]
+        else:
+            tag = "master"
         if "esmsparse" in submodule_desc:
-            if "esmtag" in submodule_desc:
-                tag = submodule_desc["esmtag"]
-            else:
-                tag = "master"
             submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
                                       submodule_desc["esmsparse"], tag)
             continue
diff --git a/modules/gitinterface.py b/modules/gitinterface.py
index 06203bb9d..baf98f35d 100644
--- a/modules/gitinterface.py
+++ b/modules/gitinterface.py
@@ -40,6 +40,7 @@ def _init_git_repo(self):
     def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *args)
         if isinstance(command, list):
+            print(command)
             return utils.execute_subprocess(command, output_to_caller=True)
         else:
             return command

From a3fcaf20dad5a7bcbd5c4e3dc0ac916911172494 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 15:59:04 -0700
Subject: [PATCH 006/131] submodule needs path not name

---
 git_setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/git_setup.py b/git_setup.py
index 82c044cff..79ee8f551 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -69,10 +69,11 @@ def submodule_checkout(name, url, path, tag, esmrequired):
     git = GitInterface(os.getcwd())
     topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
     repodir = os.path.join(topdir, path)
-    git.git_operation("submodule","update","--init", name)
+    git.git_operation("submodule","update","--init", path)
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
+        print(
         read_gitmodules_file(repodir, esmrequired)
     print(f"Successfully checked out {name}")
     return

From eed05bd0e63cd08b9d8d6d388cdfc65764ba8204 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:06:35 -0700
Subject: [PATCH 007/131] improve error checking

---
 git_setup.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 79ee8f551..fc3616c26 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -73,9 +73,11 @@ def submodule_checkout(name, url, path, tag, esmrequired):
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
-        print(
         read_gitmodules_file(repodir, esmrequired)
-    print(f"Successfully checked out {name}")
+    if os.path.exists(os.path.join(repodir,".git")):
+        print(f"Successfully checked out {name}")
+    else:
+        utils.fatal_error(f"Failed to checkout {name}")
     return
     
 def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):

From 20431710b064b0cf4876de54065a38d962facb23 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:14:05 -0700
Subject: [PATCH 008/131] add more logging

---
 git_setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/git_setup.py b/git_setup.py
index fc3616c26..daf4181df 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -73,6 +73,7 @@ def submodule_checkout(name, url, path, tag, esmrequired):
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
+        print(f"Recursively checking out submodules of {name}")
         read_gitmodules_file(repodir, esmrequired)
     if os.path.exists(os.path.join(repodir,".git")):
         print(f"Successfully checked out {name}")

From 9c35b88bcf355d48a262e82d04daba36ed4071aa Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:34:46 -0700
Subject: [PATCH 009/131] submodule always usees shell

---
 modules/gitinterface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/gitinterface.py b/modules/gitinterface.py
index baf98f35d..2fc7a131b 100644
--- a/modules/gitinterface.py
+++ b/modules/gitinterface.py
@@ -24,7 +24,7 @@ def __init__(self, repo_path):
                 
     def _git_command(self, operation, *args):
         logging.info(operation)
-        if self._use_module:
+        if self._use_module and operation != "submodule":
             return getattr(self.repo.git, operation)(*args)
         else:
             return ["git", "-C",self.repo_path, operation] + list(args)

From 8c5dfcdc0e7eba4cf369e15e40f19a68de42f6a5 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:42:31 -0700
Subject: [PATCH 010/131] fix toplevel

---
 git_setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/git_setup.py b/git_setup.py
index daf4181df..c22fa1355 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -67,7 +67,8 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
 
 def submodule_checkout(name, url, path, tag, esmrequired):
     git = GitInterface(os.getcwd())
-    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
+    topdir = os.getcwd()
+#    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
     repodir = os.path.join(topdir, path)
     git.git_operation("submodule","update","--init", path)
     # Look for a .gitmodules file in the newly checkedout repo

From d431e3b7cc23672a7cd081c795ee0607e17beb3b Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:44:33 -0700
Subject: [PATCH 011/131] fix toplevel

---
 git_setup.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index c22fa1355..5355b2106 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -65,11 +65,10 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
     sprepo_git.git_operation( "checkout", tag)
     print(f"Successfully checked out {name}")
 
-def submodule_checkout(name, url, path, tag, esmrequired):
+def submodule_checkout(root, name, url, path, tag, esmrequired):
     git = GitInterface(os.getcwd())
-    topdir = os.getcwd()
 #    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
-    repodir = os.path.join(topdir, path)
+    repodir = os.path.join(root, path)
     git.git_operation("submodule","update","--init", path)
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
@@ -116,7 +115,7 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
         for setting in esmrequired:
             if setting.startswith("I:"):
                 Iesmrequired.append(setting) 
-        submodule_checkout(name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
+        submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
 
 
 

From b6ee263ce6ee009f40288275866f45b471b8b6c8 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 4 Jan 2024 16:48:32 -0700
Subject: [PATCH 012/131] try again

---
 git_setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git_setup.py b/git_setup.py
index 5355b2106..3be8e3dfc 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -73,7 +73,7 @@ def submodule_checkout(root, name, url, path, tag, esmrequired):
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
-        print(f"Recursively checking out submodules of {name}")
+        print(f"Recursively checking out submodules of {name} {repodir}")
         read_gitmodules_file(repodir, esmrequired)
     if os.path.exists(os.path.join(repodir,".git")):
         print(f"Successfully checked out {name}")

From 996dfb6323dfd25e93c3a55ed1efb18d1f0b148a Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 5 Jan 2024 09:04:48 -0700
Subject: [PATCH 013/131] add command line options and support for update

---
 git_setup.py | 190 ++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 159 insertions(+), 31 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 3be8e3dfc..8361fe9a8 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -1,12 +1,119 @@
 #!/usr/bin/env python
+import sys
 import os
 import shutil
 import logging
+import argparse
 from modules import utils
 from configparser import ConfigParser
 from modules.lstripreader import LstripReader
 from modules.gitinterface import GitInterface
 
+from contextlib import contextmanager
+
+
+@contextmanager
+def pushd(new_dir):
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+def commandline_arguments(args=None):
+    description = '''
+    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
+    '''
+    parser = argparse.ArgumentParser(
+        description=description, 
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    #
+    # user options
+    #
+    parser.add_argument("components", nargs="*",
+                        help="Specific component(s) to checkout. By default, "
+                        "all required submodules are checked out.")
+
+    parser.add_argument('-C', '--path', default=os.getcwd(),
+                        help='Toplevel repository directory.  Defaults to current directory.')
+
+    parser.add_argument('-x', '--exclude', nargs='*',
+                        help='Component(s) listed in the gitmodules file which should be ignored.')
+
+    parser.add_argument('-o', '--optional', action='store_true', default=False,
+                        help='By default only the required submodules '
+                        'are checked out. This flag will also checkout the '
+                        'optional submodules relative to the toplevel directory.')
+
+    parser.add_argument('-S', '--status', action='store_true', default=False,
+                        help='Output the status of the repositories managed by '
+                        '%(prog)s. By default only summary information '
+                        'is provided. Use the verbose option to see details.')
+
+    parser.add_argument('-u', '--update', action='store_true', default=False,
+                        help='Update submodules to the tags defined in .gitmodules.')
+
+    parser.add_argument('-v', '--verbose', action='count', default=0,
+                        help='Output additional information to '
+                        'the screen and log file. This flag can be '
+                        'used up to two times, increasing the '
+                        'verbosity level each time.')
+
+    parser.add_argument('-V', '--version', action='store_true', default=False,
+                        help='Print manage_externals version and exit.')
+
+    #
+    # developer options
+    #
+    parser.add_argument('--backtrace', action='store_true',
+                        help='DEVELOPER: show exception backtraces as extra '
+                        'debugging output')
+
+    parser.add_argument('-d', '--debug', action='store_true', default=False,
+                        help='DEVELOPER: output additional debugging '
+                        'information to the screen and log file.')
+
+    logging_group = parser.add_mutually_exclusive_group()
+
+    logging_group.add_argument('--logging', dest='do_logging',
+                               action='store_true',
+                               help='DEVELOPER: enable logging.')
+    logging_group.add_argument('--no-logging', dest='do_logging',
+                               action='store_false', default=False,
+                               help='DEVELOPER: disable logging '
+                               '(this is the default)')
+    if args:
+        options = parser.parse_args(args)
+    else:
+        options = parser.parse_args()
+
+    if options.optional:
+        esmrequired = 'T:'
+    else:
+        esmrequired = 'T:T'
+        
+    if options.status:
+        action = 'status'
+    elif options.update:
+        action = 'update'
+    else:
+        action = 'install'
+
+    if options.version:
+        version_info = ''
+        version_file_path = os.path.join(os.path.dirname(__file__),'version.txt')
+        with open(version_file_path) as f:
+            version_info = f.readlines()[0].strip()
+        print(version_info)
+        sys.exit(0)
+
+
+        
+    return options.rootdir, esmrequired, options.components, options.exclude, options.verbose, action
+
+        
 def parse_submodules_desc_section(section, section_items):
     """Create a dict for this submodule description"""
     desc = {}
@@ -66,8 +173,7 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
     print(f"Successfully checked out {name}")
 
 def submodule_checkout(root, name, url, path, tag, esmrequired):
-    git = GitInterface(os.getcwd())
-#    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
+    git = GitInterface(root)
     repodir = os.path.join(root, path)
     git.git_operation("submodule","update","--init", path)
     # Look for a .gitmodules file in the newly checkedout repo
@@ -80,15 +186,26 @@ def submodule_checkout(root, name, url, path, tag, esmrequired):
     else:
         utils.fatal_error(f"Failed to checkout {name}")
     return
-    
-def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
+
+def submodule_update(root_dir, url, tag):
+    with pushd(root_dir):
+        git = GitInterface(root_dir)
+        # first make sure the url is correct
+        upstream = git.git_operation("ls-remote","--git-url")
+        if upstream != url:
+            # TODO - this needs to be a unique name
+            git.git_operation("remote","add","newbranch",url)
+        git.git_operation("checkout", tag)
+
+
+def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", gitmodulelist=None, action='install'):
     root_dir = os.path.abspath(root_dir)
 
     msg = 'In directory : {0}'.format(root_dir)
     logging.info(msg)
 
     file_path = os.path.join(root_dir, file_name)
-    if not os.path.exists(file_name):
+    if not os.path.exists(file_path):
         msg = ('ERROR: submodules description file, "{0}", does not '
                'exist in dir:\n    {1}'.format(file_name, root_dir))
         utils.fatal_error(msg)
@@ -97,31 +214,42 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules"):
     for section in config.sections():
         name = section[11:-1]
         submodule_desc = parse_submodules_desc_section(section,config.items(section))
-        if submodule_desc["esmrequired"] not in esmrequired:
-            if "T:F" in esmrequired or submodule_desc["esmrequired"].startswith("I:"):
-                print(f"Skipping optional component {section}")
-                # TODO change to logging
-                #                logging.info(f"Skipping optional component {section}")
-            continue
-        if "esmtag" in submodule_desc:
-            tag = submodule_desc["esmtag"]
-        else:
-            tag = "master"
-        if "esmsparse" in submodule_desc:
-            submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
-                                      submodule_desc["esmsparse"], tag)
-            continue
-        Iesmrequired = []
-        for setting in esmrequired:
-            if setting.startswith("I:"):
-                Iesmrequired.append(setting) 
-        submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
-
-
-
-
-esmrequired = ("I:T", "T:T")
-root_dir = os.getcwd()
-gitmodules = read_gitmodules_file(root_dir, esmrequired)
 
+        if action == 'install':
+            # Recursively install submodules, honering esm tags in .gitmodules
+            if submodule_desc["esmrequired"] not in esmrequired:
+                if "T:F" in esmrequired or submodule_desc["esmrequired"].startswith("I:"):
+                    print(f"Skipping optional component {section}")
+                    # TODO change to logging
+                    #                logging.info(f"Skipping optional component {section}")
+                    continue
+            if "esmtag" in submodule_desc:
+                tag = submodule_desc["esmtag"]
+            else:
+                tag = "master"
+            if "esmsparse" in submodule_desc:
+                submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
+                                          submodule_desc["esmsparse"], tag)
+                continue
+            Iesmrequired = []
+            for setting in esmrequired:
+                if setting.startswith("I:"):
+                    Iesmrequired.append(setting) 
+                submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
+
+        if action == 'update':
+            # update the submodules to the tags defined in .gitmodules
+            if "esmtag" in submodule_desc:
+                submod_dir = os.path.join(root_dir,submodule_desc['path'])
+                if os.path.exists(os.path.join(submod_dir,".git")):
+                    submodule_update(submod_dir, submodule_desc['url'], submodule_desc["esmtag"])
+
+                
+                
+
+if __name__ == '__main__':
+    root_dir, esmrequired, includelist, excludelist, verbose, action = commandline_arguments()
+    esmrequired = ("I:T", "T:T")
+    root_dir = os.getcwd()
+    read_gitmodules_file(root_dir, esmrequired, gitmodulelist, action, verbose)
     

From 647a8deea48c13ad9dbbb55fbcf294ff6eec6546 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 5 Jan 2024 10:23:07 -0700
Subject: [PATCH 014/131] fix new issues

---
 git_setup.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 8361fe9a8..82100a09c 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -39,6 +39,11 @@ def commandline_arguments(args=None):
     parser.add_argument('-C', '--path', default=os.getcwd(),
                         help='Toplevel repository directory.  Defaults to current directory.')
 
+    parser.add_argument('-g', '--gitmodules', nargs='?',
+                        default='.gitmodules',
+                        help='The submodule description filename. '
+                        'Default: %(default)s.')
+
     parser.add_argument('-x', '--exclude', nargs='*',
                         help='Component(s) listed in the gitmodules file which should be ignored.')
 
@@ -109,9 +114,7 @@ def commandline_arguments(args=None):
         print(version_info)
         sys.exit(0)
 
-
-        
-    return options.rootdir, esmrequired, options.components, options.exclude, options.verbose, action
+    return options.path, options.gitmodules, esmrequired, options.components, options.exclude, options.verbose, action
 
         
 def parse_submodules_desc_section(section, section_items):
@@ -191,14 +194,15 @@ def submodule_update(root_dir, url, tag):
     with pushd(root_dir):
         git = GitInterface(root_dir)
         # first make sure the url is correct
-        upstream = git.git_operation("ls-remote","--git-url")
+        upstream = git.git_operation("ls-remote","--get-url").rstrip()
+        print(f"Here {upstream} and {url}")
         if upstream != url:
             # TODO - this needs to be a unique name
             git.git_operation("remote","add","newbranch",url)
         git.git_operation("checkout", tag)
 
 
-def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", gitmodulelist=None, action='install'):
+def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", includelist=None, excludelist=None, action='install'):
     root_dir = os.path.abspath(root_dir)
 
     msg = 'In directory : {0}'.format(root_dir)
@@ -213,6 +217,10 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", gitmodu
     config.read_file(LstripReader(file_path), source=file_name)
     for section in config.sections():
         name = section[11:-1]
+        if includelist and name not in includelist:
+            continue
+        if excludelist and name in excludelist:
+            continue
         submodule_desc = parse_submodules_desc_section(section,config.items(section))
 
         if action == 'install':
@@ -248,8 +256,6 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", gitmodu
                 
 
 if __name__ == '__main__':
-    root_dir, esmrequired, includelist, excludelist, verbose, action = commandline_arguments()
-    esmrequired = ("I:T", "T:T")
-    root_dir = os.getcwd()
-    read_gitmodules_file(root_dir, esmrequired, gitmodulelist, action, verbose)
+    root_dir, file_name, esmrequired, includelist, excludelist, verbose, action = commandline_arguments()
+    read_gitmodules_file(root_dir, file_name=file_name, esmrequired=esmrequired, includelist=includelist, excludelist=excludelist, action=action)
     

From fbb8eaf70bd134f8ec5de662294f033c1d18b8ed Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 5 Jan 2024 10:44:48 -0700
Subject: [PATCH 015/131] update esmrequired variable

---
 git_setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index 82100a09c..baacdb3d7 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -95,9 +95,9 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
 
     if options.optional:
-        esmrequired = 'T:'
+        esmrequired = ('T:T','T:F','I:T')
     else:
-        esmrequired = 'T:T'
+        esmrequired = ('T:T','I:T')
         
     if options.status:
         action = 'status'

From e22d95b589470d1d4f93970951b555b1fdfa7462 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 5 Jan 2024 11:34:38 -0700
Subject: [PATCH 016/131] fix recursive issue

---
 git_setup.py            | 19 +++++++------------
 modules/gitinterface.py |  1 -
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index baacdb3d7..d8b027534 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -183,7 +183,7 @@ def submodule_checkout(root, name, url, path, tag, esmrequired):
     if os.path.exists(os.path.join(repodir,".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir}")
-        read_gitmodules_file(repodir, esmrequired)
+        read_gitmodules_file(repodir, ("I:T"))
     if os.path.exists(os.path.join(repodir,".git")):
         print(f"Successfully checked out {name}")
     else:
@@ -195,7 +195,6 @@ def submodule_update(root_dir, url, tag):
         git = GitInterface(root_dir)
         # first make sure the url is correct
         upstream = git.git_operation("ls-remote","--get-url").rstrip()
-        print(f"Here {upstream} and {url}")
         if upstream != url:
             # TODO - this needs to be a unique name
             git.git_operation("remote","add","newbranch",url)
@@ -226,11 +225,10 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", include
         if action == 'install':
             # Recursively install submodules, honering esm tags in .gitmodules
             if submodule_desc["esmrequired"] not in esmrequired:
-                if "T:F" in esmrequired or submodule_desc["esmrequired"].startswith("I:"):
-                    print(f"Skipping optional component {section}")
-                    # TODO change to logging
-                    #                logging.info(f"Skipping optional component {section}")
-                    continue
+                print(f"Skipping optional component {section}")
+                # TODO change to logging
+                #                logging.info(f"Skipping optional component {section}")
+                continue
             if "esmtag" in submodule_desc:
                 tag = submodule_desc["esmtag"]
             else:
@@ -239,11 +237,8 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", include
                 submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
                                           submodule_desc["esmsparse"], tag)
                 continue
-            Iesmrequired = []
-            for setting in esmrequired:
-                if setting.startswith("I:"):
-                    Iesmrequired.append(setting) 
-                submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, Iesmrequired)
+            
+            submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, esmrequired)
 
         if action == 'update':
             # update the submodules to the tags defined in .gitmodules
diff --git a/modules/gitinterface.py b/modules/gitinterface.py
index 2fc7a131b..a4e252925 100644
--- a/modules/gitinterface.py
+++ b/modules/gitinterface.py
@@ -40,7 +40,6 @@ def _init_git_repo(self):
     def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *args)
         if isinstance(command, list):
-            print(command)
             return utils.execute_subprocess(command, output_to_caller=True)
         else:
             return command

From 99d5439757f52b01fa0ae9f0924c472b255083eb Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 5 Jan 2024 13:52:33 -0700
Subject: [PATCH 017/131] add status flag support

---
 git_setup.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/git_setup.py b/git_setup.py
index d8b027534..f5334faef 100644
--- a/git_setup.py
+++ b/git_setup.py
@@ -200,7 +200,17 @@ def submodule_update(root_dir, url, tag):
             git.git_operation("remote","add","newbranch",url)
         git.git_operation("checkout", tag)
 
-
+def submodule_status(root_dir, name, url, path, tag):
+    with pushd(path):
+        git = GitInterface(os.path.join(root_dir,path))
+        atag = git.git_operation("describe","--tags","--always").rstrip()
+        if tag and atag != tag:
+            print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
+        elif tag:
+            print(f"Submodule {name} at tag {tag}")
+        else:
+            print(f"Submodule {name} has no tag defined in .gitmodules")
+            
 def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", includelist=None, excludelist=None, action='install'):
     root_dir = os.path.abspath(root_dir)
 
@@ -222,10 +232,19 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", include
             continue
         submodule_desc = parse_submodules_desc_section(section,config.items(section))
 
+        if action == 'status':
+            if "esmtag" in submodule_desc:
+                tag = submodule_desc["esmtag"]
+            else:
+                tag = None
+            submodule_status(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag)
+        
+        
         if action == 'install':
             # Recursively install submodules, honering esm tags in .gitmodules
             if submodule_desc["esmrequired"] not in esmrequired:
-                print(f"Skipping optional component {section}")
+                if 'T:F' in submodule_desc["esmrequired"]:
+                    print(f"Skipping optional component {section}")
                 # TODO change to logging
                 #                logging.info(f"Skipping optional component {section}")
                 continue
@@ -252,5 +271,6 @@ def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", include
 
 if __name__ == '__main__':
     root_dir, file_name, esmrequired, includelist, excludelist, verbose, action = commandline_arguments()
+    print(f"action is {action}")
     read_gitmodules_file(root_dir, file_name=file_name, esmrequired=esmrequired, includelist=includelist, excludelist=excludelist, action=action)
     

From 671bb1df07b611db05947f63572e38c2f6286a9a Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 09:28:06 -0700
Subject: [PATCH 018/131] add a gitmodules class

---
 git-fleximod.py       | 302 ++++++++++++++++++++++++++++++++++++++++++
 git_setup             |  94 -------------
 git_setup.py          | 276 --------------------------------------
 git_sparse_checkout   |  75 -----------
 modules/gitmodules.py |  53 ++++++++
 modules/utils.py      | 137 +++++++++++--------
 show-tags             |  20 ---
 7 files changed, 437 insertions(+), 520 deletions(-)
 create mode 100644 git-fleximod.py
 delete mode 100755 git_setup
 delete mode 100644 git_setup.py
 delete mode 100755 git_sparse_checkout
 create mode 100644 modules/gitmodules.py
 delete mode 100755 show-tags

diff --git a/git-fleximod.py b/git-fleximod.py
new file mode 100644
index 000000000..ded13eb9d
--- /dev/null
+++ b/git-fleximod.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+import sys
+import os
+import shutil
+import logging
+import argparse
+from modules import utils
+from modules.gitinterface import GitInterface
+from modules.gitmodules import GitModules
+
+
+def commandline_arguments(args=None):
+    description = """
+    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
+    """
+    parser = argparse.ArgumentParser(
+        description=description, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    #
+    # user options
+    #
+    parser.add_argument(
+        "components",
+        nargs="*",
+        help="Specific component(s) to checkout. By default, "
+        "all required submodules are checked out.",
+    )
+
+    parser.add_argument(
+        "-C",
+        "--path",
+        default=os.getcwd(),
+        help="Toplevel repository directory.  Defaults to current directory.",
+    )
+
+    parser.add_argument(
+        "-g",
+        "--gitmodules",
+        nargs="?",
+        default=".gitmodules",
+        help="The submodule description filename. " "Default: %(default)s.",
+    )
+
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        nargs="*",
+        help="Component(s) listed in the gitmodules file which should be ignored.",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--optional",
+        action="store_true",
+        default=False,
+        help="By default only the required submodules "
+        "are checked out. This flag will also checkout the "
+        "optional submodules relative to the toplevel directory.",
+    )
+
+    parser.add_argument(
+        "-S",
+        "--status",
+        action="store_true",
+        default=False,
+        help="Output the status of the repositories managed by "
+        "%(prog)s. By default only summary information "
+        "is provided. Use the verbose option to see details.",
+    )
+
+    parser.add_argument(
+        "-u",
+        "--update",
+        action="store_true",
+        default=False,
+        help="Update submodules to the tags defined in .gitmodules.",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Output additional information to "
+        "the screen and log file. This flag can be "
+        "used up to two times, increasing the "
+        "verbosity level each time.",
+    )
+
+    parser.add_argument(
+        "-V",
+        "--version",
+        action="store_true",
+        default=False,
+        help="Print manage_externals version and exit.",
+    )
+
+    #
+    # developer options
+    #
+    parser.add_argument(
+        "--backtrace",
+        action="store_true",
+        help="DEVELOPER: show exception backtraces as extra " "debugging output",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        default=False,
+        help="DEVELOPER: output additional debugging "
+        "information to the screen and log file.",
+    )
+
+    logging_group = parser.add_mutually_exclusive_group()
+
+    logging_group.add_argument(
+        "--logging",
+        dest="do_logging",
+        action="store_true",
+        help="DEVELOPER: enable logging.",
+    )
+    logging_group.add_argument(
+        "--no-logging",
+        dest="do_logging",
+        action="store_false",
+        default=False,
+        help="DEVELOPER: disable logging " "(this is the default)",
+    )
+    if args:
+        options = parser.parse_args(args)
+    else:
+        options = parser.parse_args()
+
+    if options.optional:
+        esmrequired = ("T:T", "T:F", "I:T")
+    else:
+        esmrequired = ("T:T", "I:T")
+
+    if options.status:
+        action = "status"
+    elif options.update:
+        action = "update"
+    else:
+        action = "install"
+
+    if options.version:
+        version_info = ""
+        version_file_path = os.path.join(os.path.dirname(__file__), "version.txt")
+        with open(version_file_path) as f:
+            version_info = f.readlines()[0].strip()
+        print(version_info)
+        sys.exit(0)
+
+    return (
+        options.path,
+        options.gitmodules,
+        esmrequired,
+        options.components,
+        options.exclude,
+        options.verbose,
+        action,
+    )
+
+
+def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
+    # first create the module directory
+    if not os.path.isdir(path):
+        os.makedirs(path)
+    # Check first if the module is already defined
+    # and the sparse-checkout file exists
+    git = GitInterface(os.getcwd())
+    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
+
+    topgit = os.path.join(topdir, ".git", "modules")
+    gitsparse = os.path.join(topgit, name, "info", "sparse-checkout")
+    if os.path.isfile(gitsparse):
+        logging.warning("submodule {} is already initialized".format(name))
+        return
+
+    # initialize a new git repo and set the sparse checkout flag
+    sprepo_git = GitInterface(os.path.join(topdir, path))
+    sprepo_git.config_set_value("core", "sparseCheckout", "true")
+
+    # set the repository remote
+    sprepo_git.git_operation("remote", "add", "origin", url)
+
+    if not os.path.isdir(topgit):
+        os.makedirs(topgit)
+    topgit = os.path.join(topgit, name)
+
+    shutil.move(os.path.join(path, ".git"), topgit)
+
+    shutil.copy(os.path.join(path, sparsefile), gitsparse)
+
+    with open(os.path.join(path, ".git"), "w") as f:
+        f.write("gitdir: " + os.path.relpath(topgit, path))
+
+    # Finally checkout the repo
+    sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
+    sprepo_git.git_operation("checkout", tag)
+    print(f"Successfully checked out {name}")
+
+
+def submodule_checkout(root, name, path):
+    git = GitInterface(root)
+    repodir = os.path.join(root, path)
+    git.git_operation("submodule", "update", "--init", path)
+    # Look for a .gitmodules file in the newly checkedout repo
+    if os.path.exists(os.path.join(repodir, ".gitmodules")):
+        # recursively handle this checkout
+        print(f"Recursively checking out submodules of {name} {repodir}")
+        gitmodules = GitModules(repodir)
+        submodules_install(gitmodules, repodir, ("I:T"))
+    if os.path.exists(os.path.join(repodir, ".git")):
+        print(f"Successfully checked out {name}")
+    else:
+        utils.fatal_error(f"Failed to checkout {name}")
+    return
+
+
+def submodules_status(gitmodules, root_dir):
+    for name in gitmodules.sections():
+        path = gitmodules.get(name, "path")
+        tag = gitmodules.git(name, "esmtag")
+        with utils.pushd(path):
+            git = GitInterface(os.path.join(root_dir, path))
+            atag = git.git_operation("describe", "--tags", "--always").rstrip()
+            if tag and atag != tag:
+                print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
+            elif tag:
+                print(f"Submodule {name} at tag {tag}")
+            else:
+                print(f"Submodule {name} has no tag defined in .gitmodules")
+
+
+def submodules_update(gitmodules, root_dir):
+    for name in gitmodules.sections():
+        esmtag = gitmodules.get(name, "esmtag")
+        path = gitmodules.get(name, "path")
+        url = gitmodules.get(name, "url")
+        if os.path.exists(os.path.join(path, ".git")):
+            with utils.pushd(root_dir):
+                git = GitInterface(root_dir)
+                # first make sure the url is correct
+                upstream = git.git_operation("ls-remote", "--get-url").rstrip()
+                if upstream != url:
+                    # TODO - this needs to be a unique name
+                    git.git_operation("remote", "add", "newbranch", url)
+                    git.git_operation("checkout", esmtag)
+
+
+def submodules_install(gitmodules, root_dir, requiredlist):
+    for name in gitmodules.sections():
+        esmrequired = gitmodules.get(name, "esmrequired")
+        esmsparse = gitmodules.get(name, "esmsparse")
+        esmtag = gitmodules.get(name, "esmtag")
+        path = gitmodules.get(name, "path")
+        url = gitmodules.get(name, "url")
+        if esmrequired not in requiredlist:
+            if "T:F" in esmrequired:
+                print("Skipping optional component {}".format(name))
+            continue
+        if esmsparse:
+            submodule_sparse_checkout(name, url, path, esmsparse, tag=esmtag)
+        else:
+            submodule_checkout(root_dir, name, path)
+
+
+def _main_func():
+    (
+        root_dir,
+        file_name,
+        esmrequired,
+        includelist,
+        excludelist,
+        verbose,
+        action,
+    ) = commandline_arguments()
+    if verbose:
+        print(f"action is {action}")
+    gitmodules = GitModules(
+        confpath=root_dir,
+        conffile=file_name,
+        includelist=includelist,
+        excludelist=excludelist,
+    )
+
+    if action == "update":
+        submodules_update(gitmodules, root_dir)
+    elif action == "install":
+        submodules_install(gitmodules, root_dir, esmrequired)
+    elif action == "status":
+        submodules_status(gitmodules, root_dir)
+    else:
+        utils.fatal_error(f"unrecognized action request {action}")
+
+
+if __name__ == "__main__":
+    _main_func()
diff --git a/git_setup b/git_setup
deleted file mode 100755
index a5b27b13f..000000000
--- a/git_setup
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-# Script to fetch submodules for esm applications 
-# if argument -internal-only is supplied then only checkout internal submodules, otherwise checkout everything needed for
-# a top level repository
-#
-set -e
-script=$0
-function usage {
-    echo -e "\nusage: $script [-i/--internal-only]\n"
-}
-# Set default arguments
-internal_only=0
-
-# Process arguments
-while [ "$1" != "" ];
-do
-    case $1 in
-
-        # Only checkout CISM internal submodules
-        -i | -internal-only | --internal-only)
-            internal_only=1
-            ;;
-
-        *)
-            echo "$script: illegal option $1"
-            usage
-            exit 1
-            ;;
-    esac
-    shift
-done
-
-########################
-declare -A submods
-submod_list=$(git config -f .gitmodules --list)
-
-while IFS= read -r line;
-do
-    readarray -d. -t strarr <<< "$line" 
-    if [[ "strarr[3]" == "path" ]]; then
-        echo "path is $strarr[4]"
-    fi
-done <<< "$submod_list"
-exit 0
-
-# Start with sparse checkouts
-if ! test -f bin/git_sparse_checkout; then
-  echo "Getting git_sparse_checkout script"
-  git clone https://github.com/jedwards4b/gitscripts --depth 1 bin
-fi
-
-./bin/git_sparse_checkout  https://github.com/larson-group/clubb_release clubb_4ncar_20221129_59cb19f_branch src/physics/clubb -- src/CLUBB_core/ src/SILHS/
-git restore src/physics/clubb
-
-./bin/git_sparse_checkout https://github.com/CFMIP/COSPv2.0 master src/physics/cosp2/src -- src/
-git restore src/physics/cosp2/src
-
-./bin/git_sparse_checkout https://github.com/MPAS-Dev/MPAS-Model.git develop src/dynamics/mpas/dycore -- src/external/ src/operators/ src/tools/ \
-                      src/core_atmosphere/ src/framework/
-git restore src/dynamics/mpas/dycore
-
-submodules=('chem_proc' 'src/physics/carma/base' 'src/physics/pumas' 'src/physics/pumas-frozen' 'src/physics/ali_arms' 'src/atmos_phys' 'src/dynamics/fv3/atmos_cubed_sphere' 'src/hemco')
-for mod in "${submodules[@]}"
-do
-   echo "Initializing $mod"
-   git submodule update --init $mod
-done
-
-if [ ${internal_only} -eq 1 ]
-  then
-    exit 0
-fi
-
-
-submodules=('ccs_config' 'components/cice5' 'components/cice' 'components/cmeps' \
-           'components/cdeps' 'components/cpl7' 'share' 'libraries/mct' \
-           'libraries/parallelio' 'cime' 'libraries/FMS' 'components/mosart' \
-           'components/rtm')
-
-for mod in "${submodules[@]}"
-do
-   echo "Initializing $mod"
-   git submodule update --init --recursive $mod
-done
-
-sparse_submodules=('components/clm' 'components/cism')
-for mod in "${sparse_submodules[@]}"
-do
-   echo "Initializing $mod"
-   git submodule update --init $mod
-   pushd $mod 1>/dev/null
-   bash ./install -internal-only
-   popd 1>/dev/null
-done
diff --git a/git_setup.py b/git_setup.py
deleted file mode 100644
index f5334faef..000000000
--- a/git_setup.py
+++ /dev/null
@@ -1,276 +0,0 @@
-#!/usr/bin/env python
-import sys
-import os
-import shutil
-import logging
-import argparse
-from modules import utils
-from configparser import ConfigParser
-from modules.lstripreader import LstripReader
-from modules.gitinterface import GitInterface
-
-from contextlib import contextmanager
-
-
-@contextmanager
-def pushd(new_dir):
-    previous_dir = os.getcwd()
-    os.chdir(new_dir)
-    try:
-        yield
-    finally:
-        os.chdir(previous_dir)
-
-def commandline_arguments(args=None):
-    description = '''
-    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
-    '''
-    parser = argparse.ArgumentParser(
-        description=description, 
-        formatter_class=argparse.RawDescriptionHelpFormatter)
-
-    #
-    # user options
-    #
-    parser.add_argument("components", nargs="*",
-                        help="Specific component(s) to checkout. By default, "
-                        "all required submodules are checked out.")
-
-    parser.add_argument('-C', '--path', default=os.getcwd(),
-                        help='Toplevel repository directory.  Defaults to current directory.')
-
-    parser.add_argument('-g', '--gitmodules', nargs='?',
-                        default='.gitmodules',
-                        help='The submodule description filename. '
-                        'Default: %(default)s.')
-
-    parser.add_argument('-x', '--exclude', nargs='*',
-                        help='Component(s) listed in the gitmodules file which should be ignored.')
-
-    parser.add_argument('-o', '--optional', action='store_true', default=False,
-                        help='By default only the required submodules '
-                        'are checked out. This flag will also checkout the '
-                        'optional submodules relative to the toplevel directory.')
-
-    parser.add_argument('-S', '--status', action='store_true', default=False,
-                        help='Output the status of the repositories managed by '
-                        '%(prog)s. By default only summary information '
-                        'is provided. Use the verbose option to see details.')
-
-    parser.add_argument('-u', '--update', action='store_true', default=False,
-                        help='Update submodules to the tags defined in .gitmodules.')
-
-    parser.add_argument('-v', '--verbose', action='count', default=0,
-                        help='Output additional information to '
-                        'the screen and log file. This flag can be '
-                        'used up to two times, increasing the '
-                        'verbosity level each time.')
-
-    parser.add_argument('-V', '--version', action='store_true', default=False,
-                        help='Print manage_externals version and exit.')
-
-    #
-    # developer options
-    #
-    parser.add_argument('--backtrace', action='store_true',
-                        help='DEVELOPER: show exception backtraces as extra '
-                        'debugging output')
-
-    parser.add_argument('-d', '--debug', action='store_true', default=False,
-                        help='DEVELOPER: output additional debugging '
-                        'information to the screen and log file.')
-
-    logging_group = parser.add_mutually_exclusive_group()
-
-    logging_group.add_argument('--logging', dest='do_logging',
-                               action='store_true',
-                               help='DEVELOPER: enable logging.')
-    logging_group.add_argument('--no-logging', dest='do_logging',
-                               action='store_false', default=False,
-                               help='DEVELOPER: disable logging '
-                               '(this is the default)')
-    if args:
-        options = parser.parse_args(args)
-    else:
-        options = parser.parse_args()
-
-    if options.optional:
-        esmrequired = ('T:T','T:F','I:T')
-    else:
-        esmrequired = ('T:T','I:T')
-        
-    if options.status:
-        action = 'status'
-    elif options.update:
-        action = 'update'
-    else:
-        action = 'install'
-
-    if options.version:
-        version_info = ''
-        version_file_path = os.path.join(os.path.dirname(__file__),'version.txt')
-        with open(version_file_path) as f:
-            version_info = f.readlines()[0].strip()
-        print(version_info)
-        sys.exit(0)
-
-    return options.path, options.gitmodules, esmrequired, options.components, options.exclude, options.verbose, action
-
-        
-def parse_submodules_desc_section(section, section_items):
-    """Create a dict for this submodule description"""
-    desc = {}
-    esmrequired_options = ("T:T", "I:T", "I:F", "T:F")
-    for item in section_items:
-        name = item[0].strip().lower()
-        desc[name] = item[1].strip()
-        # e3sm needs to have ssh protocol urls, we don't
-        if name == "url" and desc[name].startswith("git@github"):
-            desc[name] = desc[name].replace("git@github.com:","https://github.com/")
-    if not "esmrequired" in desc:
-        desc["esmrequired"] = "I:T"
-    
-    if desc["esmrequired"] not in esmrequired_options:
-        val = desc["esmrequired"]
-        utils.fatal_error(f"esmrequired set to {val} which is not a supported option {esmrequired_options}")
-    return desc
-
-
-def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
-    # first create the module directory
-    if not os.path.isdir(path):
-        os.makedirs(path)
-    # Check first if the module is already defined
-    # and the sparse-checkout file exists
-    git = GitInterface(os.getcwd())
-    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
-
-    topgit = os.path.join(topdir, ".git", "modules")
-    gitsparse = os.path.join(topgit, name, "info","sparse-checkout")
-    if os.path.isfile(gitsparse):
-        logging.warning(f"submodule {name} is already initialized")
-        return
-    
-    #initialize a new git repo and set the sparse checkout flag
-    sprepo_git = GitInterface(os.path.join(topdir,path))
-    sprepo_git.config_set_value("core", "sparseCheckout", "true")
-
-    # set the repository remote
-    cmd = ("remote", "add", "origin", url)
-    sprepo_git.git_operation("remote", "add", "origin", url)
-    
-    if not os.path.isdir(topgit):
-        os.makedirs(topgit)
-    topgit = os.path.join(topgit,name)
-    
-    shutil.move(os.path.join(path, ".git"), topgit)
-
-    shutil.copy(os.path.join(path,sparsefile), gitsparse)
-        
-    with open(os.path.join(path, ".git"), "w") as f:
-        f.write("gitdir: " + os.path.relpath(topgit, path))
-
-    #Finally checkout the repo
-    sprepo_git.git_operation( "fetch", "--depth=1", "origin", "--tags")
-    sprepo_git.git_operation( "checkout", tag)
-    print(f"Successfully checked out {name}")
-
-def submodule_checkout(root, name, url, path, tag, esmrequired):
-    git = GitInterface(root)
-    repodir = os.path.join(root, path)
-    git.git_operation("submodule","update","--init", path)
-    # Look for a .gitmodules file in the newly checkedout repo
-    if os.path.exists(os.path.join(repodir,".gitmodules")):
-        # recursively handle this checkout
-        print(f"Recursively checking out submodules of {name} {repodir}")
-        read_gitmodules_file(repodir, ("I:T"))
-    if os.path.exists(os.path.join(repodir,".git")):
-        print(f"Successfully checked out {name}")
-    else:
-        utils.fatal_error(f"Failed to checkout {name}")
-    return
-
-def submodule_update(root_dir, url, tag):
-    with pushd(root_dir):
-        git = GitInterface(root_dir)
-        # first make sure the url is correct
-        upstream = git.git_operation("ls-remote","--get-url").rstrip()
-        if upstream != url:
-            # TODO - this needs to be a unique name
-            git.git_operation("remote","add","newbranch",url)
-        git.git_operation("checkout", tag)
-
-def submodule_status(root_dir, name, url, path, tag):
-    with pushd(path):
-        git = GitInterface(os.path.join(root_dir,path))
-        atag = git.git_operation("describe","--tags","--always").rstrip()
-        if tag and atag != tag:
-            print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
-        elif tag:
-            print(f"Submodule {name} at tag {tag}")
-        else:
-            print(f"Submodule {name} has no tag defined in .gitmodules")
-            
-def read_gitmodules_file(root_dir, esmrequired, file_name=".gitmodules", includelist=None, excludelist=None, action='install'):
-    root_dir = os.path.abspath(root_dir)
-
-    msg = 'In directory : {0}'.format(root_dir)
-    logging.info(msg)
-
-    file_path = os.path.join(root_dir, file_name)
-    if not os.path.exists(file_path):
-        msg = ('ERROR: submodules description file, "{0}", does not '
-               'exist in dir:\n    {1}'.format(file_name, root_dir))
-        utils.fatal_error(msg)
-    config = ConfigParser()
-    config.read_file(LstripReader(file_path), source=file_name)
-    for section in config.sections():
-        name = section[11:-1]
-        if includelist and name not in includelist:
-            continue
-        if excludelist and name in excludelist:
-            continue
-        submodule_desc = parse_submodules_desc_section(section,config.items(section))
-
-        if action == 'status':
-            if "esmtag" in submodule_desc:
-                tag = submodule_desc["esmtag"]
-            else:
-                tag = None
-            submodule_status(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag)
-        
-        
-        if action == 'install':
-            # Recursively install submodules, honering esm tags in .gitmodules
-            if submodule_desc["esmrequired"] not in esmrequired:
-                if 'T:F' in submodule_desc["esmrequired"]:
-                    print(f"Skipping optional component {section}")
-                # TODO change to logging
-                #                logging.info(f"Skipping optional component {section}")
-                continue
-            if "esmtag" in submodule_desc:
-                tag = submodule_desc["esmtag"]
-            else:
-                tag = "master"
-            if "esmsparse" in submodule_desc:
-                submodule_sparse_checkout(name, submodule_desc["url"], submodule_desc["path"],
-                                          submodule_desc["esmsparse"], tag)
-                continue
-            
-            submodule_checkout(root_dir, name, submodule_desc["url"], submodule_desc["path"], tag, esmrequired)
-
-        if action == 'update':
-            # update the submodules to the tags defined in .gitmodules
-            if "esmtag" in submodule_desc:
-                submod_dir = os.path.join(root_dir,submodule_desc['path'])
-                if os.path.exists(os.path.join(submod_dir,".git")):
-                    submodule_update(submod_dir, submodule_desc['url'], submodule_desc["esmtag"])
-
-                
-                
-
-if __name__ == '__main__':
-    root_dir, file_name, esmrequired, includelist, excludelist, verbose, action = commandline_arguments()
-    print(f"action is {action}")
-    read_gitmodules_file(root_dir, file_name=file_name, esmrequired=esmrequired, includelist=includelist, excludelist=excludelist, action=action)
-    
diff --git a/git_sparse_checkout b/git_sparse_checkout
deleted file mode 100755
index 9b52bb29a..000000000
--- a/git_sparse_checkout
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/bash -eu
-
-# see also: git_origin and git_clone
-
-function git_sparse_checkout {
-    local self=$(readlink -f "${BASH_SOURCE[0]}")
-    local app=$(basename $self)
-    local usage=\
-"USAGE: ${app} repository-URL [branch] [project-directory] [[--] [list-of-files-or-directories]]
-  where:
-    'repository-URL' is a valid URL pointing to a Git repository.
-    'branch' is a branch, atag or a commit id. Default: master.
-    'project-directory' is a folder to be created and populated. Default: the project name.
-    'list-of-files-or-directories' is a list of file names or directories separated by spaces.
-  Examples:
-    ${app} http://github.com/frgomes/bash-scripts -- README.md
-    ${app} http://github.com/frgomes/bash-scripts develop -- README.md bin/
-    ${app} http://github.com/frgomes/bash-scripts develop tmpdir -- README.md bin/ docs/"
-
-    # obtain repository-URL, e.g.: http://github.com/frgomes/bash-scripts
-    [[ $# != 0 ]] || (echo "${usage}" 1>&2 ; return 1)
-    local arg=${1}
-    [[ "${arg}" != "--" ]] || (echo "${usage}" 1>&2 ; return 1)
-    local url="${arg}"
-    [[ $# == 0 ]] || shift
-
-    # obtain branch, which the default is master for historical reasons
-    if [[ "${arg}" != "--" ]] ;then arg="${1:-master}" ;fi
-    if [[ "${arg}" == "--" ]] ;then
-      local tag=master
-    else
-      local tag="${arg}"
-      [[ $# == 0 ]] || shift
-    fi
-
-    # obtain the project directory, which defaults to the repository name
-    local prj=$(echo "$url" | sed 's:/:\n:g' | tail -1)
-
-    if [[ "${arg}" != "--" ]] ;then arg="${1:-.}" ;fi
-    if [[ "${arg}" == "--" || "${arg}" == "." ]] ;then
-      local dir=$(readlink -f "./${prj}")
-    else
-      local dir=$(readlink -f "${arg}")
-      [[ $# == 0 ]] || shift
-    fi
-
-    if [[ "${arg}" == "--" ]] ;then [[ $# == 0 ]] || shift; fi
-    if [[ "${1:-}" == "--" ]] ;then [[ $# == 0 ]] || shift; fi
-
-    # Note: any remaining arguments after these above are considered as a
-    # list of files or directories to be downloaded. Names of directories
-    # must be followed by a slash /.
-
-    local sparse=true
-    local opts='--depth=1'
-
-    # now perform the sparse checkout
-    
-    mkdir -p "${dir}"
-    git -C "${dir}" init
-    git -C "${dir}" config core.sparseCheckout ${sparse}
-    git -C "${dir}" remote add origin ${url}
-    super=$(git rev-parse --show-toplevel)
-    pathtodotgit=$(realpath -m --relative-to=${dir} ${super})/.git
-    relpathfromsuper=$(realpath -m --relative-to=${super} ${dir})
-    mv ${dir}/.git  ${super}/.git/modules/${relpathfromsuper}
-    echo "gitdir: ${pathtodotgit}/modules/${relpathfromsuper}" > ${dir}/.git
-    for path in $* ;do
-        echo "${path}" >> ${super}/.git/modules/${relpathfromsuper}/info/sparse-checkout
-    done
-    git -C "${dir}" fetch ${opts} origin ${tag}
-    git -C "${dir}" checkout ${tag}
-}
-
-git_sparse_checkout $@
diff --git a/modules/gitmodules.py b/modules/gitmodules.py
new file mode 100644
index 000000000..3ef3ed87f
--- /dev/null
+++ b/modules/gitmodules.py
@@ -0,0 +1,53 @@
+import os
+from configparser import ConfigParser
+
+from modules.lstripreader import LstripReader
+
+
+class GitModules(ConfigParser.ConfigParser):
+    def __init__(
+        self,
+        confpath=os.getcwd(),
+        conffile=".gitmodules",
+        includelist=None,
+        excludelist=None,
+    ):
+        ConfigParser.ConfigParser.__init__(self)
+        self.read_file(LstripReader(confpath), source=conffile)
+        self.conf_file = os.path.join(confpath, conffile)
+        self.includelist = includelist
+        self.excludelist = excludelist
+
+    def set(self, name, option, value):
+        section = f'submodule "{name}"'
+        if not self.has_section(section):
+            self.add_section(section)
+        ConfigParser.ConfigParser.set(self, section, option, str(value))
+
+    def get(self, name, option):
+        section = f'submodule "{name}"'
+        try:
+            return ConfigParser.ConfigParser.get(self, section, option)
+        except ConfigParser.NoOptionError:
+            return None
+
+    def save(self):
+        self.write(open(self.conf_file, "w"))
+
+    def __del__(self):
+        self.save()
+
+    def sections(self):
+        names = []
+        for section in ConfigParser.ConfigParser.sections(self):
+            name = section[11:-1]
+            if self.includelist and name not in self.includelist:
+                continue
+            if self.excludelist and name in self.excludelist:
+                continue
+            names.append(name)
+        return names
+
+    def items(self, name):
+        section = f'submodule "{name}"'
+        return ConfigParser.ConfigParser.items(section)
diff --git a/modules/utils.py b/modules/utils.py
index 827153879..41c6bb25c 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -10,12 +10,24 @@
 import sys
 from threading import Timer
 
-LOCAL_PATH_INDICATOR = '.'
+LOCAL_PATH_INDICATOR = "."
 # ---------------------------------------------------------------------
 #
-# screen and logging output and functions to massage text for output
+# functions to massage text for output and other useful utilities
 #
 # ---------------------------------------------------------------------
+from contextlib import contextmanager
+
+
+@contextmanager
+def pushd(new_dir):
+    """context for chdir. usage: with pushd(new_dir)"""
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
 
 
 def log_process_output(output):
@@ -25,7 +37,7 @@ def log_process_output(output):
     line. This makes it hard to filter with grep.
 
     """
-    output = output.split('\n')
+    output = output.split("\n")
     for line in output:
         logging.debug(line)
 
@@ -63,9 +75,9 @@ def last_n_lines(the_string, n_lines, truncation_message=None):
         return_val = the_string
     else:
         lines_subset = lines[-n_lines:]
-        str_truncated = ''.join(lines_subset)
+        str_truncated = "".join(lines_subset)
         if truncation_message:
-            str_truncated = truncation_message + '\n' + str_truncated
+            str_truncated = truncation_message + "\n" + str_truncated
         return_val = str_truncated
 
     return return_val
@@ -85,9 +97,10 @@ def indent_string(the_string, indent_level):
     """
 
     lines = the_string.splitlines(True)
-    padding = ' ' * indent_level
+    padding = " " * indent_level
     lines_indented = [padding + line for line in lines]
-    return ''.join(lines_indented)
+    return "".join(lines_indented)
+
 
 # ---------------------------------------------------------------------
 #
@@ -116,24 +129,26 @@ def str_to_bool(bool_str):
     """
     value = None
     str_lower = bool_str.lower()
-    if str_lower in ('true', 't'):
+    if str_lower in ("true", "t"):
         value = True
-    elif str_lower in ('false', 'f'):
+    elif str_lower in ("false", "f"):
         value = False
     if value is None:
-        msg = ('ERROR: invalid boolean string value "{0}". '
-               'Must be "true" or "false"'.format(bool_str))
+        msg = (
+            'ERROR: invalid boolean string value "{0}". '
+            'Must be "true" or "false"'.format(bool_str)
+        )
         fatal_error(msg)
     return value
 
 
-REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@']
+REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"]
 
 
 def is_remote_url(url):
     """check if the user provided a local file path instead of a
-       remote. If so, it must be expanded to an absolute
-       path.
+    remote. If so, it must be expanded to an absolute
+    path.
 
     """
     remote_url = False
@@ -145,7 +160,7 @@ def is_remote_url(url):
 
 def split_remote_url(url):
     """check if the user provided a local file path or a
-       remote. If remote, try to strip off protocol info.
+    remote. If remote, try to strip off protocol info.
 
     """
     remote_url = is_remote_url(url)
@@ -153,13 +168,13 @@ def split_remote_url(url):
         return url
 
     for prefix in REMOTE_PREFIXES:
-        url = url.replace(prefix, '')
+        url = url.replace(prefix, "")
 
-    if '@' in url:
-        url = url.split('@')[1]
+    if "@" in url:
+        url = url.split("@")[1]
 
-    if ':' in url:
-        url = url.split(':')[1]
+    if ":" in url:
+        url = url.split(":")[1]
 
     return url
 
@@ -181,10 +196,12 @@ def expand_local_url(url, field):
             url = os.path.expandvars(url)
             url = os.path.expanduser(url)
             if not os.path.isabs(url):
-                msg = ('WARNING: Externals description for "{0}" contains a '
-                       'url that is not remote and does not expand to an '
-                       'absolute path. Version control operations may '
-                       'fail.\n\nurl={1}'.format(field, url))
+                msg = (
+                    'WARNING: Externals description for "{0}" contains a '
+                    "url that is not remote and does not expand to an "
+                    "absolute path. Version control operations may "
+                    "fail.\n\nurl={1}".format(field, url)
+                )
                 printlog(msg)
             else:
                 url = os.path.normpath(url)
@@ -203,7 +220,8 @@ def expand_local_url(url, field):
 
 
 def _hanging_msg(working_directory, command):
-    print("""
+    print(
+        """
 
 Command '{command}'
 from directory {working_directory}
@@ -217,13 +235,15 @@ def _hanging_msg(working_directory, command):
 will appear to hang. Ensure you can run svn and git manually and access
 all repositories without entering your authentication information.
 
-""".format(command=command,
-           working_directory=working_directory,
-           hanging_sec=_HANGING_SEC))
+""".format(
+            command=command,
+            working_directory=working_directory,
+            hanging_sec=_HANGING_SEC,
+        )
+    )
 
 
-def execute_subprocess(commands, status_to_caller=False,
-                       output_to_caller=False):
+def execute_subprocess(commands, status_to_caller=False, output_to_caller=False):
     """Wrapper around subprocess.check_output to handle common
     exceptions.
 
@@ -237,32 +257,35 @@ def execute_subprocess(commands, status_to_caller=False,
 
     """
     cwd = os.getcwd()
-    msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd)
+    msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd)
     logging.info(msg)
-    commands_str = ' '.join(commands)
+    commands_str = " ".join(commands)
     logging.info(commands_str)
     return_to_caller = status_to_caller or output_to_caller
     status = -1
-    output = ''
-    hanging_timer = Timer(_HANGING_SEC, _hanging_msg,
-                          kwargs={"working_directory": cwd,
-                                  "command": commands_str})
+    output = ""
+    hanging_timer = Timer(
+        _HANGING_SEC,
+        _hanging_msg,
+        kwargs={"working_directory": cwd, "command": commands_str},
+    )
     hanging_timer.start()
     try:
-        output = subprocess.check_output(commands, stderr=subprocess.STDOUT,
-                                         universal_newlines=True)
+        output = subprocess.check_output(
+            commands, stderr=subprocess.STDOUT, universal_newlines=True
+        )
         log_process_output(output)
         status = 0
     except OSError as error:
         msg = failed_command_msg(
-            'Command execution failed. Does the executable exist?',
-            commands)
+            "Command execution failed. Does the executable exist?", commands
+        )
         logging.error(error)
         fatal_error(msg)
     except ValueError as error:
         msg = failed_command_msg(
-            'DEV_ERROR: Invalid arguments trying to run subprocess',
-            commands)
+            "DEV_ERROR: Invalid arguments trying to run subprocess", commands
+        )
         logging.error(error)
         fatal_error(msg)
     except subprocess.CalledProcessError as error:
@@ -272,10 +295,11 @@ def execute_subprocess(commands, status_to_caller=False,
         # responsibility determine if an error occurred and handle it
         # appropriately.
         if not return_to_caller:
-            msg_context = ('Process did not run successfully; '
-                           'returned status {0}'.format(error.returncode))
-            msg = failed_command_msg(msg_context, commands,
-                                     output=error.output)
+            msg_context = (
+                "Process did not run successfully; "
+                "returned status {0}".format(error.returncode)
+            )
+            msg = failed_command_msg(msg_context, commands, output=error.output)
             logging.error(error)
             logging.error(msg)
             log_process_output(error.output)
@@ -304,22 +328,25 @@ def failed_command_msg(msg_context, command, output=None):
     """
 
     if output:
-        output_truncated = last_n_lines(output, 20,
-                                        truncation_message='[... Output truncated for brevity ...]')
-        errmsg = ('Failed with output:\n' +
-                  indent_string(output_truncated, 4) +
-                  '\nERROR: ')
+        output_truncated = last_n_lines(
+            output, 20, truncation_message="[... Output truncated for brevity ...]"
+        )
+        errmsg = (
+            "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: "
+        )
     else:
-        errmsg = ''
+        errmsg = ""
 
-    command_str = ' '.join(command)
+    command_str = " ".join(command)
     errmsg += """In directory
     {cwd}
 {context}:
     {command}
-""".format(cwd=os.getcwd(), context=msg_context, command=command_str)
+""".format(
+        cwd=os.getcwd(), context=msg_context, command=command_str
+    )
 
     if output:
-        errmsg += 'See above for output from failed command.\n'
+        errmsg += "See above for output from failed command.\n"
 
     return errmsg
diff --git a/show-tags b/show-tags
deleted file mode 100755
index 56c7d9c5e..000000000
--- a/show-tags
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-this_dir=$(pwd)
-printf "\nSubmodule status\n"
-printf "(currently checked out commit for each submodule)\n"
-printf "(when the submodule is initialized and a tag exists, the commit is shown as: 'most recent tag-commits since tag-commit hash')\n"
-printf "(when the submodule is not initialized, only the checked out commit is shown)\n\n"
-grep path .gitmodules | sed 's/.*= //' | while read x
-do
-  cd "$this_dir"
-  printf "$x\n   - current commit: "
-  if [ "$(ls -A $x)" ] ; then
-    cd "$x"
-    git describe --tags --always
-  else
-    git submodule status $x | sed 's/^-//' | awk '{ print $1 }'
-  fi
-done
-printf "\n"
-

From 815e6cd0d9f00fdf1cd06bd67ca912006c8d3a46 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 10:34:47 -0700
Subject: [PATCH 019/131] now renamed git-fleximod for git interface

---
 git-fleximod.py => git-fleximod | 24 ++++++++++++++++++------
 modules/gitmodules.py           | 21 ++++++++++++---------
 2 files changed, 30 insertions(+), 15 deletions(-)
 rename git-fleximod.py => git-fleximod (92%)
 mode change 100644 => 100755

diff --git a/git-fleximod.py b/git-fleximod
old mode 100644
new mode 100755
similarity index 92%
rename from git-fleximod.py
rename to git-fleximod
index ded13eb9d..00fc2446c
--- a/git-fleximod.py
+++ b/git-fleximod
@@ -8,6 +8,9 @@
 from modules.gitinterface import GitInterface
 from modules.gitmodules import GitModules
 
+logger = logging.getLogger(__name__)
+logger.propogate = False
+
 
 def commandline_arguments(args=None):
     description = """
@@ -135,9 +138,9 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
 
     if options.optional:
-        esmrequired = ("T:T", "T:F", "I:T")
+        esmrequired = ["T:T", "T:F", "I:T"]
     else:
-        esmrequired = ("T:T", "I:T")
+        esmrequired = ["T:T", "I:T"]
 
     if options.status:
         action = "status"
@@ -177,7 +180,7 @@ def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
     topgit = os.path.join(topdir, ".git", "modules")
     gitsparse = os.path.join(topgit, name, "info", "sparse-checkout")
     if os.path.isfile(gitsparse):
-        logging.warning("submodule {} is already initialized".format(name))
+        logger.warning("submodule {} is already initialized".format(name))
         return
 
     # initialize a new git repo and set the sparse checkout flag
@@ -213,7 +216,7 @@ def submodule_checkout(root, name, path):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir}")
         gitmodules = GitModules(repodir)
-        submodules_install(gitmodules, repodir, ("I:T"))
+        submodules_install(gitmodules, repodir, ["I:T"])
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
     else:
@@ -253,19 +256,27 @@ def submodules_update(gitmodules, root_dir):
 
 
 def submodules_install(gitmodules, root_dir, requiredlist):
+    print(gitmodules.sections())
     for name in gitmodules.sections():
         esmrequired = gitmodules.get(name, "esmrequired")
         esmsparse = gitmodules.get(name, "esmsparse")
         esmtag = gitmodules.get(name, "esmtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-        if esmrequired not in requiredlist:
-            if "T:F" in esmrequired:
+
+        if esmrequired and esmrequired not in requiredlist:
+            if "T:F" == esmrequired:
                 print("Skipping optional component {}".format(name))
             continue
         if esmsparse:
+            logger.debug(
+                f"Callng submodule_sparse_checkout({name} {url}, {path}, {esmsparse}, {esmtag}"
+            )
             submodule_sparse_checkout(name, url, path, esmsparse, tag=esmtag)
         else:
+            logger.debug(
+                "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
+            )
             submodule_checkout(root_dir, name, path)
 
 
@@ -281,6 +292,7 @@ def _main_func():
     ) = commandline_arguments()
     if verbose:
         print(f"action is {action}")
+
     gitmodules = GitModules(
         confpath=root_dir,
         conffile=file_name,
diff --git a/modules/gitmodules.py b/modules/gitmodules.py
index 3ef3ed87f..285862927 100644
--- a/modules/gitmodules.py
+++ b/modules/gitmodules.py
@@ -4,7 +4,7 @@
 from modules.lstripreader import LstripReader
 
 
-class GitModules(ConfigParser.ConfigParser):
+class GitModules(ConfigParser):
     def __init__(
         self,
         confpath=os.getcwd(),
@@ -12,9 +12,9 @@ def __init__(
         includelist=None,
         excludelist=None,
     ):
-        ConfigParser.ConfigParser.__init__(self)
-        self.read_file(LstripReader(confpath), source=conffile)
+        ConfigParser.__init__(self)
         self.conf_file = os.path.join(confpath, conffile)
+        self.read_file(LstripReader(self.conf_file), source=conffile)
         self.includelist = includelist
         self.excludelist = excludelist
 
@@ -22,12 +22,15 @@ def set(self, name, option, value):
         section = f'submodule "{name}"'
         if not self.has_section(section):
             self.add_section(section)
-        ConfigParser.ConfigParser.set(self, section, option, str(value))
+        ConfigParser.set(self, section, option, str(value))
 
-    def get(self, name, option):
+    # pylint: disable=redefined-builtin, arguments-differ
+    def get(self, name, option, raw=False, vars=None, fallback=None):
         section = f'submodule "{name}"'
         try:
-            return ConfigParser.ConfigParser.get(self, section, option)
+            return ConfigParser.get(
+                self, section, option, raw=raw, vars=vars, fallback=fallback
+            )
         except ConfigParser.NoOptionError:
             return None
 
@@ -39,7 +42,7 @@ def __del__(self):
 
     def sections(self):
         names = []
-        for section in ConfigParser.ConfigParser.sections(self):
+        for section in ConfigParser.sections(self):
             name = section[11:-1]
             if self.includelist and name not in self.includelist:
                 continue
@@ -48,6 +51,6 @@ def sections(self):
             names.append(name)
         return names
 
-    def items(self, name):
+    def items(self, name, raw=False, vars=None):
         section = f'submodule "{name}"'
-        return ConfigParser.ConfigParser.items(section)
+        return ConfigParser.items(section, raw=raw, vars=vars)

From a6ce960bc9d102007608ab7dde8b9fd4ad590dec Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 11:41:25 -0700
Subject: [PATCH 020/131] clean up and refactor, rename to git-fleximod

---
 git-fleximod     | 48 ++++++++++++++++++++++++++++++++++--------------
 modules/utils.py | 13 +++++++++++++
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 00fc2446c..44a86c50b 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -168,16 +168,14 @@ def commandline_arguments(args=None):
     )
 
 
-def submodule_sparse_checkout(name, url, path, sparsefile, tag="master"):
+def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"):
     # first create the module directory
     if not os.path.isdir(path):
         os.makedirs(path)
     # Check first if the module is already defined
     # and the sparse-checkout file exists
-    git = GitInterface(os.getcwd())
-    topdir = git.git_operation("rev-parse", "--show-toplevel").rstrip()
-
-    topgit = os.path.join(topdir, ".git", "modules")
+    git = GitInterface(root_dir)
+    topgit = os.path.join(root_dir, ".git", "modules")
     gitsparse = os.path.join(topgit, name, "info", "sparse-checkout")
     if os.path.isfile(gitsparse):
         logger.warning("submodule {} is already initialized".format(name))
@@ -227,7 +225,7 @@ def submodule_checkout(root, name, path):
 def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
-        tag = gitmodules.git(name, "esmtag")
+        tag = gitmodules.get(name, "esmtag")
         with utils.pushd(path):
             git = GitInterface(os.path.join(root_dir, path))
             atag = git.git_operation("describe", "--tags", "--always").rstrip()
@@ -236,7 +234,12 @@ def submodules_status(gitmodules, root_dir):
             elif tag:
                 print(f"Submodule {name} at tag {tag}")
             else:
-                print(f"Submodule {name} has no tag defined in .gitmodules")
+                print(
+                    f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
+                )
+            status = git.git_operation("status")
+            if "nothing to commit" not in status:
+                print(status)
 
 
 def submodules_update(gitmodules, root_dir):
@@ -245,18 +248,26 @@ def submodules_update(gitmodules, root_dir):
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         if os.path.exists(os.path.join(path, ".git")):
-            with utils.pushd(root_dir):
-                git = GitInterface(root_dir)
+            submoddir = os.path.join(root_dir, path)
+            with utils.pushd(submoddir):
+                git = GitInterface(submoddir)
                 # first make sure the url is correct
                 upstream = git.git_operation("ls-remote", "--get-url").rstrip()
+                newremote = "origin"
                 if upstream != url:
                     # TODO - this needs to be a unique name
-                    git.git_operation("remote", "add", "newbranch", url)
-                    git.git_operation("checkout", esmtag)
+                    remotes = git.git_operation("remote", "-v")
+                    newremote = "newremote"
+                    git.git_operation("remote", "add", newremote, url)
+
+                tags = git.git_operation("tag", "-l")
+                if esmtag not in tags:
+                    git.git_operation("fetch", newremote, "--tags")
+
+                git.git_operation("checkout", esmtag)
 
 
 def submodules_install(gitmodules, root_dir, requiredlist):
-    print(gitmodules.sections())
     for name in gitmodules.sections():
         esmrequired = gitmodules.get(name, "esmrequired")
         esmsparse = gitmodules.get(name, "esmsparse")
@@ -270,9 +281,9 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             continue
         if esmsparse:
             logger.debug(
-                f"Callng submodule_sparse_checkout({name} {url}, {path}, {esmsparse}, {esmtag}"
+                f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {esmsparse}, {esmtag}"
             )
-            submodule_sparse_checkout(name, url, path, esmsparse, tag=esmtag)
+            submodule_sparse_checkout(root_dir, name, url, path, esmsparse, tag=esmtag)
         else:
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
@@ -293,6 +304,15 @@ def _main_func():
     if verbose:
         print(f"action is {action}")
 
+    if not os.path.isfile(os.path.join(root_dir, file_name)):
+        file_path = utils.find_upwards(root_dir, file_name)
+
+        if file_path is None:
+            fatal_error(
+                "No {} found in {} or any of it's parents".format(file_name, root_dir)
+            )
+        root_dir = os.path.dirname(file_path)
+    print(f"root_dir is {root_dir}")
     gitmodules = GitModules(
         confpath=root_dir,
         conffile=file_name,
diff --git a/modules/utils.py b/modules/utils.py
index 41c6bb25c..f0753367e 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -9,6 +9,7 @@
 import subprocess
 import sys
 from threading import Timer
+from pathlib import Path
 
 LOCAL_PATH_INDICATOR = "."
 # ---------------------------------------------------------------------
@@ -55,6 +56,18 @@ def printlog(msg, **kwargs):
     sys.stdout.flush()
 
 
+def find_upwards(root_dir, filename):
+    """Find a file in root dir or any of it's parents"""
+    d = Path(root_dir)
+    root = Path(d.root)
+    while d != root:
+        attempt = d / filename
+        if attempt.exists():
+            return attempt
+        d = d.parent
+    return None
+
+
 def last_n_lines(the_string, n_lines, truncation_message=None):
     """Returns the last n lines of the given string
 

From 816158ceb02abca2b360811beba25a5c1b98c5e2 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 12:11:04 -0700
Subject: [PATCH 021/131] add version.txt and .pre-commit-config.yaml

---
 .pre-commit-config.yaml | 18 ++++++++++++++++++
 version.txt             |  1 +
 2 files changed, 19 insertions(+)
 create mode 100644 .pre-commit-config.yaml
 create mode 100644 version.txt

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..2f6089da7
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,18 @@
+exclude: ^utils/.*$
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/pylint
+    rev: v2.11.1
+    hooks:
+      - id: pylint
+        args:
+          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member
diff --git a/version.txt b/version.txt
new file mode 100644
index 000000000..a3dce6cd3
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+v0.0.2

From 3be6537e39f7d9209be9b66b36756da022455612 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 15:21:15 -0700
Subject: [PATCH 022/131] fix issue in update

---
 git-fleximod | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 44a86c50b..e5978cff6 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -257,11 +257,20 @@ def submodules_update(gitmodules, root_dir):
                 if upstream != url:
                     # TODO - this needs to be a unique name
                     remotes = git.git_operation("remote", "-v")
-                    newremote = "newremote"
-                    git.git_operation("remote", "add", newremote, url)
+                    if url in remotes:
+                        for line in remotes:
+                            if url in line and "fetch" in line:
+                                newremote = line.split()[0]
+                                break
+                    else:
+                        i = 0
+                        while newremote in remotes:
+                            i = i + 1
+                            newremote = f"newremote.{i:02d}"
+                        git.git_operation("remote", "add", newremote, url)
 
                 tags = git.git_operation("tag", "-l")
-                if esmtag not in tags:
+                if esmtag and esmtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
 
                 git.git_operation("checkout", esmtag)

From d709c124c284c1ad007caed586db93914d78be47 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 15:47:38 -0700
Subject: [PATCH 023/131] make action a required subargument to look more like
 a git subcommand

---
 git-fleximod          | 38 +++++++++++++-------------------------
 modules/gitmodules.py |  3 ---
 2 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index e5978cff6..d13f1b9f0 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -23,6 +23,14 @@ def commandline_arguments(args=None):
     #
     # user options
     #
+    choices = ["update", "install", "status"]
+    parser.add_argument(
+        "action",
+        choices=choices,
+        default="install",
+        help=f"Subcommand of fleximod, choices are {choices}",
+    )
+
     parser.add_argument(
         "components",
         nargs="*",
@@ -62,24 +70,6 @@ def commandline_arguments(args=None):
         "optional submodules relative to the toplevel directory.",
     )
 
-    parser.add_argument(
-        "-S",
-        "--status",
-        action="store_true",
-        default=False,
-        help="Output the status of the repositories managed by "
-        "%(prog)s. By default only summary information "
-        "is provided. Use the verbose option to see details.",
-    )
-
-    parser.add_argument(
-        "-u",
-        "--update",
-        action="store_true",
-        default=False,
-        help="Update submodules to the tags defined in .gitmodules.",
-    )
-
     parser.add_argument(
         "-v",
         "--verbose",
@@ -142,11 +132,8 @@ def commandline_arguments(args=None):
     else:
         esmrequired = ["T:T", "I:T"]
 
-    if options.status:
-        action = "status"
-    elif options.update:
-        action = "update"
-    else:
+    action = options.action
+    if not action:
         action = "install"
 
     if options.version:
@@ -226,8 +213,9 @@ def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "esmtag")
-        with utils.pushd(path):
-            git = GitInterface(os.path.join(root_dir, path))
+        newpath = os.path.join(root_dir, path)
+        with utils.pushd(newpath):
+            git = GitInterface(newpath)
             atag = git.git_operation("describe", "--tags", "--always").rstrip()
             if tag and atag != tag:
                 print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
diff --git a/modules/gitmodules.py b/modules/gitmodules.py
index 285862927..5af10be40 100644
--- a/modules/gitmodules.py
+++ b/modules/gitmodules.py
@@ -37,9 +37,6 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
     def save(self):
         self.write(open(self.conf_file, "w"))
 
-    def __del__(self):
-        self.save()
-
     def sections(self):
         names = []
         for section in ConfigParser.sections(self):

From 83aea04a4366e246e5a107e88ef29aa347657e24 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 6 Jan 2024 16:02:03 -0700
Subject: [PATCH 024/131] add documentation

---
 modules/gitmodules.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/gitmodules.py b/modules/gitmodules.py
index 5af10be40..9e38a5806 100644
--- a/modules/gitmodules.py
+++ b/modules/gitmodules.py
@@ -12,6 +12,12 @@ def __init__(
         includelist=None,
         excludelist=None,
     ):
+        """
+        confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory).
+        conffile: Name of the configuration file (defaults to .gitmodules).
+        includelist: Optional list of submodules to include.
+        excludelist: Optional list of submodules to exclude.
+        """
         ConfigParser.__init__(self)
         self.conf_file = os.path.join(confpath, conffile)
         self.read_file(LstripReader(self.conf_file), source=conffile)
@@ -19,6 +25,11 @@ def __init__(
         self.excludelist = excludelist
 
     def set(self, name, option, value):
+        """
+        Sets a configuration value for a specific submodule:
+        Ensures the appropriate section exists for the submodule.
+        Calls the parent class's set method to store the value.
+        """
         section = f'submodule "{name}"'
         if not self.has_section(section):
             self.add_section(section)
@@ -26,6 +37,11 @@ def set(self, name, option, value):
 
     # pylint: disable=redefined-builtin, arguments-differ
     def get(self, name, option, raw=False, vars=None, fallback=None):
+        """
+        Retrieves a configuration value for a specific submodule:
+        Uses the parent class's get method to access the value.
+        Handles potential errors if the section or option doesn't exist.
+        """
         section = f'submodule "{name}"'
         try:
             return ConfigParser.get(
@@ -38,6 +54,7 @@ def save(self):
         self.write(open(self.conf_file, "w"))
 
     def sections(self):
+        """Strip the submodule part out of section and just use the name"""
         names = []
         for section in ConfigParser.sections(self):
             name = section[11:-1]

From 905824aefcd187a25f4133826bd0a0a76edcfcdd Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 09:09:23 -0700
Subject: [PATCH 025/131] add README and License

---
 License   | 20 ++++++++++++++++++++
 README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 License
 create mode 100644 README.md

diff --git a/License b/License
new file mode 100644
index 000000000..2c6fe768c
--- /dev/null
+++ b/License
@@ -0,0 +1,20 @@
+Copyright 2024 National Center for Atmospheric Sciences (NCAR)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 000000000..4ee56bc4a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,53 @@
+# git-fleximod
+
+Flexible Submodule Management for Git
+
+## Overview
+
+Git-fleximod is a Python-based tool that extends Git's submodule capabilities, offering additional features for managing submodules in a more flexible and efficient way.
+
+## Installation
+
+    Install using pip:
+        pip install git-fleximod
+
+## Usage
+
+    Basic Usage:
+        git fleximod <command> [options]
+    Available Commands:
+        install: Install submodules according to configuration.
+        status: Display the status of submodules.
+        update: Update submodules to their latest commits.
+    Additional Options:
+        See git fleximod --help for more details.
+
+## Supported .gitmodules Variables
+
+    fxtag: Specify a specific tag or branch to checkout for a submodule.
+    fxrequired: Mark a submodule's checkout behavior, with allowed values:
+      - T:T: Top-level and required (checked out only when this is the Toplevel module).
+      - T:F: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
+      - I:T: Internal and required (always checked out).
+      - I:F: Internal and optional (checked out with --optional flag).
+    fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
+
+## Sparse Checkouts
+
+    To enable sparse checkout for a submodule, set the fxsparse variable
+    in the .gitmodules file to the path of a file containing the desired
+    sparse checkout paths. Git-fleximod will automatically configure
+    sparse checkout based on this file when applicable commands are run.
+
+## Examples
+
+    Installing submodules with optional ones: git fleximod install --optional
+    Checking out a specific tag for a submodule: git fleximod update --fxtag=v1.2.3 submodule-name
+
+## Contributing
+
+We welcome contributions! Please see the CONTRIBUTING.md file for guidelines.
+
+## License
+
+Git-fleximod is released under the MIT License.

From 3c723a86b39db2274c223e4a8ffb1d83f5b01ec1 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 10:56:42 -0700
Subject: [PATCH 026/131] update README

---
 README.md    | 34 +++++++++++++++++++++++++++++-----
 git-fleximod | 39 +++++++++++++++++++++------------------
 2 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 4ee56bc4a..da725ab5f 100644
--- a/README.md
+++ b/README.md
@@ -8,8 +8,9 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 
 ## Installation
 
-    Install using pip:
-        pip install git-fleximod
+#TODO    Install using pip:
+#        pip install git-fleximod
+  If you choose to locate git-fleximod in your path you can access it via command: git fleximod
 
 ## Usage
 
@@ -18,7 +19,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
     Available Commands:
         install: Install submodules according to configuration.
         status: Display the status of submodules.
-        update: Update submodules to their latest commits.
+        update: Update submodules to the tag indicated in .gitmodules variable fxtag.
     Additional Options:
         See git fleximod --help for more details.
 
@@ -38,11 +39,34 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
     in the .gitmodules file to the path of a file containing the desired
     sparse checkout paths. Git-fleximod will automatically configure
     sparse checkout based on this file when applicable commands are run.
+    See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) for details on the format of this file.
 
 ## Examples
 
-    Installing submodules with optional ones: git fleximod install --optional
-    Checking out a specific tag for a submodule: git fleximod update --fxtag=v1.2.3 submodule-name
+    Installing submodules including optional ones: git fleximod install --optional
+    Update a specific submodule to the fxtag indicated in .gitmodules: git fleximod update submodule-name
+    Example .gitmodules entry:
+      [submodule "cosp2"]
+         path = src/physics/cosp2/src
+         url = https://github.com/CFMIP/COSPv2.0
+         fxsparse = ../.cosp_sparse_checkout
+         fxtag = v2.1.4cesm
+
+   This indicates that submodule named cosp2 at tag v2.1.4cesm should
+   be checked out into directory src/physics/cosp2/src relative to the
+   .gitmodules directory from the given url and that it should use the
+   sparse checkout as described in file ../.cosp_sparse_checkout relative
+   to the path directory.
+
+   [submodule "cime"]
+       path = cime
+       url = https://github.com/jedwards4b/cime
+       fxrequired = T:T
+       fxtag = cime6.0.198_rme01
+
+   This indicates that submodule cime should be checked out into a directory cime
+   at tag cime6.0.198_rme01 from url https://github.com/jedwards4b/cime, this should
+   only be done if this .gitmodules file is at the TopLevel of the repository clone.
 
 ## Contributing
 
diff --git a/git-fleximod b/git-fleximod
index d13f1b9f0..f03984f83 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -128,9 +128,9 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
 
     if options.optional:
-        esmrequired = ["T:T", "T:F", "I:T"]
+        fxrequired = ["T:T", "T:F", "I:T"]
     else:
-        esmrequired = ["T:T", "I:T"]
+        fxrequired = ["T:T", "I:T"]
 
     action = options.action
     if not action:
@@ -147,7 +147,7 @@ def commandline_arguments(args=None):
     return (
         options.path,
         options.gitmodules,
-        esmrequired,
+        fxrequired,
         options.components,
         options.exclude,
         options.verbose,
@@ -212,7 +212,7 @@ def submodule_checkout(root, name, path):
 def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
-        tag = gitmodules.get(name, "esmtag")
+        tag = gitmodules.get(name, "fxtag")
         newpath = os.path.join(root_dir, path)
         with utils.pushd(newpath):
             git = GitInterface(newpath)
@@ -232,9 +232,10 @@ def submodules_status(gitmodules, root_dir):
 
 def submodules_update(gitmodules, root_dir):
     for name in gitmodules.sections():
-        esmtag = gitmodules.get(name, "esmtag")
+        fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
+        print(f"name={name} path={path} url={url} fxtag={fxtag}")
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
@@ -258,29 +259,31 @@ def submodules_update(gitmodules, root_dir):
                         git.git_operation("remote", "add", newremote, url)
 
                 tags = git.git_operation("tag", "-l")
-                if esmtag and esmtag not in tags:
+                if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
-
-                git.git_operation("checkout", esmtag)
+                if fxtag:
+                    git.git_operation("checkout", fxtag)
+                else:
+                    print(f"No fxtag found for submodule {name}")
 
 
 def submodules_install(gitmodules, root_dir, requiredlist):
     for name in gitmodules.sections():
-        esmrequired = gitmodules.get(name, "esmrequired")
-        esmsparse = gitmodules.get(name, "esmsparse")
-        esmtag = gitmodules.get(name, "esmtag")
+        fxrequired = gitmodules.get(name, "fxrequired")
+        fxsparse = gitmodules.get(name, "fxsparse")
+        fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
 
-        if esmrequired and esmrequired not in requiredlist:
-            if "T:F" == esmrequired:
+        if fxrequired and fxrequired not in requiredlist:
+            if "T:F" == fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
-        if esmsparse:
+        if fxsparse:
             logger.debug(
-                f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {esmsparse}, {esmtag}"
+                f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {fxsparse}, {fxtag}"
             )
-            submodule_sparse_checkout(root_dir, name, url, path, esmsparse, tag=esmtag)
+            submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag)
         else:
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
@@ -292,7 +295,7 @@ def _main_func():
     (
         root_dir,
         file_name,
-        esmrequired,
+        fxrequired,
         includelist,
         excludelist,
         verbose,
@@ -320,7 +323,7 @@ def _main_func():
     if action == "update":
         submodules_update(gitmodules, root_dir)
     elif action == "install":
-        submodules_install(gitmodules, root_dir, esmrequired)
+        submodules_install(gitmodules, root_dir, fxrequired)
     elif action == "status":
         submodules_status(gitmodules, root_dir)
     else:

From 3903206397dfebfd0ca2f02a53761466b79cfc89 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 11:05:00 -0700
Subject: [PATCH 027/131] trying to improve markdown formatting

---
 README.md | 71 ++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 47 insertions(+), 24 deletions(-)

diff --git a/README.md b/README.md
index da725ab5f..c5a885bd7 100644
--- a/README.md
+++ b/README.md
@@ -43,30 +43,53 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 
 ## Examples
 
-    Installing submodules including optional ones: git fleximod install --optional
-    Update a specific submodule to the fxtag indicated in .gitmodules: git fleximod update submodule-name
-    Example .gitmodules entry:
-      [submodule "cosp2"]
-         path = src/physics/cosp2/src
-         url = https://github.com/CFMIP/COSPv2.0
-         fxsparse = ../.cosp_sparse_checkout
-         fxtag = v2.1.4cesm
-
-   This indicates that submodule named cosp2 at tag v2.1.4cesm should
-   be checked out into directory src/physics/cosp2/src relative to the
-   .gitmodules directory from the given url and that it should use the
-   sparse checkout as described in file ../.cosp_sparse_checkout relative
-   to the path directory.
-
-   [submodule "cime"]
-       path = cime
-       url = https://github.com/jedwards4b/cime
-       fxrequired = T:T
-       fxtag = cime6.0.198_rme01
-
-   This indicates that submodule cime should be checked out into a directory cime
-   at tag cime6.0.198_rme01 from url https://github.com/jedwards4b/cime, this should
-   only be done if this .gitmodules file is at the TopLevel of the repository clone.
+Here are some common usage examples:
+
+    Installing submodules, including optional ones:
+    Bash
+
+    git fleximod install --optional
+
+
+Updating a specific submodule to the fxtag indicated in .gitmodules:
+
+Bash
+
+git fleximod update submodule-name
+
+Example .gitmodules entry:
+Ini, TOML
+
+[submodule "cosp2"]
+    path = src/physics/cosp2/src
+    url = https://github.com/CFMIP/COSPv2.0
+    fxsparse = ../.cosp_sparse_checkout
+    fxtag = v2.1.4cesm
+
+Explanation:
+
+    This entry indicates that the submodule named cosp2 at tag
+    v2.1.4cesm should be checked out into the directory
+    src/physics/cosp2/src relative to the .gitmodules directory.  It
+    should be checked out from the URL
+    https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
+    described in the file ../.cosp_sparse_checkout relative to the
+    path directory.
+
+Additional example:
+Ini, TOML
+
+[submodule "cime"]
+    path = cime
+    url = https://github.com/jedwards4b/cime
+    fxrequired = T:T
+    fxtag = cime6.0.198_rme01
+
+
+Explanation:
+
+    This entry indicates that the submodule cime should be checked out into a directory named cime at tag cime6.0.198_rme01 from the URL https://github.com/jedwards4b/cime.
+    This should only be done if the .gitmodules file is at the top level of the repository clone.
 
 ## Contributing
 

From 8bd89cf9b7bdd710b0aa23def9a4b2f21c047b84 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 11:09:47 -0700
Subject: [PATCH 028/131] trying to improve markdown

---
 README.md | 66 +++++++++++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 34 deletions(-)

diff --git a/README.md b/README.md
index c5a885bd7..f3a0f889c 100644
--- a/README.md
+++ b/README.md
@@ -46,27 +46,24 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 Here are some common usage examples:
 
     Installing submodules, including optional ones:
-    Bash
-
+    ```Bash
     git fleximod install --optional
-
-
-Updating a specific submodule to the fxtag indicated in .gitmodules:
-
-Bash
-
-git fleximod update submodule-name
-
-Example .gitmodules entry:
-Ini, TOML
-
-[submodule "cosp2"]
-    path = src/physics/cosp2/src
-    url = https://github.com/CFMIP/COSPv2.0
-    fxsparse = ../.cosp_sparse_checkout
-    fxtag = v2.1.4cesm
-
-Explanation:
+    ```
+
+    Updating a specific submodule to the fxtag indicated in .gitmodules:
+
+    ```Bash
+    git fleximod update submodule-name
+    ```
+    Example .gitmodules entry:
+    ```Ini, TOML
+    [submodule "cosp2"]
+        path = src/physics/cosp2/src
+        url = https://github.com/CFMIP/COSPv2.0
+        fxsparse = ../.cosp_sparse_checkout
+        fxtag = v2.1.4cesm
+    ```
+    Explanation:
 
     This entry indicates that the submodule named cosp2 at tag
     v2.1.4cesm should be checked out into the directory
@@ -76,20 +73,21 @@ Explanation:
     described in the file ../.cosp_sparse_checkout relative to the
     path directory.
 
-Additional example:
-Ini, TOML
-
-[submodule "cime"]
-    path = cime
-    url = https://github.com/jedwards4b/cime
-    fxrequired = T:T
-    fxtag = cime6.0.198_rme01
-
-
-Explanation:
-
-    This entry indicates that the submodule cime should be checked out into a directory named cime at tag cime6.0.198_rme01 from the URL https://github.com/jedwards4b/cime.
-    This should only be done if the .gitmodules file is at the top level of the repository clone.
+    Additional example:
+    ```Ini, TOML
+    [submodule "cime"]
+        path = cime
+        url = https://github.com/jedwards4b/cime
+        fxrequired = T:T
+        fxtag = cime6.0.198_rme01
+    ```
+
+    Explanation:
+
+    This entry indicates that the submodule cime should be checked out
+    into a directory named cime at tag cime6.0.198_rme01 from the URL
+    https://github.com/jedwards4b/cime.  This should only be done if
+    the .gitmodules file is at the top level of the repository clone.
 
 ## Contributing
 

From e83bc7a865540751093c393f9edd8f66a450cf40 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 11:12:13 -0700
Subject: [PATCH 029/131] trying to improve markdown

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index f3a0f889c..199d2064c 100644
--- a/README.md
+++ b/README.md
@@ -46,17 +46,17 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 Here are some common usage examples:
 
     Installing submodules, including optional ones:
-    ```Bash
-    git fleximod install --optional
-    ```
+```bash
+  git fleximod install --optional
+```
 
     Updating a specific submodule to the fxtag indicated in .gitmodules:
 
-    ```Bash
+    ```bash
     git fleximod update submodule-name
     ```
     Example .gitmodules entry:
-    ```Ini, TOML
+    ```ini, toml
     [submodule "cosp2"]
         path = src/physics/cosp2/src
         url = https://github.com/CFMIP/COSPv2.0
@@ -74,7 +74,7 @@ Here are some common usage examples:
     path directory.
 
     Additional example:
-    ```Ini, TOML
+    ```ini, toml
     [submodule "cime"]
         path = cime
         url = https://github.com/jedwards4b/cime

From 967edf4ced928a444162079227637c794ed562e0 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 11:14:22 -0700
Subject: [PATCH 030/131] trying to improve markdown

---
 README.md | 49 ++++++++++++++++++++++++-------------------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 199d2064c..910858e79 100644
--- a/README.md
+++ b/README.md
@@ -45,49 +45,48 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 
 Here are some common usage examples:
 
-    Installing submodules, including optional ones:
+Installing submodules, including optional ones:
 ```bash
   git fleximod install --optional
 ```
 
-    Updating a specific submodule to the fxtag indicated in .gitmodules:
+Updating a specific submodule to the fxtag indicated in .gitmodules:
 
-    ```bash
+```bash
     git fleximod update submodule-name
-    ```
-    Example .gitmodules entry:
-    ```ini, toml
+```
+Example .gitmodules entry:
+```ini, toml
     [submodule "cosp2"]
         path = src/physics/cosp2/src
         url = https://github.com/CFMIP/COSPv2.0
         fxsparse = ../.cosp_sparse_checkout
         fxtag = v2.1.4cesm
-    ```
-    Explanation:
-
-    This entry indicates that the submodule named cosp2 at tag
-    v2.1.4cesm should be checked out into the directory
-    src/physics/cosp2/src relative to the .gitmodules directory.  It
-    should be checked out from the URL
-    https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
-    described in the file ../.cosp_sparse_checkout relative to the
-    path directory.
-
-    Additional example:
-    ```ini, toml
+```
+Explanation:
+
+This entry indicates that the submodule named cosp2 at tag v2.1.4cesm
+should be checked out into the directory src/physics/cosp2/src
+relative to the .gitmodules directory.  It should be checked out from
+the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
+described in the file ../.cosp_sparse_checkout relative to the path
+directory.
+
+Additional example:
+```ini, toml
     [submodule "cime"]
         path = cime
         url = https://github.com/jedwards4b/cime
         fxrequired = T:T
         fxtag = cime6.0.198_rme01
-    ```
+```
 
-    Explanation:
+Explanation:
 
-    This entry indicates that the submodule cime should be checked out
-    into a directory named cime at tag cime6.0.198_rme01 from the URL
-    https://github.com/jedwards4b/cime.  This should only be done if
-    the .gitmodules file is at the top level of the repository clone.
+This entry indicates that the submodule cime should be checked out
+into a directory named cime at tag cime6.0.198_rme01 from the URL
+https://github.com/jedwards4b/cime.  This should only be done if
+the .gitmodules file is at the top level of the repository clone.
 
 ## Contributing
 

From 4926864512c75496530bc713ddadb26071f16ac9 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 13:17:00 -0700
Subject: [PATCH 031/131] add pip setup.py file

---
 setup.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 setup.py

diff --git a/setup.py b/setup.py
new file mode 100644
index 000000000..2d4cb3642
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,26 @@
+import setuptools
+import os
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+with open("version.txt", "r") as fh:
+    version = fh.read()
+    cwd = os.getcwd()
+setuptools.setup(
+    name="git-fleximod",                     # This is the name of the package
+    version=version,                        # The initial release version
+    author="Jim Edwards",                     # Full name of the author
+    description="Extended support for git-submodule and git-sparse-checkout",
+    long_description=long_description,      # Long description read from the the readme file
+    long_description_content_type="text/markdown",
+    packages=setuptools.find_packages(),    # List of all python modules to be installed
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],                                      # Information to filter the project on PyPi website
+    python_requires='>=3.6',                # Minimum version requirement of the package
+    py_modules=['git-fleximod'],             # Name of the python package
+    package_dir={'git-fleximod':'.'},     # Directory of the source code of the package
+    install_requires=[]                     # Install other dependencies if any
+)

From d2b3985e4c0d43eb5afbf497f50fd67adde2d0e3 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 13:22:48 -0700
Subject: [PATCH 032/131] bug fix

---
 git-fleximod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-fleximod b/git-fleximod
index f03984f83..0f0fc3634 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -169,7 +169,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
         return
 
     # initialize a new git repo and set the sparse checkout flag
-    sprepo_git = GitInterface(os.path.join(topdir, path))
+    sprepo_git = GitInterface(os.path.join(root_dir, path))
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
 
     # set the repository remote

From 27375e88a3ee5d08ffc567f2d52f7ab6a7d1df7e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 7 Jan 2024 14:23:55 -0700
Subject: [PATCH 033/131] improve messages

---
 git-fleximod | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 0f0fc3634..69491a5fd 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -235,7 +235,7 @@ def submodules_update(gitmodules, root_dir):
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-        print(f"name={name} path={path} url={url} fxtag={fxtag}")
+        logger.info(f"name={name} path={path} url={url} fxtag={fxtag}")
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
@@ -261,10 +261,14 @@ def submodules_update(gitmodules, root_dir):
                 tags = git.git_operation("tag", "-l")
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
-                if fxtag:
+                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                if fxtag and fxtag != atag:
+                    print(f"Updating {name} to {fxtag}")
                     git.git_operation("checkout", fxtag)
-                else:
+                elif not fxtag:
                     print(f"No fxtag found for submodule {name}")
+                else:
+                    print(f"submodule {name} up to date.")
 
 
 def submodules_install(gitmodules, root_dir, requiredlist):

From a17fe845273a99aa767558b88a020c1d82724f98 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 12:03:03 -0700
Subject: [PATCH 034/131] fix some issues in sparse checkout

---
 git-fleximod | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 69491a5fd..51d037265 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -162,11 +162,6 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # Check first if the module is already defined
     # and the sparse-checkout file exists
     git = GitInterface(root_dir)
-    topgit = os.path.join(root_dir, ".git", "modules")
-    gitsparse = os.path.join(topgit, name, "info", "sparse-checkout")
-    if os.path.isfile(gitsparse):
-        logger.warning("submodule {} is already initialized".format(name))
-        return
 
     # initialize a new git repo and set the sparse checkout flag
     sprepo_git = GitInterface(os.path.join(root_dir, path))
@@ -175,16 +170,31 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
     sprepo_git.git_operation("remote", "add", "origin", url)
 
+
+    if os.path.isfile(os.path.join(root_dir, ".git")):
+        with open(os.path.join(root_dir, ".git")) as f:
+            gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1]))
+        print(f"gitpath is {gitpath}")
+        topgit = os.path.join(gitpath, "modules")
+    else:                      
+        topgit = os.path.join(root_dir, ".git", "modules")
+    print(f"topgit is {topgit}")
     if not os.path.isdir(topgit):
         os.makedirs(topgit)
     topgit = os.path.join(topgit, name)
+    print(f"topgit is {topgit}")
+    if os.path.isdir(os.path.join(root_dir,path,".git")):
+        shutil.move(os.path.join(root_dir,path, ".git"), topgit)
+        with open(os.path.join(root_dir,path, ".git"), "w") as f:
+            f.write("gitdir: " + os.path.relpath(topgit, path))
+            
+    gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
+    if os.path.isfile(gitsparse):
+        logger.warning("submodule {} is already initialized".format(name))
+        return
 
-    shutil.move(os.path.join(path, ".git"), topgit)
-
-    shutil.copy(os.path.join(path, sparsefile), gitsparse)
 
-    with open(os.path.join(path, ".git"), "w") as f:
-        f.write("gitdir: " + os.path.relpath(topgit, path))
+    shutil.copy(os.path.join(root_dir,path, sparsefile), gitsparse)
 
     # Finally checkout the repo
     sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")

From 8c9c123bd687e857a0d01205ddd0a3d9849fe6ec Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 14:35:57 -0700
Subject: [PATCH 035/131] need to replace ssh with https

---
 git-fleximod | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 51d037265..fbd6099a2 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -162,7 +162,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # Check first if the module is already defined
     # and the sparse-checkout file exists
     git = GitInterface(root_dir)
-
+    print(f"root_dir is {root_dir}")
     # initialize a new git repo and set the sparse checkout flag
     sprepo_git = GitInterface(os.path.join(root_dir, path))
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
@@ -170,7 +170,6 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
     sprepo_git.git_operation("remote", "add", "origin", url)
 
-
     if os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
             gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1]))
@@ -224,20 +223,23 @@ def submodules_status(gitmodules, root_dir):
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
         newpath = os.path.join(root_dir, path)
-        with utils.pushd(newpath):
-            git = GitInterface(newpath)
-            atag = git.git_operation("describe", "--tags", "--always").rstrip()
-            if tag and atag != tag:
-                print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
-            elif tag:
-                print(f"Submodule {name} at tag {tag}")
-            else:
-                print(
-                    f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
-                )
-            status = git.git_operation("status")
-            if "nothing to commit" not in status:
-                print(status)
+        if not os.path.exists(os.path.join(newpath, ".git")):
+            print(f"Submodule {name} not checked out")
+        else:
+            with utils.pushd(newpath):
+                git = GitInterface(newpath)
+                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                if tag and atag != tag:
+                    print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
+                elif tag:
+                    print(f"Submodule {name} at tag {tag}")
+                else:
+                    print(
+                        f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
+                    )
+                status = git.git_operation("status")
+                if "nothing to commit" not in status:
+                    print(status)
 
 
 def submodules_update(gitmodules, root_dir):
@@ -293,6 +295,11 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             if "T:F" == fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
+        # ssh urls cause problems for those who dont have git accounts with ssh keys defined
+        # but cime has one since e3sm prefers ssh to https
+        if url.startswith("git@"):
+            url = url.replace("git@github.com:", "https://github.com")
+            
         if fxsparse:
             logger.debug(
                 f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {fxsparse}, {fxtag}"

From 2752eccf37f28d3b7505d7d3c1faeb352c1a1bd0 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 14:45:48 -0700
Subject: [PATCH 036/131] change from ssh to https

---
 git-fleximod | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index fbd6099a2..179bc2ca9 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -201,9 +201,12 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     print(f"Successfully checked out {name}")
 
 
-def submodule_checkout(root, name, path):
+def submodule_checkout(root, name, path, url=None):
     git = GitInterface(root)
     repodir = os.path.join(root, path)
+    # if url is provided update to the new url
+    if url:
+        git.git_operation("submodule", "set-url", path, url)
     git.git_operation("submodule", "update", "--init", path)
     # Look for a .gitmodules file in the newly checkedout repo
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
@@ -309,7 +312,7 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
-            submodule_checkout(root_dir, name, path)
+            submodule_checkout(root_dir, name, path, url=url)
 
 
 def _main_func():

From 2092f3ff672051f4c34de4986f7b59debbc30920 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 14:58:44 -0700
Subject: [PATCH 037/131] fix ssh issue

---
 git-fleximod | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 179bc2ca9..0fcbc1fee 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -205,10 +205,19 @@ def submodule_checkout(root, name, path, url=None):
     git = GitInterface(root)
     repodir = os.path.join(root, path)
     # if url is provided update to the new url
+    tmpurl = None
     if url:
+        # ssh urls cause problems for those who dont have git accounts with ssh keys defined
+        # but cime has one since e3sm prefers ssh to https
+        if url.startswith("git@"):
+            tmpurl = url
+            url = url.replace("git@github.com:", "https://github.com")            
         git.git_operation("submodule", "set-url", path, url)
     git.git_operation("submodule", "update", "--init", path)
     # Look for a .gitmodules file in the newly checkedout repo
+    if tmpurl:
+        git.git_operation("submodule", "set-url", path, tmpurl)
+        
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir}")
@@ -298,10 +307,6 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             if "T:F" == fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
-        # ssh urls cause problems for those who dont have git accounts with ssh keys defined
-        # but cime has one since e3sm prefers ssh to https
-        if url.startswith("git@"):
-            url = url.replace("git@github.com:", "https://github.com")
             
         if fxsparse:
             logger.debug(
@@ -312,6 +317,7 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
+            
             submodule_checkout(root_dir, name, path, url=url)
 
 

From 4fd30a41f604b2bb6fbb8533429b60b4f1d0adc5 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 15:07:31 -0700
Subject: [PATCH 038/131] remove debug print statments

---
 git-fleximod | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 0fcbc1fee..404b598f2 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -162,7 +162,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # Check first if the module is already defined
     # and the sparse-checkout file exists
     git = GitInterface(root_dir)
-    print(f"root_dir is {root_dir}")
+
     # initialize a new git repo and set the sparse checkout flag
     sprepo_git = GitInterface(os.path.join(root_dir, path))
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
@@ -173,15 +173,14 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     if os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
             gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1]))
-        print(f"gitpath is {gitpath}")
         topgit = os.path.join(gitpath, "modules")
     else:                      
         topgit = os.path.join(root_dir, ".git", "modules")
-    print(f"topgit is {topgit}")
+
     if not os.path.isdir(topgit):
         os.makedirs(topgit)
     topgit = os.path.join(topgit, name)
-    print(f"topgit is {topgit}")
+
     if os.path.isdir(os.path.join(root_dir,path,".git")):
         shutil.move(os.path.join(root_dir,path, ".git"), topgit)
         with open(os.path.join(root_dir,path, ".git"), "w") as f:

From 86a5a72abcbf9709a01ad01fa6c01229700a13f2 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 8 Jan 2024 16:59:21 -0700
Subject: [PATCH 039/131] fix issue with sparse checkout

---
 git-fleximod | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 404b598f2..cb1c948d6 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -170,21 +170,23 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
     sprepo_git.git_operation("remote", "add", "origin", url)
 
+    superroot = git.git_operation("rev-parse", "--show-superproject-working-tree")
     if os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
             gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1]))
-        topgit = os.path.join(gitpath, "modules")
+        topgit = os.path.abspath(os.path.join(gitpath, "modules"))
     else:                      
-        topgit = os.path.join(root_dir, ".git", "modules")
+        topgit = os.path.abspath(os.path.join(root_dir, ".git", "modules"))
 
     if not os.path.isdir(topgit):
         os.makedirs(topgit)
     topgit = os.path.join(topgit, name)
+    logger.debug(f"root_dir is {root_dir} topgit is {topgit} superroot is {superroot}")
 
     if os.path.isdir(os.path.join(root_dir,path,".git")):
         shutil.move(os.path.join(root_dir,path, ".git"), topgit)
         with open(os.path.join(root_dir,path, ".git"), "w") as f:
-            f.write("gitdir: " + os.path.relpath(topgit, path))
+            f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir,path)))
             
     gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
     if os.path.isfile(gitsparse):

From a9130f296d068f441994084f86c2fd376c66d6be Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 9 Jan 2024 09:42:17 -0700
Subject: [PATCH 040/131] more error checking

---
 git-fleximod          | 4 +++-
 modules/gitmodules.py | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index cb1c948d6..2a00d4106 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -221,7 +221,7 @@ def submodule_checkout(root, name, path, url=None):
         
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
-        print(f"Recursively checking out submodules of {name} {repodir}")
+        print(f"Recursively checking out submodules of {name} ")
         gitmodules = GitModules(repodir)
         submodules_install(gitmodules, repodir, ["I:T"])
     if os.path.exists(os.path.join(repodir, ".git")):
@@ -235,6 +235,8 @@ def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
+        if not path:
+            utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
         if not os.path.exists(os.path.join(newpath, ".git")):
             print(f"Submodule {name} not checked out")
diff --git a/modules/gitmodules.py b/modules/gitmodules.py
index 9e38a5806..64b539a38 100644
--- a/modules/gitmodules.py
+++ b/modules/gitmodules.py
@@ -51,7 +51,8 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
             return None
 
     def save(self):
-        self.write(open(self.conf_file, "w"))
+        print("Called gitmodules save, not expected")
+        #        self.write(open(self.conf_file, "w"))
 
     def sections(self):
         """Strip the submodule part out of section and just use the name"""

From 3ccf8a3adf15825ff440f9430c9cf3f56b99fe4b Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 9 Jan 2024 17:44:28 -0700
Subject: [PATCH 041/131] hack to prevent file corruption

---
 git-fleximod | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/git-fleximod b/git-fleximod
index 2a00d4106..e8a719aae 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -203,6 +203,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
 
 
 def submodule_checkout(root, name, path, url=None):
+    shutil.copy(os.path.join(root,".gitmodules"), os.path.join(root,".save.gitmodules"))
     git = GitInterface(root)
     repodir = os.path.join(root, path)
     # if url is provided update to the new url
@@ -214,7 +215,9 @@ def submodule_checkout(root, name, path, url=None):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
         git.git_operation("submodule", "set-url", path, url)
-    git.git_operation("submodule", "update", "--init", path)
+    git.git_operation("submodule", "update", "--init", "--", path)
+    shutil.copy(os.path.join(root,".save.gitmodules"), os.path.join(root,".gitmodules"))
+
     # Look for a .gitmodules file in the newly checkedout repo
     if tmpurl:
         git.git_operation("submodule", "set-url", path, tmpurl)

From 840499c5601b1c944ac509b38558da0e8323ed9b Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 9 Jan 2024 18:01:01 -0700
Subject: [PATCH 042/131] ignore untracked files in submodules

---
 git-fleximod | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index e8a719aae..04fcf2045 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -216,7 +216,8 @@ def submodule_checkout(root, name, path, url=None):
             url = url.replace("git@github.com:", "https://github.com")            
         git.git_operation("submodule", "set-url", path, url)
     git.git_operation("submodule", "update", "--init", "--", path)
-    shutil.copy(os.path.join(root,".save.gitmodules"), os.path.join(root,".gitmodules"))
+    
+    shutil.move(os.path.join(root,".save.gitmodules"), os.path.join(root,".gitmodules"))
 
     # Look for a .gitmodules file in the newly checkedout repo
     if tmpurl:
@@ -255,7 +256,7 @@ def submodules_status(gitmodules, root_dir):
                     print(
                         f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
                     )
-                status = git.git_operation("status")
+                status = git.git_operation("status","--ignore-submodules","untracked")
                 if "nothing to commit" not in status:
                     print(status)
 

From a25b257efc706ebc03468be128ec9c566791b05e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 10 Jan 2024 05:24:18 -0700
Subject: [PATCH 043/131] a better solution to the file coruption issue

---
 git-fleximod          | 9 ++++-----
 modules/gitmodules.py | 6 ++++++
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 04fcf2045..972605910 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -203,22 +203,21 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
 
 
 def submodule_checkout(root, name, path, url=None):
-    shutil.copy(os.path.join(root,".gitmodules"), os.path.join(root,".save.gitmodules"))
     git = GitInterface(root)
     repodir = os.path.join(root, path)
     # if url is provided update to the new url
     tmpurl = None
     if url:
         # ssh urls cause problems for those who dont have git accounts with ssh keys defined
-        # but cime has one since e3sm prefers ssh to https
+        # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was
+        # opened with a GitModules object we don't need to worry about restoring the file here
+        # it will be done by the GitModules class
         if url.startswith("git@"):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
         git.git_operation("submodule", "set-url", path, url)
     git.git_operation("submodule", "update", "--init", "--", path)
     
-    shutil.move(os.path.join(root,".save.gitmodules"), os.path.join(root,".gitmodules"))
-
     # Look for a .gitmodules file in the newly checkedout repo
     if tmpurl:
         git.git_operation("submodule", "set-url", path, tmpurl)
@@ -345,7 +344,7 @@ def _main_func():
         file_path = utils.find_upwards(root_dir, file_name)
 
         if file_path is None:
-            fatal_error(
+            utils.fatal_error(
                 "No {} found in {} or any of it's parents".format(file_name, root_dir)
             )
         root_dir = os.path.dirname(file_path)
diff --git a/modules/gitmodules.py b/modules/gitmodules.py
index 64b539a38..fe05e3ca6 100644
--- a/modules/gitmodules.py
+++ b/modules/gitmodules.py
@@ -1,4 +1,5 @@
 import os
+import shutil
 from configparser import ConfigParser
 
 from modules.lstripreader import LstripReader
@@ -20,6 +21,8 @@ def __init__(
         """
         ConfigParser.__init__(self)
         self.conf_file = os.path.join(confpath, conffile)
+        # first create a backup of this file to be restored on deletion of the object
+        shutil.copy(self.conf_file, self.conf_file+".save")
         self.read_file(LstripReader(self.conf_file), source=conffile)
         self.includelist = includelist
         self.excludelist = excludelist
@@ -54,6 +57,9 @@ def save(self):
         print("Called gitmodules save, not expected")
         #        self.write(open(self.conf_file, "w"))
 
+    def __del__(self):
+        shutil.move(self.conf_file+".save", self.conf_file)
+            
     def sections(self):
         """Strip the submodule part out of section and just use the name"""
         names = []

From 657f110a761e2c49524e2f2dd64c139968b3ef2c Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 10 Jan 2024 10:48:30 -0700
Subject: [PATCH 044/131] awkward solution for ssh urls

---
 git-fleximod | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 972605910..206359e39 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -202,11 +202,14 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     print(f"Successfully checked out {name}")
 
 
-def submodule_checkout(root, name, path, url=None):
+def submodule_checkout(root, name, path, url=None, tag=None):
     git = GitInterface(root)
     repodir = os.path.join(root, path)
+
     # if url is provided update to the new url
     tmpurl = None
+
+    # Look for a .gitmodules file in the newly checkedout repo
     if url:
         # ssh urls cause problems for those who dont have git accounts with ssh keys defined
         # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was
@@ -215,22 +218,31 @@ def submodule_checkout(root, name, path, url=None):
         if url.startswith("git@"):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
-        git.git_operation("submodule", "set-url", path, url)
-    git.git_operation("submodule", "update", "--init", "--", path)
-    
-    # Look for a .gitmodules file in the newly checkedout repo
-    if tmpurl:
-        git.git_operation("submodule", "set-url", path, tmpurl)
-        
+
+            git.git_operation("clone", "-b", tag, url, path)
+            print(f"Using {url} for {name}")
+            #            print(git.git_operation("submodule", "set-url", name, url))
+            #        if "cprnc" in repodir:
+            #            with open(os.path.join(root,".gitmodules"),"r") as f:
+            #                print(f.read())
+        print(f"calling update for {name}")
+    if not tmpurl:
+        logger.debug(git.git_operation("submodule", "update", "--init", "--", name))
+
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
-        print(f"Recursively checking out submodules of {name} ")
+        print(f"Recursively checking out submodules of {name} {repodir} {url}")
         gitmodules = GitModules(repodir)
         submodules_install(gitmodules, repodir, ["I:T"])
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
     else:
         utils.fatal_error(f"Failed to checkout {name}")
+
+    
+    if tmpurl:
+        print(git.git_operation("restore", ".gitmodules"))
+        print(f"Using {tmpurl} for {name}")
     return
 
 
@@ -324,7 +336,7 @@ def submodules_install(gitmodules, root_dir, requiredlist):
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
             
-            submodule_checkout(root_dir, name, path, url=url)
+            submodule_checkout(root_dir, name, path, url=url, tag=fxtag)
 
 
 def _main_func():

From 4879c7530523b8b7a16c11a9c113755d8b0e4a85 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 10 Jan 2024 10:57:31 -0700
Subject: [PATCH 045/131] remove debug print statements

---
 git-fleximod | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/git-fleximod b/git-fleximod
index 206359e39..f78dfa25e 100755
--- a/git-fleximod
+++ b/git-fleximod
@@ -218,16 +218,10 @@ def submodule_checkout(root, name, path, url=None, tag=None):
         if url.startswith("git@"):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
-
             git.git_operation("clone", "-b", tag, url, path)
-            print(f"Using {url} for {name}")
-            #            print(git.git_operation("submodule", "set-url", name, url))
-            #        if "cprnc" in repodir:
-            #            with open(os.path.join(root,".gitmodules"),"r") as f:
-            #                print(f.read())
-        print(f"calling update for {name}")
+
     if not tmpurl:
-        logger.debug(git.git_operation("submodule", "update", "--init", "--", name))
+        logger.debug(git.git_operation("submodule", "update", "--init", "--", path))
 
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
@@ -239,10 +233,9 @@ def submodule_checkout(root, name, path, url=None, tag=None):
     else:
         utils.fatal_error(f"Failed to checkout {name}")
 
-    
     if tmpurl:
         print(git.git_operation("restore", ".gitmodules"))
-        print(f"Using {tmpurl} for {name}")
+
     return
 
 

From 661ba5efc813fa910e7fc43172a513a177d2e013 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 10 Jan 2024 17:29:53 -0700
Subject: [PATCH 046/131] update for distribution

---
 setup.py                                  | 25 +++++----
 {modules => src/fleximod}/__init__.py     |  0
 {modules => src/fleximod}/gitinterface.py | 13 +++--
 {modules => src/fleximod}/gitmodules.py   | 12 +++--
 {modules => src/fleximod}/lstripreader.py |  0
 {modules => src/fleximod}/utils.py        |  0
 src/fleximod/version.py                   |  1 +
 git-fleximod => src/git-fleximod          | 62 +++++++++++++----------
 version.txt                               |  1 -
 9 files changed, 70 insertions(+), 44 deletions(-)
 rename {modules => src/fleximod}/__init__.py (100%)
 rename {modules => src/fleximod}/gitinterface.py (87%)
 rename {modules => src/fleximod}/gitmodules.py (83%)
 rename {modules => src/fleximod}/lstripreader.py (100%)
 rename {modules => src/fleximod}/utils.py (100%)
 create mode 100644 src/fleximod/version.py
 rename git-fleximod => src/git-fleximod (90%)
 delete mode 100644 version.txt

diff --git a/setup.py b/setup.py
index 2d4cb3642..1abf83925 100644
--- a/setup.py
+++ b/setup.py
@@ -1,26 +1,33 @@
 import setuptools
 import os
+from setuptools import setup, find_packages
+from distutils.util import convert_path
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
-with open("version.txt", "r") as fh:
-    version = fh.read()
-    cwd = os.getcwd()
+
+main_ns = {}
+ver_path = convert_path('src/fleximod/version.py')
+with open(ver_path) as ver_file:
+    exec(ver_file.read(), main_ns)
+
+    
 setuptools.setup(
-    name="git-fleximod",                     # This is the name of the package
-    version=version,                        # The initial release version
+    scripts=["src/git-fleximod"],                     # This is the name of the package
+    version=main_ns['__version__'],                        # The initial release version
     author="Jim Edwards",                     # Full name of the author
     description="Extended support for git-submodule and git-sparse-checkout",
     long_description=long_description,      # Long description read from the the readme file
     long_description_content_type="text/markdown",
-    packages=setuptools.find_packages(),    # List of all python modules to be installed
+    packages=['fleximod'],    # List of all python modules to be installed
+    package_dir={'fleximod': 'src/fleximod'},
+    package_data={"":['version.txt']},
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],                                      # Information to filter the project on PyPi website
     python_requires='>=3.6',                # Minimum version requirement of the package
-    py_modules=['git-fleximod'],             # Name of the python package
-    package_dir={'git-fleximod':'.'},     # Directory of the source code of the package
-    install_requires=[]                     # Install other dependencies if any
+#    py_modules=['git-fleximod'],             # Name of the python package
+    install_requires=["PyGit"]                     # Install other dependencies if any
 )
diff --git a/modules/__init__.py b/src/fleximod/__init__.py
similarity index 100%
rename from modules/__init__.py
rename to src/fleximod/__init__.py
diff --git a/modules/gitinterface.py b/src/fleximod/gitinterface.py
similarity index 87%
rename from modules/gitinterface.py
rename to src/fleximod/gitinterface.py
index a4e252925..c127163dc 100644
--- a/modules/gitinterface.py
+++ b/src/fleximod/gitinterface.py
@@ -1,11 +1,12 @@
 import os
 import logging
-from modules import utils
+from fleximod import utils
 
 class GitInterface:
-    def __init__(self, repo_path):
+    def __init__(self, repo_path, logger):
+        logger.debug("Initialize GitInterface for {}".format(repo_path))
         self.repo_path = repo_path
-
+        self.logger = logger
         try:
             import git
             self._use_module = True
@@ -20,10 +21,10 @@ def __init__(self, repo_path):
             if not os.path.exists(os.path.join(repo_path,".git")):
                 self._init_git_repo()
             msg = "Using shell interface to git"
-        logging.info(msg)
+        self.logger.info(msg)
                 
     def _git_command(self, operation, *args):
-        logging.info(operation)
+        self.logger.info(operation)
         if self._use_module and operation != "submodule":
             return getattr(self.repo.git, operation)(*args)
         else:
@@ -39,6 +40,7 @@ def _init_git_repo(self):
 
     def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *args)
+        self.logger.info(command)
         if isinstance(command, list):
             return utils.execute_subprocess(command, output_to_caller=True)
         else:
@@ -60,4 +62,5 @@ def config_set_value(self, section, name, value):
             writer.release()  # Ensure changes are saved
         else:
             cmd = ("git","-C",self.repo_path,"config", f"{section}.{name}", value)
+            self.logger.info(cmd)
             utils.execute_subprocess(cmd, output_to_caller=True)
diff --git a/modules/gitmodules.py b/src/fleximod/gitmodules.py
similarity index 83%
rename from modules/gitmodules.py
rename to src/fleximod/gitmodules.py
index fe05e3ca6..a6f758931 100644
--- a/modules/gitmodules.py
+++ b/src/fleximod/gitmodules.py
@@ -1,13 +1,12 @@
 import os
 import shutil
 from configparser import ConfigParser
-
-from modules.lstripreader import LstripReader
-
+from fleximod.lstripreader import LstripReader
 
 class GitModules(ConfigParser):
     def __init__(
         self,
+        logger,
         confpath=os.getcwd(),
         conffile=".gitmodules",
         includelist=None,
@@ -19,6 +18,8 @@ def __init__(
         includelist: Optional list of submodules to include.
         excludelist: Optional list of submodules to exclude.
         """
+        self.logger = logger
+        self.logger.debug("Creating a GitModules object {} {} {} {}".format(confpath,conffile,includelist,excludelist))
         ConfigParser.__init__(self)
         self.conf_file = os.path.join(confpath, conffile)
         # first create a backup of this file to be restored on deletion of the object
@@ -33,6 +34,7 @@ def set(self, name, option, value):
         Ensures the appropriate section exists for the submodule.
         Calls the parent class's set method to store the value.
         """
+        self.logger.debug("set called {} {} {}".format(name,option,value))
         section = f'submodule "{name}"'
         if not self.has_section(section):
             self.add_section(section)
@@ -45,6 +47,7 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
         Uses the parent class's get method to access the value.
         Handles potential errors if the section or option doesn't exist.
         """
+        self.logger.debug("get called {} {}".format(name,option))
         section = f'submodule "{name}"'
         try:
             return ConfigParser.get(
@@ -58,10 +61,12 @@ def save(self):
         #        self.write(open(self.conf_file, "w"))
 
     def __del__(self):
+        self.logger.debug("Destroying GitModules object")
         shutil.move(self.conf_file+".save", self.conf_file)
             
     def sections(self):
         """Strip the submodule part out of section and just use the name"""
+        self.logger.debug("calling GitModules sections iterator")
         names = []
         for section in ConfigParser.sections(self):
             name = section[11:-1]
@@ -73,5 +78,6 @@ def sections(self):
         return names
 
     def items(self, name, raw=False, vars=None):
+        self.logger.debug("calling GitModules items for {}".format(name))
         section = f'submodule "{name}"'
         return ConfigParser.items(section, raw=raw, vars=vars)
diff --git a/modules/lstripreader.py b/src/fleximod/lstripreader.py
similarity index 100%
rename from modules/lstripreader.py
rename to src/fleximod/lstripreader.py
diff --git a/modules/utils.py b/src/fleximod/utils.py
similarity index 100%
rename from modules/utils.py
rename to src/fleximod/utils.py
diff --git a/src/fleximod/version.py b/src/fleximod/version.py
new file mode 100644
index 000000000..8ce9b3623
--- /dev/null
+++ b/src/fleximod/version.py
@@ -0,0 +1 @@
+__version__ = '0.1.3'
diff --git a/git-fleximod b/src/git-fleximod
similarity index 90%
rename from git-fleximod
rename to src/git-fleximod
index f78dfa25e..dd2adb0f9 100755
--- a/git-fleximod
+++ b/src/git-fleximod
@@ -4,13 +4,12 @@ import os
 import shutil
 import logging
 import argparse
-from modules import utils
-from modules.gitinterface import GitInterface
-from modules.gitmodules import GitModules
-
-logger = logging.getLogger(__name__)
-logger.propogate = False
-
+from fleximod import utils
+from fleximod.gitinterface import GitInterface
+from fleximod.gitmodules import GitModules
+from fleximod.version import __version__
+# logger variable is global
+logger = None
 
 def commandline_arguments(args=None):
     description = """
@@ -84,9 +83,9 @@ def commandline_arguments(args=None):
     parser.add_argument(
         "-V",
         "--version",
-        action="store_true",
-        default=False,
-        help="Print manage_externals version and exit.",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="Print version and exit.",
     )
 
     #
@@ -136,14 +135,21 @@ def commandline_arguments(args=None):
     if not action:
         action = "install"
 
+    if options.debug:
+        level = logging.DEBUG
+    elif options.verbose:
+        level = logging.INFO
+    else:
+        level = logging.WARNING
+    # Configure the root logger
+    logging.basicConfig(
+        level=level,
+        format="%(name)s - %(levelname)s - %(message)s",
+        handlers=[logging.FileHandler("fleximod.log"), logging.StreamHandler()],
+    )
     if options.version:
-        version_info = ""
-        version_file_path = os.path.join(os.path.dirname(__file__), "version.txt")
-        with open(version_file_path) as f:
-            version_info = f.readlines()[0].strip()
-        print(version_info)
-        sys.exit(0)
-
+        exit()
+    
     return (
         options.path,
         options.gitmodules,
@@ -161,10 +167,10 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
         os.makedirs(path)
     # Check first if the module is already defined
     # and the sparse-checkout file exists
-    git = GitInterface(root_dir)
+    git = GitInterface(root_dir, logger)
 
     # initialize a new git repo and set the sparse checkout flag
-    sprepo_git = GitInterface(os.path.join(root_dir, path))
+    sprepo_git = GitInterface(os.path.join(root_dir, path), logger)
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
 
     # set the repository remote
@@ -203,7 +209,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
 
 
 def submodule_checkout(root, name, path, url=None, tag=None):
-    git = GitInterface(root)
+    git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
 
     # if url is provided update to the new url
@@ -226,7 +232,7 @@ def submodule_checkout(root, name, path, url=None, tag=None):
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir} {url}")
-        gitmodules = GitModules(repodir)
+        gitmodules = GitModules(logger,confpath=repodir)
         submodules_install(gitmodules, repodir, ["I:T"])
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
@@ -250,7 +256,7 @@ def submodules_status(gitmodules, root_dir):
             print(f"Submodule {name} not checked out")
         else:
             with utils.pushd(newpath):
-                git = GitInterface(newpath)
+                git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 if tag and atag != tag:
                     print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
@@ -274,7 +280,7 @@ def submodules_update(gitmodules, root_dir):
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
-                git = GitInterface(submoddir)
+                git = GitInterface(submoddir, logger)
                 # first make sure the url is correct
                 upstream = git.git_operation("ls-remote", "--get-url").rstrip()
                 newremote = "origin"
@@ -342,8 +348,11 @@ def _main_func():
         verbose,
         action,
     ) = commandline_arguments()
-    if verbose:
-        print(f"action is {action}")
+    # Get a logger for the package
+    global logger
+    logger = logging.getLogger(__name__)
+        
+    logger.info(f"action is {action}")
 
     if not os.path.isfile(os.path.join(root_dir, file_name)):
         file_path = utils.find_upwards(root_dir, file_name)
@@ -353,8 +362,9 @@ def _main_func():
                 "No {} found in {} or any of it's parents".format(file_name, root_dir)
             )
         root_dir = os.path.dirname(file_path)
-    print(f"root_dir is {root_dir}")
+    logger.info(f"root_dir is {root_dir}")
     gitmodules = GitModules(
+        logger,
         confpath=root_dir,
         conffile=file_name,
         includelist=includelist,
diff --git a/version.txt b/version.txt
deleted file mode 100644
index a3dce6cd3..000000000
--- a/version.txt
+++ /dev/null
@@ -1 +0,0 @@
-v0.0.2

From b22bab6c5c359a9635493dd90cc8d17af03321b9 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 11 Jan 2024 11:03:15 -0700
Subject: [PATCH 047/131] fix error with version flag

---
 src/git-fleximod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index dd2adb0f9..d45acb14e 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -147,7 +147,7 @@ def commandline_arguments(args=None):
         format="%(name)s - %(levelname)s - %(message)s",
         handlers=[logging.FileHandler("fleximod.log"), logging.StreamHandler()],
     )
-    if options.version:
+    if hasattr(options, 'version'):
         exit()
     
     return (

From e81099a63708e4e5cace47b6a1fe39bafee694b5 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 11 Jan 2024 11:04:10 -0700
Subject: [PATCH 048/131] update version to 0.1.4

---
 src/fleximod/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 8ce9b3623..7525d1993 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.3'
+__version__ = '0.1.4'

From 1a80e973560b86ccee228274dbb2b38e17037abc Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Mon, 15 Jan 2024 07:52:01 -0700
Subject: [PATCH 049/131] fix issues with rerunning

---
 src/fleximod/version.py |  2 +-
 src/git-fleximod        | 25 ++++++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 7525d1993..2fb25139f 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.4'
+__version__ = '0.1.6'
diff --git a/src/git-fleximod b/src/git-fleximod
index d45acb14e..d7bc2c120 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -8,6 +8,7 @@ from fleximod import utils
 from fleximod.gitinterface import GitInterface
 from fleximod.gitmodules import GitModules
 from fleximod.version import __version__
+from configparser import NoOptionError
 # logger variable is global
 logger = None
 
@@ -170,7 +171,21 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     git = GitInterface(root_dir, logger)
 
     # initialize a new git repo and set the sparse checkout flag
-    sprepo_git = GitInterface(os.path.join(root_dir, path), logger)
+    sprep_repo = os.path.join(root_dir, path)
+    sprepo_git = GitInterface(sprep_repo, logger)
+    if os.path.exists(os.path.join(sprep_repo,".git")):
+        try:
+            logger.info("Submodule {} found".format(name))
+            chk = sprepo_git.config_get_value("core", "sparseCheckout")
+            if chk == "true":
+                logger.info("Sparse submodule {} already checked out".format(name))
+                return
+        except NoOptionError:
+            logger.debug("Sparse submodule {} not present".format(name))
+        except Exception as e:
+            utils.fatal_error("Unexpected error {} occured.".format(e))
+
+
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
 
     # set the repository remote
@@ -211,7 +226,9 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
 def submodule_checkout(root, name, path, url=None, tag=None):
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
-
+    if os.path.exists(os.path.join(repodir, ".git")):
+        logger.info("Submodule {} already checked out".format(name))
+        return
     # if url is provided update to the new url
     tmpurl = None
 
@@ -225,7 +242,9 @@ def submodule_checkout(root, name, path, url=None, tag=None):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
             git.git_operation("clone", "-b", tag, url, path)
-
+            # Now need to move the .git dir to the submodule location
+            
+            
     if not tmpurl:
         logger.debug(git.git_operation("submodule", "update", "--init", "--", path))
 

From 13eb8b87bcfba34eb70c5bd3a39e570d2527a452 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 15 Jan 2024 07:53:10 -0700
Subject: [PATCH 050/131] update setup.py

---
 setup.cfg | 2 ++
 setup.py  | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 setup.cfg

diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 000000000..4f727fa0b
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[metadata]
+license_file = License
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 1abf83925..f89159386 100644
--- a/setup.py
+++ b/setup.py
@@ -16,6 +16,8 @@
     scripts=["src/git-fleximod"],                     # This is the name of the package
     version=main_ns['__version__'],                        # The initial release version
     author="Jim Edwards",                     # Full name of the author
+    maintainer="jedwards4b",
+    license="MIT License",
     description="Extended support for git-submodule and git-sparse-checkout",
     long_description=long_description,      # Long description read from the the readme file
     long_description_content_type="text/markdown",
@@ -29,5 +31,5 @@
     ],                                      # Information to filter the project on PyPi website
     python_requires='>=3.6',                # Minimum version requirement of the package
 #    py_modules=['git-fleximod'],             # Name of the python package
-    install_requires=["PyGit"]                     # Install other dependencies if any
+    install_requires=["GitPython"]                     # Install other dependencies if any
 )

From 5a590c541ca168c879d3726957c102f878af0f6d Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 15 Jan 2024 09:36:10 -0700
Subject: [PATCH 051/131] update version

---
 src/fleximod/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 2fb25139f..124e46203 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.6'
+__version__ = '0.1.7'

From 8c92b14204c80b0b444a3155a1ce52e5dca8411b Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 17 Jan 2024 16:23:33 -0700
Subject: [PATCH 052/131] provide more info for status of uninstalled
 submodules

---
 src/fleximod/version.py |  2 +-
 src/git-fleximod        | 20 ++++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 124e46203..c3bb2961b 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.7'
+__version__ = '0.1.8'
diff --git a/src/git-fleximod b/src/git-fleximod
index d7bc2c120..3e743c3f5 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -127,7 +127,8 @@ def commandline_arguments(args=None):
     else:
         options = parser.parse_args()
 
-    if options.optional:
+# explicitly listing a component overrides the optional flag    
+    if options.optional or options.components:
         fxrequired = ["T:T", "T:F", "I:T"]
     else:
         fxrequired = ["T:T", "I:T"]
@@ -271,8 +272,23 @@ def submodules_status(gitmodules, root_dir):
         if not path:
             utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
+        logger.debug("newpath is {}".format(newpath))
         if not os.path.exists(os.path.join(newpath, ".git")):
-            print(f"Submodule {name} not checked out")
+            rootgit = GitInterface(root_dir, logger)
+            # submodule commands use path, not name
+            nhash = (rootgit.git_operation("submodule","status",path).split()[0])[1:]
+            url = gitmodules.get(name, "url")
+            tags = rootgit.git_operation("ls-remote","--tags",url)
+            atag = None
+            for htag in tags.split('\n'):
+                if tag in htag:
+                    atag = (htag.split()[1])[10:]
+                    break
+            if tag == atag:
+                print(f"Submodule {name} not checked out, aligned at tag {tag}")
+            else:
+                print(f"Submodule {name} not checked out, out of sync at tag {atag}, expected tag is {tag}")
+                
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)

From 43225eac86892d074453cbdc622124b2eedf48cf Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 18 Jan 2024 16:49:52 -0700
Subject: [PATCH 053/131] add test action

---
 escomp_install          | 25 +++++++++++++++++++
 src/fleximod/version.py |  2 +-
 src/git-fleximod        | 55 ++++++++++++++++++++++++-----------------
 3 files changed, 59 insertions(+), 23 deletions(-)
 create mode 100644 escomp_install

diff --git a/escomp_install b/escomp_install
new file mode 100644
index 000000000..ae782e72a
--- /dev/null
+++ b/escomp_install
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# updates git-fleximod in an ESCOMP model
+# this script should be run from the model root directory, it expects
+# git-fleximod to already be installed with the script in bin
+# and the classes in lib/python/site-packages
+import sys
+import shutil
+import os
+
+from glob import iglob
+
+fleximod_root = sys.argv[1]
+fleximod_path = os.path.join(fleximod_root,"src","git-fleximod")
+if os.path.isfile(fleximod_path):
+   with open(fleximod_path,"r") as f:
+        fleximod = f.readlines()
+   with open(os.path.join(".","bin","git-fleximod"),"w") as f:
+        for line in fleximod:
+            f.write(line)
+            if "import argparse" in line:
+               f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n')
+
+   for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")):
+       shutil.copy(file,
+        os.path.join("lib","python","site-packages","fleximod",os.path.basename(file)))
diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index c3bb2961b..1c98a23a8 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.8'
+__version__ = '0.1.9'
diff --git a/src/git-fleximod b/src/git-fleximod
index 3e743c3f5..fce79b855 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -23,7 +23,7 @@ def commandline_arguments(args=None):
     #
     # user options
     #
-    choices = ["update", "install", "status"]
+    choices = ["update", "install", "status", "test"]
     parser.add_argument(
         "action",
         choices=choices,
@@ -107,21 +107,6 @@ def commandline_arguments(args=None):
         "information to the screen and log file.",
     )
 
-    logging_group = parser.add_mutually_exclusive_group()
-
-    logging_group.add_argument(
-        "--logging",
-        dest="do_logging",
-        action="store_true",
-        help="DEVELOPER: enable logging.",
-    )
-    logging_group.add_argument(
-        "--no-logging",
-        dest="do_logging",
-        action="store_false",
-        default=False,
-        help="DEVELOPER: disable logging " "(this is the default)",
-    )
     if args:
         options = parser.parse_args(args)
     else:
@@ -266,6 +251,7 @@ def submodule_checkout(root, name, path, url=None, tag=None):
 
 
 def submodules_status(gitmodules, root_dir):
+    testfails = 0
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
@@ -288,23 +274,27 @@ def submodules_status(gitmodules, root_dir):
                 print(f"Submodule {name} not checked out, aligned at tag {tag}")
             else:
                 print(f"Submodule {name} not checked out, out of sync at tag {atag}, expected tag is {tag}")
-                
+                testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 if tag and atag != tag:
                     print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
+                    testfails += 1
                 elif tag:
                     print(f"Submodule {name} at tag {tag}")
                 else:
                     print(
                         f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
                     )
+                    testfails += 1
+
                 status = git.git_operation("status","--ignore-submodules","untracked")
                 if "nothing to commit" not in status:
                     print(status)
-
+    
+    return testfails
 
 def submodules_update(gitmodules, root_dir):
     for name in gitmodules.sections():
@@ -372,7 +362,26 @@ def submodules_install(gitmodules, root_dir, requiredlist):
             
             submodule_checkout(root_dir, name, path, url=url, tag=fxtag)
 
+def submodules_test(gitmodules, root_dir):
+    # First check that fxtags are present and in sync with submodule hashes
+    testfails = submodules_status(gitmodules, root_dir)
+    # Then make sure that urls are consistant with fxurls (not forks and not ssh)
+    # and that sparse checkout files exist
+    for name in gitmodules.sections():
+        url = gitmodules.get(name, "url")
+        fxurl = gitmodules.get(name, "fxurl")
+        fxsparse = gitmodules.get(name, "fxsparse")
+        path = gitmodules.get(name, "path")
+        if not fxurl or url != fxurl:
+            print(f"submodule {name} url {url} not in sync with required {fxurl}")
+            testfails += 1
+        if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
+            print(f"sparse submodule {name} sparse checkout file {fxsparse} not found")
+            testfails += 1
+    return testfails
+
 
+            
 def _main_func():
     (
         root_dir,
@@ -405,16 +414,18 @@ def _main_func():
         includelist=includelist,
         excludelist=excludelist,
     )
-
+    retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir)
     elif action == "install":
         submodules_install(gitmodules, root_dir, fxrequired)
     elif action == "status":
         submodules_status(gitmodules, root_dir)
+    elif action == "test":
+        retval = submodules_test(gitmodules, root_dir)
     else:
         utils.fatal_error(f"unrecognized action request {action}")
-
-
+    return(retval)
+        
 if __name__ == "__main__":
-    _main_func()
+    sys.exit(_main_func())

From bbb1b1d98b587ddb5dd455a9ce889c64d53f7151 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 30 Jan 2024 16:18:53 -0700
Subject: [PATCH 054/131] fix issue with status

---
 README.md        | 12 +++++++++++-
 src/git-fleximod | 19 ++++++++++---------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 910858e79..cec74c5a6 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,9 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
         install: Install submodules according to configuration.
         status: Display the status of submodules.
         update: Update submodules to the tag indicated in .gitmodules variable fxtag.
+        test: Make sure that fxtags and submodule hashes are consistant,
+              make sure that official urls (as defined by fxurl) are set
+              make sure that fxtags are defined for all submodules
     Additional Options:
         See git fleximod --help for more details.
 
@@ -32,6 +35,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
       - I:T: Internal and required (always checked out).
       - I:F: Internal and optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
+    fcurl
 
 ## Sparse Checkouts
 
@@ -39,7 +43,13 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
     in the .gitmodules file to the path of a file containing the desired
     sparse checkout paths. Git-fleximod will automatically configure
     sparse checkout based on this file when applicable commands are run.
-    See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) for details on the format of this file.
+    See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set)
+    for details on the format of this file.
+
+## Tests
+
+   The git fleximod test action is designed to be used by, for example, github workflows
+   to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags
 
 ## Examples
 
diff --git a/src/git-fleximod b/src/git-fleximod
index fce79b855..9e2620d7f 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -4,6 +4,7 @@ import os
 import shutil
 import logging
 import argparse
+import textwrap
 from fleximod import utils
 from fleximod.gitinterface import GitInterface
 from fleximod.gitmodules import GitModules
@@ -23,11 +24,11 @@ def commandline_arguments(args=None):
     #
     # user options
     #
-    choices = ["update", "install", "status", "test"]
+    choices = ["update", "checkout", "status", "test"]
     parser.add_argument(
         "action",
         choices=choices,
-        default="install",
+        default="checkout",
         help=f"Subcommand of fleximod, choices are {choices}",
     )
 
@@ -120,7 +121,7 @@ def commandline_arguments(args=None):
 
     action = options.action
     if not action:
-        action = "install"
+        action = "checkout"
 
     if options.debug:
         level = logging.DEBUG
@@ -238,7 +239,7 @@ def submodule_checkout(root, name, path, url=None, tag=None):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir} {url}")
         gitmodules = GitModules(logger,confpath=repodir)
-        submodules_install(gitmodules, repodir, ["I:T"])
+        submodules_checkout(gitmodules, repodir, ["I:T"])
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
     else:
@@ -290,9 +291,9 @@ def submodules_status(gitmodules, root_dir):
                     )
                     testfails += 1
 
-                status = git.git_operation("status","--ignore-submodules","untracked")
+                status = git.git_operation("status","--ignore-submodules")
                 if "nothing to commit" not in status:
-                    print(status)
+                    print(textwrap.indent(status,'    '))
     
     return testfails
 
@@ -337,7 +338,7 @@ def submodules_update(gitmodules, root_dir):
                     print(f"submodule {name} up to date.")
 
 
-def submodules_install(gitmodules, root_dir, requiredlist):
+def submodules_checkout(gitmodules, root_dir, requiredlist):
     for name in gitmodules.sections():
         fxrequired = gitmodules.get(name, "fxrequired")
         fxsparse = gitmodules.get(name, "fxsparse")
@@ -417,8 +418,8 @@ def _main_func():
     retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir)
-    elif action == "install":
-        submodules_install(gitmodules, root_dir, fxrequired)
+    elif action == "checkout":
+        submodules_checkout(gitmodules, root_dir, fxrequired)
     elif action == "status":
         submodules_status(gitmodules, root_dir)
     elif action == "test":

From af78aa4de1632ad1bcb8e94bfa7b272294108132 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 30 Jan 2024 16:38:50 -0700
Subject: [PATCH 055/131] update version

---
 src/fleximod/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 1c98a23a8..7fd229a32 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.1.9'
+__version__ = '0.2.0'

From 13a2f6b25dda439e74396073b52c3d748b838ea6 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 12:21:58 -0700
Subject: [PATCH 056/131] cleaner format for status output, fix for special
 handling of ssh submodules, fix for 0 length log

---
 setup.cfg        |  5 ++++-
 setup.py         | 10 +++++++++-
 src/git-fleximod | 23 +++++++++++++++--------
 3 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 4f727fa0b..9a13827a3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
 [metadata]
-license_file = License
\ No newline at end of file
+license_file = License
+[build_manpages]
+manpages =
+    man/git-fleximod.1:object=parser:pyfile=bin/git-fleximod
diff --git a/setup.py b/setup.py
index f89159386..f68a4ea9b 100644
--- a/setup.py
+++ b/setup.py
@@ -2,6 +2,7 @@
 import os
 from setuptools import setup, find_packages
 from distutils.util import convert_path
+from build_manpages import build_manpages, get_build_py_cmd, get_install_cmd
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
@@ -31,5 +32,12 @@
     ],                                      # Information to filter the project on PyPi website
     python_requires='>=3.6',                # Minimum version requirement of the package
 #    py_modules=['git-fleximod'],             # Name of the python package
-    install_requires=["GitPython"]                     # Install other dependencies if any
+    install_requires=["GitPython"],                     # Install other dependencies if any
+    cmdclass={
+      'build_manpages': build_manpages,
+      # Re-define build_py and install commands so the manual pages
+      # are automatically re-generated and installed
+      'build_py': get_build_py_cmd(),
+      'install': get_install_cmd(),
+  }
 )
diff --git a/src/git-fleximod b/src/git-fleximod
index 9e2620d7f..f80d359c7 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -122,9 +122,11 @@ def commandline_arguments(args=None):
     action = options.action
     if not action:
         action = "checkout"
+    handlers=[logging.StreamHandler()]
 
     if options.debug:
         level = logging.DEBUG
+        handlers.append(logging.FileHandler("fleximod.log"))
     elif options.verbose:
         level = logging.INFO
     else:
@@ -133,8 +135,9 @@ def commandline_arguments(args=None):
     logging.basicConfig(
         level=level,
         format="%(name)s - %(levelname)s - %(message)s",
-        handlers=[logging.FileHandler("fleximod.log"), logging.StreamHandler()],
+        handlers=handlers
     )
+    
     if hasattr(options, 'version'):
         exit()
     
@@ -228,7 +231,11 @@ def submodule_checkout(root, name, path, url=None, tag=None):
         if url.startswith("git@"):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
-            git.git_operation("clone", "-b", tag, url, path)
+            git.git_operation("clone", url, path)
+            smgit = GitInterface(path, logger)
+            if not tag:
+                tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
+            smgit.git_operation("checkout",tag)
             # Now need to move the .git dir to the submodule location
             
             
@@ -272,28 +279,28 @@ def submodules_status(gitmodules, root_dir):
                     atag = (htag.split()[1])[10:]
                     break
             if tag == atag:
-                print(f"Submodule {name} not checked out, aligned at tag {tag}")
+                print(f"e {name:>20} not checked out, aligned at tag {tag}")
             else:
-                print(f"Submodule {name} not checked out, out of sync at tag {atag}, expected tag is {tag}")
+                print(f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}")
                 testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 if tag and atag != tag:
-                    print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}")
+                    print(f"s {name:>20} {atag} is out of sync with .gitmodules {tag}")
                     testfails += 1
                 elif tag:
-                    print(f"Submodule {name} at tag {tag}")
+                    print(f"  {name:>20} at tag {tag}")
                 else:
                     print(
-                        f"Submodule {name} has no tag defined in .gitmodules, module at {atag}"
+                        f"e {name:>20} has no tag defined in .gitmodules, module at {atag}"
                     )
                     testfails += 1
 
                 status = git.git_operation("status","--ignore-submodules")
                 if "nothing to commit" not in status:
-                    print(textwrap.indent(status,'    '))
+                    print('M'+textwrap.indent(status,'                      '))
     
     return testfails
 

From c21372a088fa6a72c01397eb8c955b5e5cf411d8 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 13:04:22 -0700
Subject: [PATCH 057/131] check for permission in fleximod.log file

---
 src/git-fleximod | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/git-fleximod b/src/git-fleximod
index f80d359c7..cbfa5f4f3 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -125,6 +125,10 @@ def commandline_arguments(args=None):
     handlers=[logging.StreamHandler()]
 
     if options.debug:
+        try:
+            open("fleximod.log","w")
+        except PermissionError:
+            sys.exit("ABORT: Could not write file fleximod.log")
         level = logging.DEBUG
         handlers.append(logging.FileHandler("fleximod.log"))
     elif options.verbose:

From 07eb5dd3edcde5bfd1f125c85d6935bd0fa3b3cc Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 14:02:34 -0700
Subject: [PATCH 058/131] fix issue with ssh submodule checkout

---
 src/git-fleximod | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index cbfa5f4f3..2ebb2e537 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -236,11 +236,28 @@ def submodule_checkout(root, name, path, url=None, tag=None):
             tmpurl = url
             url = url.replace("git@github.com:", "https://github.com")            
             git.git_operation("clone", url, path)
-            smgit = GitInterface(path, logger)
+            smgit = GitInterface(repodir, logger)
             if not tag:
                 tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
             smgit.git_operation("checkout",tag)
             # Now need to move the .git dir to the submodule location
+            rootdotgit = os.path.join(root,".git")
+            if os.path.isfile(rootdotgit):
+                with open(rootdotgit) as f:
+                    line = f.readline()
+                    if line.startswith("gitdir: "):
+                        rootdotgit = line[8:].rstrip()
+            
+            newpath = os.path.abspath(os.path.join(root,rootdotgit,"modules",path))
+            print(f"root is {root} rootdotgit is {rootdotgit} newpath is {newpath}")
+            if not os.path.isdir(os.path.join(newpath,os.pardir)):
+                os.makedirs(os.path.abspath(os.path.join(newpath,os.pardir)))
+
+            shutil.move(os.path.join(repodir,".git"), newpath) 
+            with open(os.path.join(repodir,".git"), "w") as f:
+                f.write("gitdir: "+newpath)
+            
+
             
             
     if not tmpurl:

From a12a5b051580b5f26048ab87ffe0521e4ba135ef Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 14:22:41 -0700
Subject: [PATCH 059/131] add --force option and cowardly refusal

---
 src/git-fleximod | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index 2ebb2e537..5489ae164 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -60,6 +60,13 @@ def commandline_arguments(args=None):
         nargs="*",
         help="Component(s) listed in the gitmodules file which should be ignored.",
     )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Override cautions and update or checkout over locally modified repository."
+    )
 
     parser.add_argument(
         "-o",
@@ -152,6 +159,7 @@ def commandline_arguments(args=None):
         options.components,
         options.exclude,
         options.verbose,
+        options.force,
         action,
     )
 
@@ -281,6 +289,7 @@ def submodule_checkout(root, name, path, url=None, tag=None):
 
 def submodules_status(gitmodules, root_dir):
     testfails = 0
+    localmods = 0
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
@@ -321,11 +330,17 @@ def submodules_status(gitmodules, root_dir):
 
                 status = git.git_operation("status","--ignore-submodules")
                 if "nothing to commit" not in status:
+                    localmods = localmods+1
                     print('M'+textwrap.indent(status,'                      '))
     
-    return testfails
+    return testfails, localmods
 
-def submodules_update(gitmodules, root_dir):
+def submodules_update(gitmodules, root_dir, force):
+    _,localmods = submodules_status(gitmodules, root_dir)
+    print("")
+    if localmods and not force:
+        print(f"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override")
+        return
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
@@ -358,15 +373,20 @@ def submodules_update(gitmodules, root_dir):
                     git.git_operation("fetch", newremote, "--tags")
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 if fxtag and fxtag != atag:
-                    print(f"Updating {name} to {fxtag}")
+                    print(f"{name:>20} updated to {fxtag}")
                     git.git_operation("checkout", fxtag)
                 elif not fxtag:
-                    print(f"No fxtag found for submodule {name}")
+                    print(f"No fxtag found for submodule {name:>20}")
                 else:
-                    print(f"submodule {name} up to date.")
+                    print(f"{name:>20} up to date.")
 
 
-def submodules_checkout(gitmodules, root_dir, requiredlist):
+def submodules_checkout(gitmodules, root_dir, requiredlist, force):
+    _,localmods = submodules_status(gitmodules, root_dir)
+    print("")
+    if localmods and not force:
+        print(f"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override")
+        return
     for name in gitmodules.sections():
         fxrequired = gitmodules.get(name, "fxrequired")
         fxsparse = gitmodules.get(name, "fxsparse")
@@ -393,7 +413,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist):
 
 def submodules_test(gitmodules, root_dir):
     # First check that fxtags are present and in sync with submodule hashes
-    testfails = submodules_status(gitmodules, root_dir)
+    testfails,localmods = submodules_status(gitmodules, root_dir)
     # Then make sure that urls are consistant with fxurls (not forks and not ssh)
     # and that sparse checkout files exist
     for name in gitmodules.sections():
@@ -407,7 +427,7 @@ def submodules_test(gitmodules, root_dir):
         if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
             print(f"sparse submodule {name} sparse checkout file {fxsparse} not found")
             testfails += 1
-    return testfails
+    return testfails+localmods
 
 
             
@@ -419,6 +439,7 @@ def _main_func():
         includelist,
         excludelist,
         verbose,
+        force,
         action,
     ) = commandline_arguments()
     # Get a logger for the package
@@ -445,9 +466,9 @@ def _main_func():
     )
     retval = 0
     if action == "update":
-        submodules_update(gitmodules, root_dir)
+        submodules_update(gitmodules, root_dir, force)
     elif action == "checkout":
-        submodules_checkout(gitmodules, root_dir, fxrequired)
+        submodules_checkout(gitmodules, root_dir, fxrequired, force)
     elif action == "status":
         submodules_status(gitmodules, root_dir)
     elif action == "test":

From ce22059f0f860c9e9f0961ee2637d21d967facfb Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 14:37:08 -0700
Subject: [PATCH 060/131] start at root git dir unless --path flag is used

---
 src/git-fleximod | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index 5489ae164..d0e6149f2 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -5,6 +5,7 @@ import shutil
 import logging
 import argparse
 import textwrap
+from pathlib import Path
 from fleximod import utils
 from fleximod.gitinterface import GitInterface
 from fleximod.gitmodules import GitModules
@@ -13,6 +14,17 @@ from configparser import NoOptionError
 # logger variable is global
 logger = None
 
+def find_root_dir(filename=".git"):
+    d = Path.cwd()
+    root = Path(d.root)
+    while d != root:
+        attempt = d / filename
+        if attempt.is_dir():
+            return attempt
+        d = d.parent
+    return None
+    
+
 def commandline_arguments(args=None):
     description = """
     %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
@@ -42,8 +54,8 @@ def commandline_arguments(args=None):
     parser.add_argument(
         "-C",
         "--path",
-        default=os.getcwd(),
-        help="Toplevel repository directory.  Defaults to current directory.",
+        default=find_root_dir(),
+        help="Toplevel repository directory.  Defaults to top git directory relative to current.",
     )
 
     parser.add_argument(
@@ -430,6 +442,7 @@ def submodules_test(gitmodules, root_dir):
     return testfails+localmods
 
 
+
             
 def _main_func():
     (
@@ -455,6 +468,7 @@ def _main_func():
             utils.fatal_error(
                 "No {} found in {} or any of it's parents".format(file_name, root_dir)
             )
+
         root_dir = os.path.dirname(file_path)
     logger.info(f"root_dir is {root_dir}")
     gitmodules = GitModules(

From 8b7267959971c16d66803d92faa833055a8ff213 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 1 Feb 2024 14:43:10 -0700
Subject: [PATCH 061/131] remove test from visable options

---
 src/git-fleximod | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index d0e6149f2..da8431a17 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -41,7 +41,7 @@ def commandline_arguments(args=None):
         "action",
         choices=choices,
         default="checkout",
-        help=f"Subcommand of fleximod, choices are {choices}",
+        help=f"Subcommand of fleximod, choices are {choices[:-1]}",
     )
 
     parser.add_argument(

From a990864cef359b6a2f6e9a75f420ec05f669f894 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 17:34:21 -0700
Subject: [PATCH 062/131] update setup.cfg

---
 setup.cfg | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 9a13827a3..599646552 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,4 +2,7 @@
 license_file = License
 [build_manpages]
 manpages =
-    man/git-fleximod.1:object=parser:pyfile=bin/git-fleximod
+    # solution 1: --split up the commandline_arguments parser
+    #man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
+    # solution 2 -- make parser global
+    man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From d2123b6022ad648ca78241a155fed93d13642ff4 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 17:34:43 -0700
Subject: [PATCH 063/131] updates for setup.py

---
 setup.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index f68a4ea9b..53cc4e43e 100644
--- a/setup.py
+++ b/setup.py
@@ -2,6 +2,8 @@
 import os
 from setuptools import setup, find_packages
 from distutils.util import convert_path
+from setuptools.command.build_py import build_py
+from setuptools.command.install import install
 from build_manpages import build_manpages, get_build_py_cmd, get_install_cmd
 
 with open("README.md", "r") as fh:
@@ -14,7 +16,8 @@
 
     
 setuptools.setup(
-    scripts=["src/git-fleximod"],                     # This is the name of the package
+    name="git-fleximod",                      # package name
+    #scripts=["src/git-fleximod"],                     # This is the name of the package
     version=main_ns['__version__'],                        # The initial release version
     author="Jim Edwards",                     # Full name of the author
     maintainer="jedwards4b",
@@ -38,6 +41,6 @@
       # Re-define build_py and install commands so the manual pages
       # are automatically re-generated and installed
       'build_py': get_build_py_cmd(),
-      'install': get_install_cmd(),
+      'install': get_install_cmd(install),
   }
 )

From 89dfbbe5e6a6d2ba532d4ec8a05ddd83ed3abb79 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 17:39:19 -0700
Subject: [PATCH 064/131] setup.cfg

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 599646552..8df52e87a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,7 +2,7 @@
 license_file = License
 [build_manpages]
 manpages =
-    # solution 1: --split up the commandline_arguments parser
+    # solution 1: --split up the commandline_arguments parser --better
     #man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
     # solution 2 -- make parser global
     man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From 3fdff62ff545bf6fda5c7e475e7fc2c98f1e1f12 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 17:47:53 -0700
Subject: [PATCH 065/131] pointing to the better solution

---
 setup.cfg | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 8df52e87a..7bd0d34d1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,7 +2,7 @@
 license_file = License
 [build_manpages]
 manpages =
-    # solution 1: --split up the commandline_arguments parser --better
-    #man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
+    # solution 1: --split up the commandline_arguments parser
+    man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
     # solution 2 -- make parser global
-    man/git-fleximod.2:pyfile=src/git-fleximod:object=parser
+    #man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From 49e8fadbf834e1c03effd933e7bd5698b7555cc0 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 18:11:22 -0700
Subject: [PATCH 066/131] minimal changes required

---
 src/git-fleximod | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index d0e6149f2..26c00f551 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -24,8 +24,7 @@ def find_root_dir(filename=".git"):
         d = d.parent
     return None
     
-
-def commandline_arguments(args=None):
+def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
     """
@@ -127,6 +126,11 @@ def commandline_arguments(args=None):
         "information to the screen and log file.",
     )
 
+    return parser
+
+def commandline_arguments(args=None):
+    parser = get_parser()
+
     if args:
         options = parser.parse_args(args)
     else:

From 8c09b5b016629660613f0c0acb3889c5b8d5209b Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 18:22:59 -0700
Subject: [PATCH 067/131] updates to setup.cfg

---
 setup.cfg | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 7bd0d34d1..f6b737006 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,7 +2,4 @@
 license_file = License
 [build_manpages]
 manpages =
-    # solution 1: --split up the commandline_arguments parser
     man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
-    # solution 2 -- make parser global
-    #man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From a9bd103363ec9fe041daf4ee36bc915fe7209e15 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 18:23:22 -0700
Subject: [PATCH 068/131] minor changes

---
 setup.cfg | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.cfg b/setup.cfg
index f6b737006..7bd0d34d1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,4 +2,7 @@
 license_file = License
 [build_manpages]
 manpages =
+    # solution 1: --split up the commandline_arguments parser
     man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
+    # solution 2 -- make parser global
+    #man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From 3b82247ef72c9c3f2f9e4d5697bbb735bb7181d2 Mon Sep 17 00:00:00 2001
From: Negin Sobhani <negin513@gmail.com>
Date: Thu, 1 Feb 2024 18:26:19 -0700
Subject: [PATCH 069/131] fix syntax and new function name

---
 setup.cfg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 7bd0d34d1..66ff24ada 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,7 +2,7 @@
 license_file = License
 [build_manpages]
 manpages =
-    # solution 1: --split up the commandline_arguments parser
-    man/git-fleximod.1:function:commandline_arguments:pyfile=src/git-fleximod
+    # solution 1: -- use parser function
+    man/git-fleximod.1:function=get_parser:pyfile=src/git-fleximod
     # solution 2 -- make parser global
     #man/git-fleximod.2:pyfile=src/git-fleximod:object=parser

From 23e0813eb3340dd410e989a48a7b753a69d0ae74 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 07:21:14 -0700
Subject: [PATCH 070/131] names were too similar, force needs to be optional

---
 src/git-fleximod | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index 1a55e9168..984ae9e1d 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -241,7 +241,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     print(f"Successfully checked out {name}")
 
 
-def submodule_checkout(root, name, path, url=None, tag=None):
+def single_submodule_checkout(root, name, path, url=None, tag=None, force=False):
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
     if os.path.exists(os.path.join(repodir, ".git")):
@@ -291,7 +291,7 @@ def submodule_checkout(root, name, path, url=None, tag=None):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir} {url}")
         gitmodules = GitModules(logger,confpath=repodir)
-        submodules_checkout(gitmodules, repodir, ["I:T"])
+        submodules_checkout(gitmodules, repodir, ["I:T"], force=force)
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
     else:
@@ -397,7 +397,7 @@ def submodules_update(gitmodules, root_dir, force):
                     print(f"{name:>20} up to date.")
 
 
-def submodules_checkout(gitmodules, root_dir, requiredlist, force):
+def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
     _,localmods = submodules_status(gitmodules, root_dir)
     print("")
     if localmods and not force:
@@ -425,7 +425,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force):
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
             
-            submodule_checkout(root_dir, name, path, url=url, tag=fxtag)
+            single_submodule_checkout(root_dir, name, path, url=url, tag=fxtag, force=force)
 
 def submodules_test(gitmodules, root_dir):
     # First check that fxtags are present and in sync with submodule hashes

From ae4c9c5962fe0ddbcc8043d9a647115dfb7e89e8 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 09:30:33 -0700
Subject: [PATCH 071/131] more cleanup

---
 setup.py         | 12 ++++++++----
 src/git-fleximod | 12 ++++++++----
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index 53cc4e43e..259f9589c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,8 @@
-import setuptools
+import sys
 import os
+sys.path.insert(0,os.path.join(os.getenv("CONDA_PREFIX"),"lib","python3.12","site-packages"))
+
+import setuptools
 from setuptools import setup, find_packages
 from distutils.util import convert_path
 from setuptools.command.build_py import build_py
@@ -17,7 +20,7 @@
     
 setuptools.setup(
     name="git-fleximod",                      # package name
-    #scripts=["src/git-fleximod"],                     # This is the name of the package
+    scripts=["src/git-fleximod"],                     # This is the name of the package
     version=main_ns['__version__'],                        # The initial release version
     author="Jim Edwards",                     # Full name of the author
     maintainer="jedwards4b",
@@ -25,8 +28,9 @@
     description="Extended support for git-submodule and git-sparse-checkout",
     long_description=long_description,      # Long description read from the the readme file
     long_description_content_type="text/markdown",
-    packages=['fleximod'],    # List of all python modules to be installed
-    package_dir={'fleximod': 'src/fleximod'},
+    packages=find_packages(),    # List of all python modules to be installed
+    package_dir={'git-fleximod': 'src',
+                 'fleximod': 'src/fleximod'},
     package_data={"":['version.txt']},
     classifiers=[
         "Programming Language :: Python :: 3",
diff --git a/src/git-fleximod b/src/git-fleximod
index 984ae9e1d..0706a5623 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -273,7 +273,6 @@ def single_submodule_checkout(root, name, path, url=None, tag=None, force=False)
                         rootdotgit = line[8:].rstrip()
             
             newpath = os.path.abspath(os.path.join(root,rootdotgit,"modules",path))
-            print(f"root is {root} rootdotgit is {rootdotgit} newpath is {newpath}")
             if not os.path.isdir(os.path.join(newpath,os.pardir)):
                 os.makedirs(os.path.abspath(os.path.join(newpath,os.pardir)))
 
@@ -321,14 +320,17 @@ def submodules_status(gitmodules, root_dir):
             tags = rootgit.git_operation("ls-remote","--tags",url)
             atag = None
             for htag in tags.split('\n'):
-                if tag in htag:
+                if tag and tag in htag:
                     atag = (htag.split()[1])[10:]
                     break
-            if tag == atag:
+            if tag and tag == atag:
                 print(f"e {name:>20} not checked out, aligned at tag {tag}")
-            else:
+            elif tag:
                 print(f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}")
                 testfails += 1
+            else:
+                print(f"e {name:>20} has no fxtag defined in .gitmodules")
+                testfails +=1 
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
@@ -482,6 +484,8 @@ def _main_func():
         includelist=includelist,
         excludelist=excludelist,
     )
+    if not gitmodules.sections():
+        sys.exit("No submodule components found")
     retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir, force)

From 140529afedf13f88ba3673526917d6dd27750c3e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 10:00:15 -0700
Subject: [PATCH 072/131] pre-commit cleanup

---
 setup.py                     |  51 ++++++------
 src/fleximod/gitinterface.py |  15 ++--
 src/fleximod/gitmodules.py   |  17 ++--
 src/fleximod/lstripreader.py |   7 +-
 src/fleximod/version.py      |   2 +-
 src/git-fleximod             | 150 +++++++++++++++++++----------------
 6 files changed, 129 insertions(+), 113 deletions(-)

diff --git a/setup.py b/setup.py
index 259f9589c..8f8a49cbe 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,13 @@
 import sys
 import os
-sys.path.insert(0,os.path.join(os.getenv("CONDA_PREFIX"),"lib","python3.12","site-packages"))
+
+sys.path.insert(
+    0, os.path.join(os.getenv("CONDA_PREFIX"), "lib", "python3.12", "site-packages")
+)
 
 import setuptools
-from setuptools import setup, find_packages
-from distutils.util import convert_path
-from setuptools.command.build_py import build_py
+from setuptools import find_packages
+from setuptools.util import convert_path
 from setuptools.command.install import install
 from build_manpages import build_manpages, get_build_py_cmd, get_install_cmd
 
@@ -13,38 +15,37 @@
     long_description = fh.read()
 
 main_ns = {}
-ver_path = convert_path('src/fleximod/version.py')
+ver_path = convert_path("src/fleximod/version.py")
 with open(ver_path) as ver_file:
     exec(ver_file.read(), main_ns)
 
-    
+
 setuptools.setup(
-    name="git-fleximod",                      # package name
-    scripts=["src/git-fleximod"],                     # This is the name of the package
-    version=main_ns['__version__'],                        # The initial release version
-    author="Jim Edwards",                     # Full name of the author
+    name="git-fleximod",  # package name
+    scripts=["src/git-fleximod"],  # This is the name of the package
+    version=main_ns["__version__"],  # The initial release version
+    author="Jim Edwards",  # Full name of the author
     maintainer="jedwards4b",
     license="MIT License",
     description="Extended support for git-submodule and git-sparse-checkout",
-    long_description=long_description,      # Long description read from the the readme file
+    long_description=long_description,  # Long description read from the the readme file
     long_description_content_type="text/markdown",
-    packages=find_packages(),    # List of all python modules to be installed
-    package_dir={'git-fleximod': 'src',
-                 'fleximod': 'src/fleximod'},
-    package_data={"":['version.txt']},
+    packages=find_packages(),  # List of all python modules to be installed
+    package_dir={"git-fleximod": "src", "fleximod": "src/fleximod"},
+    package_data={"": ["version.txt"]},
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
-    ],                                      # Information to filter the project on PyPi website
-    python_requires='>=3.6',                # Minimum version requirement of the package
-#    py_modules=['git-fleximod'],             # Name of the python package
-    install_requires=["GitPython"],                     # Install other dependencies if any
+    ],  # Information to filter the project on PyPi website
+    python_requires=">=3.6",  # Minimum version requirement of the package
+    #    py_modules=['git-fleximod'],             # Name of the python package
+    install_requires=["GitPython"],  # Install other dependencies if any
     cmdclass={
-      'build_manpages': build_manpages,
-      # Re-define build_py and install commands so the manual pages
-      # are automatically re-generated and installed
-      'build_py': get_build_py_cmd(),
-      'install': get_install_cmd(install),
-  }
+        "build_manpages": build_manpages,
+        # Re-define build_py and install commands so the manual pages
+        # are automatically re-generated and installed
+        "build_py": get_build_py_cmd(),
+        "install": get_install_cmd(install),
+    },
 )
diff --git a/src/fleximod/gitinterface.py b/src/fleximod/gitinterface.py
index c127163dc..96bb9ac52 100644
--- a/src/fleximod/gitinterface.py
+++ b/src/fleximod/gitinterface.py
@@ -1,7 +1,7 @@
 import os
-import logging
 from fleximod import utils
 
+
 class GitInterface:
     def __init__(self, repo_path, logger):
         logger.debug("Initialize GitInterface for {}".format(repo_path))
@@ -9,6 +9,7 @@ def __init__(self, repo_path, logger):
         self.logger = logger
         try:
             import git
+
             self._use_module = True
             try:
                 self.repo = git.Repo(repo_path)  # Initialize GitPython repo
@@ -18,17 +19,17 @@ def __init__(self, repo_path, logger):
             msg = "Using GitPython interface to git"
         except ImportError:
             self._use_module = False
-            if not os.path.exists(os.path.join(repo_path,".git")):
+            if not os.path.exists(os.path.join(repo_path, ".git")):
                 self._init_git_repo()
             msg = "Using shell interface to git"
         self.logger.info(msg)
-                
+
     def _git_command(self, operation, *args):
         self.logger.info(operation)
         if self._use_module and operation != "submodule":
             return getattr(self.repo.git, operation)(*args)
         else:
-            return ["git", "-C",self.repo_path, operation] + list(args)
+            return ["git", "-C", self.repo_path, operation] + list(args)
 
     def _init_git_repo(self):
         if self._use_module:
@@ -37,7 +38,7 @@ def _init_git_repo(self):
             command = ("git", "-C", self.repo_path, "init")
             utils.execute_subprocess(command)
 
-
+    # pylint: disable=unused-argument
     def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *args)
         self.logger.info(command)
@@ -51,7 +52,7 @@ def config_get_value(self, section, name):
             config = self.repo.config_reader()
             return config.get_value(section, name)
         else:
-            cmd = ("git","-C",self.repo_path,"config", "--get", f"{section}.{name}")
+            cmd = ("git", "-C", self.repo_path, "config", "--get", f"{section}.{name}")
             output = utils.execute_subprocess(cmd, output_to_caller=True)
             return output.strip()
 
@@ -61,6 +62,6 @@ def config_set_value(self, section, name, value):
                 writer.set_value(section, name, value)
             writer.release()  # Ensure changes are saved
         else:
-            cmd = ("git","-C",self.repo_path,"config", f"{section}.{name}", value)
+            cmd = ("git", "-C", self.repo_path, "config", f"{section}.{name}", value)
             self.logger.info(cmd)
             utils.execute_subprocess(cmd, output_to_caller=True)
diff --git a/src/fleximod/gitmodules.py b/src/fleximod/gitmodules.py
index a6f758931..11a245724 100644
--- a/src/fleximod/gitmodules.py
+++ b/src/fleximod/gitmodules.py
@@ -3,6 +3,7 @@
 from configparser import ConfigParser
 from fleximod.lstripreader import LstripReader
 
+
 class GitModules(ConfigParser):
     def __init__(
         self,
@@ -19,11 +20,15 @@ def __init__(
         excludelist: Optional list of submodules to exclude.
         """
         self.logger = logger
-        self.logger.debug("Creating a GitModules object {} {} {} {}".format(confpath,conffile,includelist,excludelist))
+        self.logger.debug(
+            "Creating a GitModules object {} {} {} {}".format(
+                confpath, conffile, includelist, excludelist
+            )
+        )
         ConfigParser.__init__(self)
         self.conf_file = os.path.join(confpath, conffile)
         # first create a backup of this file to be restored on deletion of the object
-        shutil.copy(self.conf_file, self.conf_file+".save")
+        shutil.copy(self.conf_file, self.conf_file + ".save")
         self.read_file(LstripReader(self.conf_file), source=conffile)
         self.includelist = includelist
         self.excludelist = excludelist
@@ -34,7 +39,7 @@ def set(self, name, option, value):
         Ensures the appropriate section exists for the submodule.
         Calls the parent class's set method to store the value.
         """
-        self.logger.debug("set called {} {} {}".format(name,option,value))
+        self.logger.debug("set called {} {} {}".format(name, option, value))
         section = f'submodule "{name}"'
         if not self.has_section(section):
             self.add_section(section)
@@ -47,7 +52,7 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
         Uses the parent class's get method to access the value.
         Handles potential errors if the section or option doesn't exist.
         """
-        self.logger.debug("get called {} {}".format(name,option))
+        self.logger.debug("get called {} {}".format(name, option))
         section = f'submodule "{name}"'
         try:
             return ConfigParser.get(
@@ -62,8 +67,8 @@ def save(self):
 
     def __del__(self):
         self.logger.debug("Destroying GitModules object")
-        shutil.move(self.conf_file+".save", self.conf_file)
-            
+        shutil.move(self.conf_file + ".save", self.conf_file)
+
     def sections(self):
         """Strip the submodule part out of section and just use the name"""
         self.logger.debug("calling GitModules sections iterator")
diff --git a/src/fleximod/lstripreader.py b/src/fleximod/lstripreader.py
index 530abd297..01d5580ee 100644
--- a/src/fleximod/lstripreader.py
+++ b/src/fleximod/lstripreader.py
@@ -1,8 +1,8 @@
-
 class LstripReader(object):
     "LstripReader formats .gitmodules files to be acceptable for configparser"
+
     def __init__(self, filename):
-        with open(filename, 'r') as infile:
+        with open(filename, "r") as infile:
             lines = infile.readlines()
         self._lines = list()
         self._num_lines = len(lines)
@@ -19,7 +19,7 @@ def readline(self, size=-1):
         try:
             line = self.next()
         except StopIteration:
-            line = ''
+            line = ""
 
         if (size > 0) and (len(line) < size):
             return line[0:size]
@@ -41,4 +41,3 @@ def next(self):
 
     def __next__(self):
         return self.next()
-
diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index 7fd229a32..d3ec452c3 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = '0.2.0'
+__version__ = "0.2.0"
diff --git a/src/git-fleximod b/src/git-fleximod
index 0706a5623..aa587ea3a 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -11,9 +11,11 @@ from fleximod.gitinterface import GitInterface
 from fleximod.gitmodules import GitModules
 from fleximod.version import __version__
 from configparser import NoOptionError
+
 # logger variable is global
 logger = None
 
+
 def find_root_dir(filename=".git"):
     d = Path.cwd()
     root = Path(d.root)
@@ -23,7 +25,8 @@ def find_root_dir(filename=".git"):
             return attempt
         d = d.parent
     return None
-    
+
+
 def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
@@ -76,7 +79,7 @@ def get_parser():
         "--force",
         action="store_true",
         default=False,
-        help="Override cautions and update or checkout over locally modified repository."
+        help="Override cautions and update or checkout over locally modified repository.",
     )
 
     parser.add_argument(
@@ -128,6 +131,7 @@ def get_parser():
 
     return parser
 
+
 def commandline_arguments(args=None):
     parser = get_parser()
 
@@ -136,7 +140,7 @@ def commandline_arguments(args=None):
     else:
         options = parser.parse_args()
 
-# explicitly listing a component overrides the optional flag    
+    # explicitly listing a component overrides the optional flag
     if options.optional or options.components:
         fxrequired = ["T:T", "T:F", "I:T"]
     else:
@@ -145,11 +149,11 @@ def commandline_arguments(args=None):
     action = options.action
     if not action:
         action = "checkout"
-    handlers=[logging.StreamHandler()]
+    handlers = [logging.StreamHandler()]
 
     if options.debug:
         try:
-            open("fleximod.log","w")
+            open("fleximod.log", "w")
         except PermissionError:
             sys.exit("ABORT: Could not write file fleximod.log")
         level = logging.DEBUG
@@ -160,21 +164,18 @@ def commandline_arguments(args=None):
         level = logging.WARNING
     # Configure the root logger
     logging.basicConfig(
-        level=level,
-        format="%(name)s - %(levelname)s - %(message)s",
-        handlers=handlers
+        level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers
     )
-    
-    if hasattr(options, 'version'):
+
+    if hasattr(options, "version"):
         exit()
-    
+
     return (
         options.path,
         options.gitmodules,
         fxrequired,
         options.components,
         options.exclude,
-        options.verbose,
         options.force,
         action,
     )
@@ -191,7 +192,7 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # initialize a new git repo and set the sparse checkout flag
     sprep_repo = os.path.join(root_dir, path)
     sprepo_git = GitInterface(sprep_repo, logger)
-    if os.path.exists(os.path.join(sprep_repo,".git")):
+    if os.path.exists(os.path.join(sprep_repo, ".git")):
         try:
             logger.info("Submodule {} found".format(name))
             chk = sprepo_git.config_get_value("core", "sparseCheckout")
@@ -203,7 +204,6 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
         except Exception as e:
             utils.fatal_error("Unexpected error {} occured.".format(e))
 
-
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
 
     # set the repository remote
@@ -212,28 +212,31 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     superroot = git.git_operation("rev-parse", "--show-superproject-working-tree")
     if os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
-            gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1]))
+            gitpath = os.path.abspath(os.path.join(root_dir, f.read().split()[1]))
         topgit = os.path.abspath(os.path.join(gitpath, "modules"))
-    else:                      
+    else:
         topgit = os.path.abspath(os.path.join(root_dir, ".git", "modules"))
 
     if not os.path.isdir(topgit):
         os.makedirs(topgit)
     topgit = os.path.join(topgit, name)
-    logger.debug(f"root_dir is {root_dir} topgit is {topgit} superroot is {superroot}")
+    logger.debug(
+        "root_dir is {} topgit is {} superroot is {}".format(
+            root_dir, topgit, superroot
+        )
+    )
+
+    if os.path.isdir(os.path.join(root_dir, path, ".git")):
+        shutil.move(os.path.join(root_dir, path, ".git"), topgit)
+        with open(os.path.join(root_dir, path, ".git"), "w") as f:
+            f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir, path)))
 
-    if os.path.isdir(os.path.join(root_dir,path,".git")):
-        shutil.move(os.path.join(root_dir,path, ".git"), topgit)
-        with open(os.path.join(root_dir,path, ".git"), "w") as f:
-            f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir,path)))
-            
     gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
     if os.path.isfile(gitsparse):
         logger.warning("submodule {} is already initialized".format(name))
         return
 
-
-    shutil.copy(os.path.join(root_dir,path, sparsefile), gitsparse)
+    shutil.copy(os.path.join(root_dir, path, sparsefile), gitsparse)
 
     # Finally checkout the repo
     sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
@@ -258,38 +261,35 @@ def single_submodule_checkout(root, name, path, url=None, tag=None, force=False)
         # it will be done by the GitModules class
         if url.startswith("git@"):
             tmpurl = url
-            url = url.replace("git@github.com:", "https://github.com")            
+            url = url.replace("git@github.com:", "https://github.com")
             git.git_operation("clone", url, path)
             smgit = GitInterface(repodir, logger)
             if not tag:
                 tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
-            smgit.git_operation("checkout",tag)
+            smgit.git_operation("checkout", tag)
             # Now need to move the .git dir to the submodule location
-            rootdotgit = os.path.join(root,".git")
+            rootdotgit = os.path.join(root, ".git")
             if os.path.isfile(rootdotgit):
                 with open(rootdotgit) as f:
                     line = f.readline()
                     if line.startswith("gitdir: "):
                         rootdotgit = line[8:].rstrip()
-            
-            newpath = os.path.abspath(os.path.join(root,rootdotgit,"modules",path))
-            if not os.path.isdir(os.path.join(newpath,os.pardir)):
-                os.makedirs(os.path.abspath(os.path.join(newpath,os.pardir)))
-
-            shutil.move(os.path.join(repodir,".git"), newpath) 
-            with open(os.path.join(repodir,".git"), "w") as f:
-                f.write("gitdir: "+newpath)
-            
-
-            
-            
+
+            newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", path))
+            if not os.path.isdir(os.path.join(newpath, os.pardir)):
+                os.makedirs(os.path.abspath(os.path.join(newpath, os.pardir)))
+
+            shutil.move(os.path.join(repodir, ".git"), newpath)
+            with open(os.path.join(repodir, ".git"), "w") as f:
+                f.write("gitdir: " + newpath)
+
     if not tmpurl:
         logger.debug(git.git_operation("submodule", "update", "--init", "--", path))
 
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir} {url}")
-        gitmodules = GitModules(logger,confpath=repodir)
+        gitmodules = GitModules(logger, confpath=repodir)
         submodules_checkout(gitmodules, repodir, ["I:T"], force=force)
     if os.path.exists(os.path.join(repodir, ".git")):
         print(f"Successfully checked out {name}")
@@ -315,22 +315,23 @@ def submodules_status(gitmodules, root_dir):
         if not os.path.exists(os.path.join(newpath, ".git")):
             rootgit = GitInterface(root_dir, logger)
             # submodule commands use path, not name
-            nhash = (rootgit.git_operation("submodule","status",path).split()[0])[1:]
             url = gitmodules.get(name, "url")
-            tags = rootgit.git_operation("ls-remote","--tags",url)
+            tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
-            for htag in tags.split('\n'):
+            for htag in tags.split("\n"):
                 if tag and tag in htag:
                     atag = (htag.split()[1])[10:]
                     break
             if tag and tag == atag:
                 print(f"e {name:>20} not checked out, aligned at tag {tag}")
             elif tag:
-                print(f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}")
+                print(
+                    f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}"
+                )
                 testfails += 1
             else:
                 print(f"e {name:>20} has no fxtag defined in .gitmodules")
-                testfails +=1 
+                testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
@@ -346,24 +347,27 @@ def submodules_status(gitmodules, root_dir):
                     )
                     testfails += 1
 
-                status = git.git_operation("status","--ignore-submodules")
+                status = git.git_operation("status", "--ignore-submodules")
                 if "nothing to commit" not in status:
-                    localmods = localmods+1
-                    print('M'+textwrap.indent(status,'                      '))
-    
+                    localmods = localmods + 1
+                    print("M" + textwrap.indent(status, "                      "))
+
     return testfails, localmods
 
+
 def submodules_update(gitmodules, root_dir, force):
-    _,localmods = submodules_status(gitmodules, root_dir)
+    _, localmods = submodules_status(gitmodules, root_dir)
     print("")
     if localmods and not force:
-        print(f"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override")
+        print(
+            "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
+        )
         return
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-        logger.info(f"name={name} path={path} url={url} fxtag={fxtag}")
+        logger.info("name={} path={} url={} fxtag={}".format(name, path, url, fxtag))
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
@@ -400,10 +404,12 @@ def submodules_update(gitmodules, root_dir, force):
 
 
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
-    _,localmods = submodules_status(gitmodules, root_dir)
+    _, localmods = submodules_status(gitmodules, root_dir)
     print("")
     if localmods and not force:
-        print(f"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override")
+        print(
+            "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
+        )
         return
     for name in gitmodules.sections():
         fxrequired = gitmodules.get(name, "fxrequired")
@@ -416,22 +422,28 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
             if "T:F" == fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
-            
+
         if fxsparse:
             logger.debug(
-                f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {fxsparse}, {fxtag}"
+                "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format(
+                    root_dir, name, url, path, fxsparse, fxtag
+                )
             )
             submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag)
         else:
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
-            
-            single_submodule_checkout(root_dir, name, path, url=url, tag=fxtag, force=force)
+
+            single_submodule_checkout(
+                root_dir, name, path, url=url, tag=fxtag, force=force
+            )
+
 
 def submodules_test(gitmodules, root_dir):
     # First check that fxtags are present and in sync with submodule hashes
-    testfails,localmods = submodules_status(gitmodules, root_dir)
+    testfails, localmods = submodules_status(gitmodules, root_dir)
+    print("")
     # Then make sure that urls are consistant with fxurls (not forks and not ssh)
     # and that sparse checkout files exist
     for name in gitmodules.sections():
@@ -440,16 +452,14 @@ def submodules_test(gitmodules, root_dir):
         fxsparse = gitmodules.get(name, "fxsparse")
         path = gitmodules.get(name, "path")
         if not fxurl or url != fxurl:
-            print(f"submodule {name} url {url} not in sync with required {fxurl}")
+            print(f"{name:>20} url {url} not in sync with required {fxurl}")
             testfails += 1
         if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
-            print(f"sparse submodule {name} sparse checkout file {fxsparse} not found")
+            print(f"{name:>20} sparse checkout file {fxsparse} not found")
             testfails += 1
-    return testfails+localmods
-
+    return testfails + localmods
 
 
-            
 def _main_func():
     (
         root_dir,
@@ -457,15 +467,14 @@ def _main_func():
         fxrequired,
         includelist,
         excludelist,
-        verbose,
         force,
         action,
     ) = commandline_arguments()
     # Get a logger for the package
     global logger
     logger = logging.getLogger(__name__)
-        
-    logger.info(f"action is {action}")
+
+    logger.info("action is {}".format(action))
 
     if not os.path.isfile(os.path.join(root_dir, file_name)):
         file_path = utils.find_upwards(root_dir, file_name)
@@ -476,7 +485,7 @@ def _main_func():
             )
 
         root_dir = os.path.dirname(file_path)
-    logger.info(f"root_dir is {root_dir}")
+    logger.info("root_dir is {}".format(root_dir))
     gitmodules = GitModules(
         logger,
         confpath=root_dir,
@@ -497,7 +506,8 @@ def _main_func():
         retval = submodules_test(gitmodules, root_dir)
     else:
         utils.fatal_error(f"unrecognized action request {action}")
-    return(retval)
-        
+    return retval
+
+
 if __name__ == "__main__":
     sys.exit(_main_func())

From 0a5cf50fb696f84ea5a71bcdb973a8972c14bb55 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 12:43:54 -0700
Subject: [PATCH 073/131] setup.py builds manpage but doesnt install

---
 .github/workflows/pre-commit | 13 ++++++
 setup.py                     | 76 +++++++++++++++++++-----------------
 2 files changed, 53 insertions(+), 36 deletions(-)
 create mode 100644 .github/workflows/pre-commit

diff --git a/.github/workflows/pre-commit b/.github/workflows/pre-commit
new file mode 100644
index 000000000..1a6ad0082
--- /dev/null
+++ b/.github/workflows/pre-commit
@@ -0,0 +1,13 @@
+name: pre-commit
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/action@v3.0.0
diff --git a/setup.py b/setup.py
index 8f8a49cbe..048db7595 100644
--- a/setup.py
+++ b/setup.py
@@ -1,51 +1,55 @@
 import sys
 import os
 
+sys.path.insert(0, os.path.join(os.getenv("CONDA_PREFIX"), "lib", "python3.12"))
 sys.path.insert(
     0, os.path.join(os.getenv("CONDA_PREFIX"), "lib", "python3.12", "site-packages")
 )
 
 import setuptools
-from setuptools import find_packages
-from setuptools.util import convert_path
-from setuptools.command.install import install
+from setuptools import find_packages, convert_path
 from build_manpages import build_manpages, get_build_py_cmd, get_install_cmd
 
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
 main_ns = {}
-ver_path = convert_path("src/fleximod/version.py")
+ver_path = convert_path(
+    os.path.join(os.path.dirname(__file__), "src", "fleximod", "version.py")
+)
+print(f"ver_path is {ver_path}")
 with open(ver_path) as ver_file:
     exec(ver_file.read(), main_ns)
 
+if __name__ == "__main__":
+    with open("README.md", "r") as fh:
+        long_description = fh.read()
 
-setuptools.setup(
-    name="git-fleximod",  # package name
-    scripts=["src/git-fleximod"],  # This is the name of the package
-    version=main_ns["__version__"],  # The initial release version
-    author="Jim Edwards",  # Full name of the author
-    maintainer="jedwards4b",
-    license="MIT License",
-    description="Extended support for git-submodule and git-sparse-checkout",
-    long_description=long_description,  # Long description read from the the readme file
-    long_description_content_type="text/markdown",
-    packages=find_packages(),  # List of all python modules to be installed
-    package_dir={"git-fleximod": "src", "fleximod": "src/fleximod"},
-    package_data={"": ["version.txt"]},
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],  # Information to filter the project on PyPi website
-    python_requires=">=3.6",  # Minimum version requirement of the package
-    #    py_modules=['git-fleximod'],             # Name of the python package
-    install_requires=["GitPython"],  # Install other dependencies if any
-    cmdclass={
-        "build_manpages": build_manpages,
-        # Re-define build_py and install commands so the manual pages
-        # are automatically re-generated and installed
-        "build_py": get_build_py_cmd(),
-        "install": get_install_cmd(install),
-    },
-)
+    setuptools.setup(
+        name="git-fleximod",  # package name
+        scripts=["src/git-fleximod"],  # This is the name of the package
+        version=main_ns["__version__"],  # The initial release version
+        author="Jim Edwards",  # Full name of the author
+        maintainer="jedwards4b",
+        license="MIT License",
+        description="Extended support for git-submodule and git-sparse-checkout",
+        long_description=long_description,  # Long description read from the the readme file
+        long_description_content_type="text/markdown",
+        packages=find_packages(
+            where="src"
+        ),  # List of all python modules to be installed
+        package_dir={"": "src", "git-fleximod": "src", "fleximod": "src/fleximod"},
+        package_data={"": ["src/fleximod/version.txt", "man/git-fleximod.1"]},
+        classifiers=[
+            "Programming Language :: Python :: 3",
+            "License :: OSI Approved :: MIT License",
+            "Operating System :: OS Independent",
+        ],  # Information to filter the project on PyPi website
+        python_requires=">=3.6",  # Minimum version requirement of the package
+        #    py_modules=['git-fleximod'],             # Name of the python package
+        install_requires=["GitPython"],  # Install other dependencies if any
+        cmdclass={
+            "build_manpages": build_manpages,
+            # Re-define build_py and install commands so the manual pages
+            # are automatically re-generated and installed
+            "build_py": get_build_py_cmd(),
+            "install": get_install_cmd(),
+        },
+    )

From 64e26fc22df3c3faa7190eff289dce5accc88669 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 12:46:58 -0700
Subject: [PATCH 074/131] update version

---
 src/fleximod/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fleximod/version.py b/src/fleximod/version.py
index d3ec452c3..493f7415d 100644
--- a/src/fleximod/version.py
+++ b/src/fleximod/version.py
@@ -1 +1 @@
-__version__ = "0.2.0"
+__version__ = "0.3.0"

From 2a2ab6503091b3336bf759a32c0874959a275dcd Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 12:58:20 -0700
Subject: [PATCH 075/131] update README

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index cec74c5a6..7c968e70b 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
     Basic Usage:
         git fleximod <command> [options]
     Available Commands:
-        install: Install submodules according to configuration.
+        checkout: Checkout submodules according to git submodule hash configuration.
         status: Display the status of submodules.
         update: Update submodules to the tag indicated in .gitmodules variable fxtag.
         test: Make sure that fxtags and submodule hashes are consistant,
@@ -35,7 +35,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
       - I:T: Internal and required (always checked out).
       - I:F: Internal and optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
-    fcurl
+    fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
 
 ## Sparse Checkouts
 
@@ -55,9 +55,9 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 
 Here are some common usage examples:
 
-Installing submodules, including optional ones:
+Checkout submodules, including optional ones:
 ```bash
-  git fleximod install --optional
+  git fleximod checkout --optional
 ```
 
 Updating a specific submodule to the fxtag indicated in .gitmodules:

From ad0a976b38cd8de97fdce0d04050e412cf2e3905 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 2 Feb 2024 17:15:26 -0700
Subject: [PATCH 076/131] add support for fxhash variable in .gitmodules

---
 src/git-fleximod | 72 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 59 insertions(+), 13 deletions(-)

diff --git a/src/git-fleximod b/src/git-fleximod
index aa587ea3a..5182bb256 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -181,7 +181,9 @@ def commandline_arguments(args=None):
     )
 
 
-def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"):
+def submodule_sparse_checkout(
+    root_dir, name, url, path, sparsefile, tag="master", fxhash=None
+):
     # first create the module directory
     if not os.path.isdir(path):
         os.makedirs(path)
@@ -239,12 +241,19 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     shutil.copy(os.path.join(root_dir, path, sparsefile), gitsparse)
 
     # Finally checkout the repo
-    sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
-    sprepo_git.git_operation("checkout", tag)
-    print(f"Successfully checked out {name}")
+    if fxhash:
+        sprepo_git.git_operation("fetch", "origin", "--tags")
+        sprepo_git.git_operation("checkout", fxhash)
+        print(f"Successfully checked out {name:>20} at {fxhash}")
+    else:
+        sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
+        sprepo_git.git_operation("checkout", tag)
+        print(f"Successfully checked out {name:>20} at {tag}")
 
 
-def single_submodule_checkout(root, name, path, url=None, tag=None, force=False):
+def single_submodule_checkout(
+    root, name, path, url=None, tag=None, force=False, fxhash=None
+):
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
     if os.path.exists(os.path.join(repodir, ".git")):
@@ -264,9 +273,12 @@ def single_submodule_checkout(root, name, path, url=None, tag=None, force=False)
             url = url.replace("git@github.com:", "https://github.com")
             git.git_operation("clone", url, path)
             smgit = GitInterface(repodir, logger)
-            if not tag:
+            if not tag and not fxhash:
                 tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
-            smgit.git_operation("checkout", tag)
+            if fxhash:
+                smgit.git_operation("checkout", fxhash)
+            else:
+                smgit.git_operation("checkout", tag)
             # Now need to move the .git dir to the submodule location
             rootdotgit = os.path.join(root, ".git")
             if os.path.isfile(rootdotgit):
@@ -308,6 +320,9 @@ def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
+        fxhash = gitmodules.get(name, "fxhash")
+        if tag and fxhash:
+            utils.fatal_error(f"{name:>20} cannot have both fxtag and fxhash")
         if not path:
             utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
@@ -329,21 +344,44 @@ def submodules_status(gitmodules, root_dir):
                     f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}"
                 )
                 testfails += 1
+            elif fxhash:
+                n = len(fxhash)
+                smhash = rootgit.git_operation(
+                    "ls-tree", "--object-only", f"--abbrev={n}", "HEAD", path
+                )
+                if smhash == fxhash:
+                    print(f"  {name:>20} not checked out, aligned at hash {fxhash}")
+                else:
+                    print(
+                        f"s {name:>20} not checked out, out of sync at hash {smhash}, expected hash is {fxhash}"
+                    )
+                    testfails += 1
             else:
-                print(f"e {name:>20} has no fxtag defined in .gitmodules")
+                print(f"e {name:>20} has no fxtag nor fxhash defined in .gitmodules")
                 testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                ahash = git.git_operation("status").partition("\n")[0].split()[-1]
                 if tag and atag != tag:
                     print(f"s {name:>20} {atag} is out of sync with .gitmodules {tag}")
                     testfails += 1
                 elif tag:
                     print(f"  {name:>20} at tag {tag}")
+                elif fxhash:
+                    rootgit = GitInterface(root_dir, logger)
+                    n = len(fxhash)
+                    if ahash.startswith(fxhash):
+                        print(f"  {name:>20} at hash {fxhash}")
+                    else:
+                        print(
+                            f"s {name:>20} {ahash} is out of sync with .gitmodules {fxhash}"
+                        )
+                        testfails += 1
                 else:
                     print(
-                        f"e {name:>20} has no tag defined in .gitmodules, module at {atag}"
+                        f"e {name:>20} has no fxtag nor fxhash defined in .gitmodules, module at {atag}"
                     )
                     testfails += 1
 
@@ -365,6 +403,7 @@ def submodules_update(gitmodules, root_dir, force):
         return
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
+        fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info("name={} path={} url={} fxtag={}".format(name, path, url, fxtag))
@@ -394,11 +433,15 @@ def submodules_update(gitmodules, root_dir, force):
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                print(f"fxtag {fxtag} fxhash {fxhash}")
                 if fxtag and fxtag != atag:
                     print(f"{name:>20} updated to {fxtag}")
                     git.git_operation("checkout", fxtag)
-                elif not fxtag:
-                    print(f"No fxtag found for submodule {name:>20}")
+                elif fxhash:
+                    print(f"{name:>20} updated to {fxhash}")
+                    git.git_operation("checkout", fxhash)
+                elif not (fxtag or fxhash):
+                    print(f"No fxtag nor fxhash found for submodule {name:>20}")
                 else:
                     print(f"{name:>20} up to date.")
 
@@ -415,6 +458,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
         fxrequired = gitmodules.get(name, "fxrequired")
         fxsparse = gitmodules.get(name, "fxsparse")
         fxtag = gitmodules.get(name, "fxtag")
+        fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
 
@@ -429,14 +473,16 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
                     root_dir, name, url, path, fxsparse, fxtag
                 )
             )
-            submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag)
+            submodule_sparse_checkout(
+                root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
+            )
         else:
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
 
             single_submodule_checkout(
-                root_dir, name, path, url=url, tag=fxtag, force=force
+                root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash
             )
 
 

From bf56a97251d0f5e5e84910a56e4c8e3036d6b057 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 3 Feb 2024 11:27:03 -0700
Subject: [PATCH 077/131] working on poetry

---
 pyproject.toml                            | 28 ++++++++++++
 setup.cfg                                 |  8 ----
 setup.py                                  | 55 -----------------------
 src/{fleximod => }/__init__.py            |  0
 src/classes/__init__.py                   |  0
 src/{fleximod => classes}/gitinterface.py |  2 +-
 src/{fleximod => classes}/gitmodules.py   |  2 +-
 src/{fleximod => classes}/lstripreader.py |  0
 src/{fleximod => classes}/utils.py        |  0
 src/{fleximod => classes}/version.py      |  0
 src/git-fleximod                          | 11 ++---
 tbump.toml                                | 43 ++++++++++++++++++
 tests/__init__.py                         |  3 ++
 tests/test_import.py                      |  9 ++++
 14 files changed, 91 insertions(+), 70 deletions(-)
 create mode 100644 pyproject.toml
 delete mode 100644 setup.cfg
 delete mode 100644 setup.py
 rename src/{fleximod => }/__init__.py (100%)
 create mode 100644 src/classes/__init__.py
 rename src/{fleximod => classes}/gitinterface.py (98%)
 rename src/{fleximod => classes}/gitmodules.py (98%)
 rename src/{fleximod => classes}/lstripreader.py (100%)
 rename src/{fleximod => classes}/utils.py (100%)
 rename src/{fleximod => classes}/version.py (100%)
 create mode 100644 tbump.toml
 create mode 100644 tests/__init__.py
 create mode 100644 tests/test_import.py

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..28e0e35a0
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "git-fleximod"
+version = "0.3.0"
+description = "Extended support for git-submodule and git-sparse-checkout"
+authors = ["Jim Edwards <jedwards@ucar.edu>"]
+maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
+license = "MIT"
+readme = "README.md"
+homepage = "https://github.com/jedwards4b/git-fleximod"
+keywords = ["git", "submodule", "sparse-checkout"]
+packages = [
+  { include = "src/git-fleximod" },
+  { include = "src/classes/*.py" },
+  { include = "License" },
+  { include = "tests" },
+
+]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+GitPython = "^3.1.0"
+
+[tools.poetry.urls]
+"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 66ff24ada..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,8 +0,0 @@
-[metadata]
-license_file = License
-[build_manpages]
-manpages =
-    # solution 1: -- use parser function
-    man/git-fleximod.1:function=get_parser:pyfile=src/git-fleximod
-    # solution 2 -- make parser global
-    #man/git-fleximod.2:pyfile=src/git-fleximod:object=parser
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 048db7595..000000000
--- a/setup.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import sys
-import os
-
-sys.path.insert(0, os.path.join(os.getenv("CONDA_PREFIX"), "lib", "python3.12"))
-sys.path.insert(
-    0, os.path.join(os.getenv("CONDA_PREFIX"), "lib", "python3.12", "site-packages")
-)
-
-import setuptools
-from setuptools import find_packages, convert_path
-from build_manpages import build_manpages, get_build_py_cmd, get_install_cmd
-
-main_ns = {}
-ver_path = convert_path(
-    os.path.join(os.path.dirname(__file__), "src", "fleximod", "version.py")
-)
-print(f"ver_path is {ver_path}")
-with open(ver_path) as ver_file:
-    exec(ver_file.read(), main_ns)
-
-if __name__ == "__main__":
-    with open("README.md", "r") as fh:
-        long_description = fh.read()
-
-    setuptools.setup(
-        name="git-fleximod",  # package name
-        scripts=["src/git-fleximod"],  # This is the name of the package
-        version=main_ns["__version__"],  # The initial release version
-        author="Jim Edwards",  # Full name of the author
-        maintainer="jedwards4b",
-        license="MIT License",
-        description="Extended support for git-submodule and git-sparse-checkout",
-        long_description=long_description,  # Long description read from the the readme file
-        long_description_content_type="text/markdown",
-        packages=find_packages(
-            where="src"
-        ),  # List of all python modules to be installed
-        package_dir={"": "src", "git-fleximod": "src", "fleximod": "src/fleximod"},
-        package_data={"": ["src/fleximod/version.txt", "man/git-fleximod.1"]},
-        classifiers=[
-            "Programming Language :: Python :: 3",
-            "License :: OSI Approved :: MIT License",
-            "Operating System :: OS Independent",
-        ],  # Information to filter the project on PyPi website
-        python_requires=">=3.6",  # Minimum version requirement of the package
-        #    py_modules=['git-fleximod'],             # Name of the python package
-        install_requires=["GitPython"],  # Install other dependencies if any
-        cmdclass={
-            "build_manpages": build_manpages,
-            # Re-define build_py and install commands so the manual pages
-            # are automatically re-generated and installed
-            "build_py": get_build_py_cmd(),
-            "install": get_install_cmd(),
-        },
-    )
diff --git a/src/fleximod/__init__.py b/src/__init__.py
similarity index 100%
rename from src/fleximod/__init__.py
rename to src/__init__.py
diff --git a/src/classes/__init__.py b/src/classes/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/fleximod/gitinterface.py b/src/classes/gitinterface.py
similarity index 98%
rename from src/fleximod/gitinterface.py
rename to src/classes/gitinterface.py
index 96bb9ac52..35c593a09 100644
--- a/src/fleximod/gitinterface.py
+++ b/src/classes/gitinterface.py
@@ -1,5 +1,5 @@
 import os
-from fleximod import utils
+from classes import utils
 
 
 class GitInterface:
diff --git a/src/fleximod/gitmodules.py b/src/classes/gitmodules.py
similarity index 98%
rename from src/fleximod/gitmodules.py
rename to src/classes/gitmodules.py
index 11a245724..dd2e1fc94 100644
--- a/src/fleximod/gitmodules.py
+++ b/src/classes/gitmodules.py
@@ -1,7 +1,7 @@
 import os
 import shutil
 from configparser import ConfigParser
-from fleximod.lstripreader import LstripReader
+from classes.lstripreader import LstripReader
 
 
 class GitModules(ConfigParser):
diff --git a/src/fleximod/lstripreader.py b/src/classes/lstripreader.py
similarity index 100%
rename from src/fleximod/lstripreader.py
rename to src/classes/lstripreader.py
diff --git a/src/fleximod/utils.py b/src/classes/utils.py
similarity index 100%
rename from src/fleximod/utils.py
rename to src/classes/utils.py
diff --git a/src/fleximod/version.py b/src/classes/version.py
similarity index 100%
rename from src/fleximod/version.py
rename to src/classes/version.py
diff --git a/src/git-fleximod b/src/git-fleximod
index aa587ea3a..1c06d0d3d 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -6,10 +6,10 @@ import logging
 import argparse
 import textwrap
 from pathlib import Path
-from fleximod import utils
-from fleximod.gitinterface import GitInterface
-from fleximod.gitmodules import GitModules
-from fleximod.version import __version__
+from classes import utils
+from classes.gitinterface import GitInterface
+from classes.gitmodules import GitModules
+from classes.version import __version__
 from configparser import NoOptionError
 
 # logger variable is global
@@ -43,7 +43,7 @@ def get_parser():
         "action",
         choices=choices,
         default="checkout",
-        help=f"Subcommand of fleximod, choices are {choices[:-1]}",
+        help=f"Subcommand of git-fleximod, choices are {choices[:-1]}",
     )
 
     parser.add_argument(
@@ -394,6 +394,7 @@ def submodules_update(gitmodules, root_dir, force):
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
+
                 if fxtag and fxtag != atag:
                     print(f"{name:>20} updated to {fxtag}")
                     git.git_operation("checkout", fxtag)
diff --git a/tbump.toml b/tbump.toml
new file mode 100644
index 000000000..0e0a63303
--- /dev/null
+++ b/tbump.toml
@@ -0,0 +1,43 @@
+# Uncomment this if your project is hosted on GitHub:
+github_url = "https://github.com/jedwards4b/git-fleximod/"
+
+[version]
+current = "0.3.0"
+
+# Example of a semver regexp.
+# Make sure this matches current_version before
+# using tbump
+regex = '''
+  (?P<major>\d+)
+  \.
+  (?P<minor>\d+)
+  \.
+  (?P<patch>\d+)
+  '''
+
+[git]
+message_template = "Bump to {new_version}"
+tag_template = "v{new_version}"
+
+# For each file to patch, add a [[file]] config
+# section containing the path of the file, relative to the
+# tbump.toml location.
+[[file]]
+src = "src/classes/version.py"
+
+[[file]]
+src = "pyproject.toml"
+
+# You can specify a list of commands to
+# run after the files have been patched
+# and before the git commit is made
+
+#  [[before_commit]]
+#  name = "check changelog"
+#  cmd = "grep -q {new_version} Changelog.rst"
+
+# Or run some commands after the git tag and the branch
+# have been pushed:
+#  [[after_push]]
+#  name = "publish"
+#  cmd = "./publish.sh"
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 000000000..4d4c66c78
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1,3 @@
+import sys, os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src"))
diff --git a/tests/test_import.py b/tests/test_import.py
new file mode 100644
index 000000000..becf75144
--- /dev/null
+++ b/tests/test_import.py
@@ -0,0 +1,9 @@
+# pylint: disable=unused-import
+from classes import utils
+from classes.gitinterface import GitInterface
+from classes.gitmodules import GitModules
+from classes.version import __version__
+
+
+def test_import():
+    print("here")

From 4f3bb556e165f9f1087de841c9b408cedb62f4a3 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 3 Feb 2024 13:59:24 -0700
Subject: [PATCH 078/131] more poetry and sphinx

---
 doc/Makefile           |  20 +++++++
 doc/conf.py            |  26 +++++++++
 doc/index.rst          |  24 ++++++++
 doc/make.bat           |  35 ++++++++++++
 pyproject.toml         |   5 +-
 src/classes/cli.py     | 120 ++++++++++++++++++++++++++++++++++++++++
 src/classes/version.py |   1 -
 src/git-fleximod       | 122 +----------------------------------------
 tbump.toml             |   2 +-
 9 files changed, 232 insertions(+), 123 deletions(-)
 create mode 100644 doc/Makefile
 create mode 100644 doc/conf.py
 create mode 100644 doc/index.rst
 create mode 100644 doc/make.bat
 create mode 100644 src/classes/cli.py
 delete mode 100644 src/classes/version.py

diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 000000000..d4bb2cbb9
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 000000000..423099eec
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,26 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "git-fleximod"
+author = "Jim Edwards <jedwards@ucar.edu>"
+release = "0.4.0"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["sphinx_argparse_cli"]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "alabaster"
+html_static_path = ["_static"]
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 000000000..fa534d0d1
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,24 @@
+.. git-fleximod documentation master file, created by
+   sphinx-quickstart on Sat Feb  3 12:02:22 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to git-fleximod's documentation!
+========================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+.. module:: sphinxcontrib.autoprogram
+.. sphinx_argparse_cli::
+   :module: classes.cli
+   :func: get_parser
+   :prog: git-fleximod
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/doc/make.bat b/doc/make.bat
new file mode 100644
index 000000000..32bb24529
--- /dev/null
+++ b/doc/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/pyproject.toml b/pyproject.toml
index 28e0e35a0..d49cbae72 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,7 +13,10 @@ packages = [
   { include = "src/classes/*.py" },
   { include = "License" },
   { include = "tests" },
-
+  { include = "doc/*.rst" },
+  { include = "doc/Makefile" },
+  { include = "doc/make.bat" },
+  { include = "doc/conf.py" },
 ]
 
 [tool.poetry.dependencies]
diff --git a/src/classes/cli.py b/src/classes/cli.py
new file mode 100644
index 000000000..437d9beec
--- /dev/null
+++ b/src/classes/cli.py
@@ -0,0 +1,120 @@
+from pathlib import Path
+import argparse
+
+__version__ = "0.3.0"
+
+
+def find_root_dir(filename=".git"):
+    d = Path.cwd()
+    root = Path(d.root)
+    while d != root:
+        attempt = d / filename
+        if attempt.is_dir():
+            return attempt
+        d = d.parent
+    return None
+
+
+def get_parser():
+    description = """
+    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
+    """
+    parser = argparse.ArgumentParser(
+        description=description, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    #
+    # user options
+    #
+    choices = ["update", "checkout", "status", "test"]
+    parser.add_argument(
+        "action",
+        choices=choices,
+        default="checkout",
+        help=f"Subcommand of git-fleximod, choices are {choices[:-1]}",
+    )
+
+    parser.add_argument(
+        "components",
+        nargs="*",
+        help="Specific component(s) to checkout. By default, "
+        "all required submodules are checked out.",
+    )
+
+    parser.add_argument(
+        "-C",
+        "--path",
+        default=find_root_dir(),
+        help="Toplevel repository directory.  Defaults to top git directory relative to current.",
+    )
+
+    parser.add_argument(
+        "-g",
+        "--gitmodules",
+        nargs="?",
+        default=".gitmodules",
+        help="The submodule description filename. " "Default: %(default)s.",
+    )
+
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        nargs="*",
+        help="Component(s) listed in the gitmodules file which should be ignored.",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="Override cautions and update or checkout over locally modified repository.",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--optional",
+        action="store_true",
+        default=False,
+        help="By default only the required submodules "
+        "are checked out. This flag will also checkout the "
+        "optional submodules relative to the toplevel directory.",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Output additional information to "
+        "the screen and log file. This flag can be "
+        "used up to two times, increasing the "
+        "verbosity level each time.",
+    )
+
+    parser.add_argument(
+        "-V",
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="Print version and exit.",
+    )
+
+    #
+    # developer options
+    #
+    parser.add_argument(
+        "--backtrace",
+        action="store_true",
+        help="DEVELOPER: show exception backtraces as extra " "debugging output",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        default=False,
+        help="DEVELOPER: output additional debugging "
+        "information to the screen and log file.",
+    )
+
+    return parser
diff --git a/src/classes/version.py b/src/classes/version.py
deleted file mode 100644
index 493f7415d..000000000
--- a/src/classes/version.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.3.0"
diff --git a/src/git-fleximod b/src/git-fleximod
index 1c06d0d3d..385515819 100755
--- a/src/git-fleximod
+++ b/src/git-fleximod
@@ -3,137 +3,19 @@ import sys
 import os
 import shutil
 import logging
-import argparse
 import textwrap
-from pathlib import Path
 from classes import utils
+from classes import cli
 from classes.gitinterface import GitInterface
 from classes.gitmodules import GitModules
-from classes.version import __version__
 from configparser import NoOptionError
 
 # logger variable is global
 logger = None
 
 
-def find_root_dir(filename=".git"):
-    d = Path.cwd()
-    root = Path(d.root)
-    while d != root:
-        attempt = d / filename
-        if attempt.is_dir():
-            return attempt
-        d = d.parent
-    return None
-
-
-def get_parser():
-    description = """
-    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
-    """
-    parser = argparse.ArgumentParser(
-        description=description, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-
-    #
-    # user options
-    #
-    choices = ["update", "checkout", "status", "test"]
-    parser.add_argument(
-        "action",
-        choices=choices,
-        default="checkout",
-        help=f"Subcommand of git-fleximod, choices are {choices[:-1]}",
-    )
-
-    parser.add_argument(
-        "components",
-        nargs="*",
-        help="Specific component(s) to checkout. By default, "
-        "all required submodules are checked out.",
-    )
-
-    parser.add_argument(
-        "-C",
-        "--path",
-        default=find_root_dir(),
-        help="Toplevel repository directory.  Defaults to top git directory relative to current.",
-    )
-
-    parser.add_argument(
-        "-g",
-        "--gitmodules",
-        nargs="?",
-        default=".gitmodules",
-        help="The submodule description filename. " "Default: %(default)s.",
-    )
-
-    parser.add_argument(
-        "-x",
-        "--exclude",
-        nargs="*",
-        help="Component(s) listed in the gitmodules file which should be ignored.",
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        default=False,
-        help="Override cautions and update or checkout over locally modified repository.",
-    )
-
-    parser.add_argument(
-        "-o",
-        "--optional",
-        action="store_true",
-        default=False,
-        help="By default only the required submodules "
-        "are checked out. This flag will also checkout the "
-        "optional submodules relative to the toplevel directory.",
-    )
-
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        action="count",
-        default=0,
-        help="Output additional information to "
-        "the screen and log file. This flag can be "
-        "used up to two times, increasing the "
-        "verbosity level each time.",
-    )
-
-    parser.add_argument(
-        "-V",
-        "--version",
-        action="version",
-        version=f"%(prog)s {__version__}",
-        help="Print version and exit.",
-    )
-
-    #
-    # developer options
-    #
-    parser.add_argument(
-        "--backtrace",
-        action="store_true",
-        help="DEVELOPER: show exception backtraces as extra " "debugging output",
-    )
-
-    parser.add_argument(
-        "-d",
-        "--debug",
-        action="store_true",
-        default=False,
-        help="DEVELOPER: output additional debugging "
-        "information to the screen and log file.",
-    )
-
-    return parser
-
-
 def commandline_arguments(args=None):
-    parser = get_parser()
+    parser = cli.get_parser()
 
     if args:
         options = parser.parse_args(args)
diff --git a/tbump.toml b/tbump.toml
index 0e0a63303..35521d04d 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -23,7 +23,7 @@ tag_template = "v{new_version}"
 # section containing the path of the file, relative to the
 # tbump.toml location.
 [[file]]
-src = "src/classes/version.py"
+src = "src/classes/cli.py"
 
 [[file]]
 src = "pyproject.toml"

From 461ee4be93bc63eb9c735e03a7ce12c04ff3fe84 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 3 Feb 2024 14:02:07 -0700
Subject: [PATCH 079/131] Bump to 0.5.0

---
 pyproject.toml     | 2 +-
 src/classes/cli.py | 2 +-
 tbump.toml         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index d49cbae72..8b4246e25 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.3.0"
+version = "0.5.0"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/src/classes/cli.py b/src/classes/cli.py
index 437d9beec..65f70cdd3 100644
--- a/src/classes/cli.py
+++ b/src/classes/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.3.0"
+__version__ = "0.5.0"
 
 
 def find_root_dir(filename=".git"):
diff --git a/tbump.toml b/tbump.toml
index 35521d04d..18e21a69f 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.3.0"
+current = "0.5.0"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From f0ab8150ba149c42d4690aa8964754bd58225a11 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 4 Feb 2024 09:15:55 -0700
Subject: [PATCH 080/131] reformed to work with poetry

---
 {src => git_fleximod}/__init__.py                |  0
 {src/classes => git_fleximod}/cli.py             |  1 -
 src/git-fleximod => git_fleximod/git-fleximod.py | 12 ++++++------
 {src/classes => git_fleximod}/gitinterface.py    |  3 +--
 {src/classes => git_fleximod}/gitmodules.py      |  2 +-
 {src/classes => git_fleximod}/lstripreader.py    |  0
 {src/classes => git_fleximod}/utils.py           |  0
 pyproject.toml                                   | 16 ++++++----------
 src/classes/__init__.py                          |  0
 9 files changed, 14 insertions(+), 20 deletions(-)
 rename {src => git_fleximod}/__init__.py (100%)
 rename {src/classes => git_fleximod}/cli.py (99%)
 rename src/git-fleximod => git_fleximod/git-fleximod.py (98%)
 rename {src/classes => git_fleximod}/gitinterface.py (98%)
 rename {src/classes => git_fleximod}/gitmodules.py (98%)
 rename {src/classes => git_fleximod}/lstripreader.py (100%)
 rename {src/classes => git_fleximod}/utils.py (100%)
 delete mode 100644 src/classes/__init__.py

diff --git a/src/__init__.py b/git_fleximod/__init__.py
similarity index 100%
rename from src/__init__.py
rename to git_fleximod/__init__.py
diff --git a/src/classes/cli.py b/git_fleximod/cli.py
similarity index 99%
rename from src/classes/cli.py
rename to git_fleximod/cli.py
index 65f70cdd3..822b31438 100644
--- a/src/classes/cli.py
+++ b/git_fleximod/cli.py
@@ -3,7 +3,6 @@
 
 __version__ = "0.5.0"
 
-
 def find_root_dir(filename=".git"):
     d = Path.cwd()
     root = Path(d.root)
diff --git a/src/git-fleximod b/git_fleximod/git-fleximod.py
similarity index 98%
rename from src/git-fleximod
rename to git_fleximod/git-fleximod.py
index 385515819..9e4969d6c 100755
--- a/src/git-fleximod
+++ b/git_fleximod/git-fleximod.py
@@ -4,10 +4,10 @@
 import shutil
 import logging
 import textwrap
-from classes import utils
-from classes import cli
-from classes.gitinterface import GitInterface
-from classes.gitmodules import GitModules
+from git_fleximod import utils
+from git_fleximod import cli
+from git_fleximod.gitinterface import GitInterface
+from git_fleximod.gitmodules import GitModules
 from configparser import NoOptionError
 
 # logger variable is global
@@ -343,7 +343,7 @@ def submodules_test(gitmodules, root_dir):
     return testfails + localmods
 
 
-def _main_func():
+def main():
     (
         root_dir,
         file_name,
@@ -393,4 +393,4 @@ def _main_func():
 
 
 if __name__ == "__main__":
-    sys.exit(_main_func())
+    sys.exit(main())
diff --git a/src/classes/gitinterface.py b/git_fleximod/gitinterface.py
similarity index 98%
rename from src/classes/gitinterface.py
rename to git_fleximod/gitinterface.py
index 35c593a09..eff155d54 100644
--- a/src/classes/gitinterface.py
+++ b/git_fleximod/gitinterface.py
@@ -1,6 +1,5 @@
 import os
-from classes import utils
-
+from . import utils
 
 class GitInterface:
     def __init__(self, repo_path, logger):
diff --git a/src/classes/gitmodules.py b/git_fleximod/gitmodules.py
similarity index 98%
rename from src/classes/gitmodules.py
rename to git_fleximod/gitmodules.py
index dd2e1fc94..ae0ebe121 100644
--- a/src/classes/gitmodules.py
+++ b/git_fleximod/gitmodules.py
@@ -1,7 +1,7 @@
 import os
 import shutil
 from configparser import ConfigParser
-from classes.lstripreader import LstripReader
+from .lstripreader import LstripReader
 
 
 class GitModules(ConfigParser):
diff --git a/src/classes/lstripreader.py b/git_fleximod/lstripreader.py
similarity index 100%
rename from src/classes/lstripreader.py
rename to git_fleximod/lstripreader.py
diff --git a/src/classes/utils.py b/git_fleximod/utils.py
similarity index 100%
rename from src/classes/utils.py
rename to git_fleximod/utils.py
diff --git a/pyproject.toml b/pyproject.toml
index 8b4246e25..666178935 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,16 +8,12 @@ license = "MIT"
 readme = "README.md"
 homepage = "https://github.com/jedwards4b/git-fleximod"
 keywords = ["git", "submodule", "sparse-checkout"]
-packages = [
-  { include = "src/git-fleximod" },
-  { include = "src/classes/*.py" },
-  { include = "License" },
-  { include = "tests" },
-  { include = "doc/*.rst" },
-  { include = "doc/Makefile" },
-  { include = "doc/make.bat" },
-  { include = "doc/conf.py" },
-]
+
+[[tool.poetry.packages]]
+include = "git_fleximod"
+
+[tool.poetry.scripts]
+git-fleximod = "git_fleximod.git-fleximod:main"
 
 [tool.poetry.dependencies]
 python = "^3.8"
diff --git a/src/classes/__init__.py b/src/classes/__init__.py
deleted file mode 100644
index e69de29bb..000000000

From 7b2a86dcfdcc039034c2ad2e2b69103ac5bf319f Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 4 Feb 2024 09:22:09 -0700
Subject: [PATCH 081/131] add a simple import test

---
 tests/test_import.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/test_import.py b/tests/test_import.py
index becf75144..d5ca878de 100644
--- a/tests/test_import.py
+++ b/tests/test_import.py
@@ -1,9 +1,8 @@
 # pylint: disable=unused-import
-from classes import utils
-from classes.gitinterface import GitInterface
-from classes.gitmodules import GitModules
-from classes.version import __version__
-
+from git_fleximod import cli
+from git_fleximod import utils
+from git_fleximod.gitinterface import GitInterface
+from git_fleximod.gitmodules import GitModules
 
 def test_import():
     print("here")

From e7fdd39fd8df3b829196f276e21c7945fd71401e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 4 Feb 2024 10:30:30 -0700
Subject: [PATCH 082/131] now includes man pages

---
 doc/index.rst  |  2 +-
 pyproject.toml | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/doc/index.rst b/doc/index.rst
index fa534d0d1..0f9c1a7f7 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -11,7 +11,7 @@ Welcome to git-fleximod's documentation!
    :caption: Contents:
 .. module:: sphinxcontrib.autoprogram
 .. sphinx_argparse_cli::
-   :module: classes.cli
+   :module: git_fleximod.cli
    :func: get_parser
    :prog: git-fleximod
 
diff --git a/pyproject.toml b/pyproject.toml
index 666178935..17843e0b8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -8,18 +8,23 @@ license = "MIT"
 readme = "README.md"
 homepage = "https://github.com/jedwards4b/git-fleximod"
 keywords = ["git", "submodule", "sparse-checkout"]
-
-[[tool.poetry.packages]]
-include = "git_fleximod"
+packages = [
+{ include = "git_fleximod"},
+{ include = "doc"},
+]
 
 [tool.poetry.scripts]
 git-fleximod = "git_fleximod.git-fleximod:main"
+fsspec = "fsspec.fuse:main"
 
 [tool.poetry.dependencies]
 python = "^3.8"
 GitPython = "^3.1.0"
+sphinx = "^5.0.0"
+fsspec = "^2023.12.2"
+wheel = "^0.42.0"
 
-[tools.poetry.urls]
+[tool.poetry.urls]
 "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"
 
 [build-system]

From 26739ada8bf21eda4aefabdf8219ade9e09e473e Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 5 Feb 2024 08:09:44 -0700
Subject: [PATCH 083/131] update tbump path

---
 tbump.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tbump.toml b/tbump.toml
index 18e21a69f..c3720e25d 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -23,7 +23,7 @@ tag_template = "v{new_version}"
 # section containing the path of the file, relative to the
 # tbump.toml location.
 [[file]]
-src = "src/classes/cli.py"
+src = "git_fleximod/cli.py"
 
 [[file]]
 src = "pyproject.toml"

From 25bcc7ea7ac8cf9e02c5dfbcd81f257c6f50b21a Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Mon, 5 Feb 2024 08:09:53 -0700
Subject: [PATCH 084/131] Bump to 0.6.0

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 822b31438..e1c6d2d5c 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index 17843e0b8..eabb7657e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.5.0"
+version = "0.6.0"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index c3720e25d..b41ec17e2 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.5.0"
+current = "0.6.0"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From ed01cb7a0f07e1c79136b50f6448463a45538acb Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Mon, 5 Feb 2024 09:06:54 -0700
Subject: [PATCH 085/131] fix issue with hyphen

---
 .../{git-fleximod.py => git_fleximod.py}      |   0
 poetry.lock                                   | 682 ++++++++++++++++++
 pyproject.toml                                |   1 +
 3 files changed, 683 insertions(+)
 rename git_fleximod/{git-fleximod.py => git_fleximod.py} (100%)
 create mode 100644 poetry.lock

diff --git a/git_fleximod/git-fleximod.py b/git_fleximod/git_fleximod.py
similarity index 100%
rename from git_fleximod/git-fleximod.py
rename to git_fleximod/git_fleximod.py
diff --git a/poetry.lock b/poetry.lock
new file mode 100644
index 000000000..6c31fac48
--- /dev/null
+++ b/poetry.lock
@@ -0,0 +1,682 @@
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+
+[[package]]
+name = "alabaster"
+version = "0.7.13"
+description = "A configurable sidebar-enabled Sphinx theme"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"},
+    {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"},
+]
+
+[[package]]
+name = "babel"
+version = "2.14.0"
+description = "Internationalization utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"},
+    {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"},
+]
+
+[package.dependencies]
+pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""}
+
+[package.extras]
+dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"]
+
+[[package]]
+name = "certifi"
+version = "2024.2.2"
+description = "Python package for providing Mozilla's CA Bundle."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
+    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.3.2"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
+    {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "docutils"
+version = "0.19"
+description = "Docutils -- Python Documentation Utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"},
+    {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"},
+]
+
+[[package]]
+name = "exceptiongroup"
+version = "1.2.0"
+description = "Backport of PEP 654 (exception groups)"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"},
+    {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"},
+]
+
+[package.extras]
+test = ["pytest (>=6)"]
+
+[[package]]
+name = "fsspec"
+version = "2023.12.2"
+description = "File-system specification"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"},
+    {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"},
+]
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+devel = ["pytest", "pytest-cov"]
+dropbox = ["dropbox", "dropboxdrivefs", "requests"]
+full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
+[[package]]
+name = "gitdb"
+version = "4.0.11"
+description = "Git Object Database"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"},
+    {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"},
+]
+
+[package.dependencies]
+smmap = ">=3.0.1,<6"
+
+[[package]]
+name = "gitpython"
+version = "3.1.41"
+description = "GitPython is a Python library used to interact with Git repositories"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"},
+    {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"},
+]
+
+[package.dependencies]
+gitdb = ">=4.0.1,<5"
+
+[package.extras]
+test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"]
+
+[[package]]
+name = "idna"
+version = "3.6"
+description = "Internationalized Domain Names in Applications (IDNA)"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"},
+    {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"},
+]
+
+[[package]]
+name = "imagesize"
+version = "1.4.1"
+description = "Getting image size from png/jpeg/jpeg2000/gif file"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"},
+    {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"},
+]
+
+[[package]]
+name = "importlib-metadata"
+version = "7.0.1"
+description = "Read metadata from Python packages"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"},
+    {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"},
+]
+
+[package.dependencies]
+zipp = ">=0.5"
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"]
+perf = ["ipython"]
+testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"]
+
+[[package]]
+name = "iniconfig"
+version = "2.0.0"
+description = "brain-dead simple config-ini parsing"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
+
+[[package]]
+name = "jinja2"
+version = "3.1.3"
+description = "A very fast and expressive template engine."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"},
+    {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"},
+]
+
+[package.dependencies]
+MarkupSafe = ">=2.0"
+
+[package.extras]
+i18n = ["Babel (>=2.7)"]
+
+[[package]]
+name = "markupsafe"
+version = "2.1.5"
+description = "Safely add untrusted strings to HTML/XML markup."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"},
+    {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"},
+    {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"},
+    {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"},
+    {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"},
+    {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"},
+    {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"},
+    {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"},
+]
+
+[[package]]
+name = "packaging"
+version = "23.2"
+description = "Core utilities for Python packages"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},
+    {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},
+]
+
+[[package]]
+name = "pluggy"
+version = "1.4.0"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"},
+    {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+testing = ["pytest", "pytest-benchmark"]
+
+[[package]]
+name = "pygments"
+version = "2.17.2"
+description = "Pygments is a syntax highlighting package written in Python."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"},
+    {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"},
+]
+
+[package.extras]
+plugins = ["importlib-metadata"]
+windows-terminal = ["colorama (>=0.4.6)"]
+
+[[package]]
+name = "pytest"
+version = "8.0.0"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"},
+    {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
+iniconfig = "*"
+packaging = "*"
+pluggy = ">=1.3.0,<2.0"
+tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
+
+[package.extras]
+testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+
+[[package]]
+name = "pytz"
+version = "2024.1"
+description = "World timezone definitions, modern and historical"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"},
+    {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"},
+]
+
+[[package]]
+name = "requests"
+version = "2.31.0"
+description = "Python HTTP for Humans."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
+    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
+]
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = ">=2,<4"
+idna = ">=2.5,<4"
+urllib3 = ">=1.21.1,<3"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
+
+[[package]]
+name = "smmap"
+version = "5.0.1"
+description = "A pure Python implementation of a sliding window memory map manager"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"},
+    {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"},
+]
+
+[[package]]
+name = "snowballstemmer"
+version = "2.2.0"
+description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms."
+optional = false
+python-versions = "*"
+files = [
+    {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"},
+    {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"},
+]
+
+[[package]]
+name = "sphinx"
+version = "5.3.0"
+description = "Python documentation generator"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"},
+    {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"},
+]
+
+[package.dependencies]
+alabaster = ">=0.7,<0.8"
+babel = ">=2.9"
+colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
+docutils = ">=0.14,<0.20"
+imagesize = ">=1.3"
+importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""}
+Jinja2 = ">=3.0"
+packaging = ">=21.0"
+Pygments = ">=2.12"
+requests = ">=2.5.0"
+snowballstemmer = ">=2.0"
+sphinxcontrib-applehelp = "*"
+sphinxcontrib-devhelp = "*"
+sphinxcontrib-htmlhelp = ">=2.0.0"
+sphinxcontrib-jsmath = "*"
+sphinxcontrib-qthelp = "*"
+sphinxcontrib-serializinghtml = ">=1.1.5"
+
+[package.extras]
+docs = ["sphinxcontrib-websupport"]
+lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"]
+test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"]
+
+[[package]]
+name = "sphinxcontrib-applehelp"
+version = "1.0.4"
+description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"},
+    {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"},
+]
+
+[package.extras]
+lint = ["docutils-stubs", "flake8", "mypy"]
+test = ["pytest"]
+
+[[package]]
+name = "sphinxcontrib-devhelp"
+version = "1.0.2"
+description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document."
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"},
+    {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"},
+]
+
+[package.extras]
+lint = ["docutils-stubs", "flake8", "mypy"]
+test = ["pytest"]
+
+[[package]]
+name = "sphinxcontrib-htmlhelp"
+version = "2.0.1"
+description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"},
+    {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"},
+]
+
+[package.extras]
+lint = ["docutils-stubs", "flake8", "mypy"]
+test = ["html5lib", "pytest"]
+
+[[package]]
+name = "sphinxcontrib-jsmath"
+version = "1.0.1"
+description = "A sphinx extension which renders display math in HTML via JavaScript"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"},
+    {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"},
+]
+
+[package.extras]
+test = ["flake8", "mypy", "pytest"]
+
+[[package]]
+name = "sphinxcontrib-qthelp"
+version = "1.0.3"
+description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document."
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"},
+    {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"},
+]
+
+[package.extras]
+lint = ["docutils-stubs", "flake8", "mypy"]
+test = ["pytest"]
+
+[[package]]
+name = "sphinxcontrib-serializinghtml"
+version = "1.1.5"
+description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)."
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"},
+    {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"},
+]
+
+[package.extras]
+lint = ["docutils-stubs", "flake8", "mypy"]
+test = ["pytest"]
+
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
+
+[[package]]
+name = "urllib3"
+version = "2.2.0"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"},
+    {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"},
+]
+
+[package.extras]
+brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
+h2 = ["h2 (>=4,<5)"]
+socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
+zstd = ["zstandard (>=0.18.0)"]
+
+[[package]]
+name = "wheel"
+version = "0.42.0"
+description = "A built-package format for Python"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"},
+    {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"},
+]
+
+[package.extras]
+test = ["pytest (>=6.0.0)", "setuptools (>=65)"]
+
+[[package]]
+name = "zipp"
+version = "3.17.0"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"},
+    {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"},
+]
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"]
+testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.8"
+content-hash = "f20e29bada880bbb0aad20d2a7ac282d09c3e89d4aac06072cb82a9157f4023a"
diff --git a/pyproject.toml b/pyproject.toml
index eabb7657e..c6ccff262 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,6 +23,7 @@ GitPython = "^3.1.0"
 sphinx = "^5.0.0"
 fsspec = "^2023.12.2"
 wheel = "^0.42.0"
+pytest = "^8.0.0"
 
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"

From 69a5dc5ae30514b2cefe8ee827b6d2a5e4322db9 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Mon, 5 Feb 2024 09:07:10 -0700
Subject: [PATCH 086/131] Bump to 0.6.1

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index e1c6d2d5c..39e83b97c 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.6.0"
+__version__ = "0.6.1"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index c6ccff262..225f7b39b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.6.0"
+version = "0.6.1"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index b41ec17e2..03b63f154 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.6.0"
+current = "0.6.1"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From f456ca059ccd6bfacd15dce2b302eafb6c09ce2c Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 6 Feb 2024 11:45:42 -0700
Subject: [PATCH 087/131] fix issue with sparse checkout dir path

---
 git_fleximod/git_fleximod.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 9e4969d6c..59656eeb0 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -65,8 +65,8 @@ def commandline_arguments(args=None):
 
 def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"):
     # first create the module directory
-    if not os.path.isdir(path):
-        os.makedirs(path)
+    if not os.path.isdir(os.path.join(root_dir,path)):
+        os.makedirs(os.path.join(root_dir,path))
     # Check first if the module is already defined
     # and the sparse-checkout file exists
     git = GitInterface(root_dir, logger)

From ac3e6ddc7f1fddc3066d81a61aa7a042683315cb Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 6 Feb 2024 13:12:28 -0700
Subject: [PATCH 088/131] Bump to 0.6.2

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 39e83b97c..719b15620 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.6.1"
+__version__ = "0.6.2"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index 225f7b39b..f5828a58e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.6.1"
+version = "0.6.2"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index 03b63f154..583cef45a 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.6.1"
+current = "0.6.2"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From 96ac371975428ddebf7744bf9304f7496c66a959 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 9 Feb 2024 14:41:10 -0700
Subject: [PATCH 089/131] add tests

---
 git_fleximod/gitinterface.py               | 11 +++++--
 poetry.lock                                | 13 +++++++-
 pyproject.toml                             |  3 +-
 tests/{test_import.py => test_a_import.py} |  0
 tests/test_checkout.py                     | 32 ++++++++++++++++++
 tests/test_sparse_checkout.py              | 38 ++++++++++++++++++++++
 6 files changed, 93 insertions(+), 4 deletions(-)
 rename tests/{test_import.py => test_a_import.py} (100%)
 create mode 100644 tests/test_checkout.py
 create mode 100644 tests/test_sparse_checkout.py

diff --git a/git_fleximod/gitinterface.py b/git_fleximod/gitinterface.py
index eff155d54..4d4c4f4b9 100644
--- a/git_fleximod/gitinterface.py
+++ b/git_fleximod/gitinterface.py
@@ -1,4 +1,5 @@
 import os
+import sys
 from . import utils
 
 class GitInterface:
@@ -26,7 +27,10 @@ def __init__(self, repo_path, logger):
     def _git_command(self, operation, *args):
         self.logger.info(operation)
         if self._use_module and operation != "submodule":
-            return getattr(self.repo.git, operation)(*args)
+            try:
+                return getattr(self.repo.git, operation)(*args)
+            except Exception as e:
+                sys.exit(e)
         else:
             return ["git", "-C", self.repo_path, operation] + list(args)
 
@@ -42,7 +46,10 @@ def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *args)
         self.logger.info(command)
         if isinstance(command, list):
-            return utils.execute_subprocess(command, output_to_caller=True)
+            try:
+                return utils.execute_subprocess(command, output_to_caller=True)
+            except Exception as e:
+                sys.exit(e)
         else:
             return command
 
diff --git a/poetry.lock b/poetry.lock
index 6c31fac48..b59ed3942 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -404,6 +404,17 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
+[[package]]
+name = "pyfakefs"
+version = "5.3.5"
+description = "pyfakefs implements a fake file system that mocks the Python file system modules."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pyfakefs-5.3.5-py3-none-any.whl", hash = "sha256:751015c1de94e1390128c82b48cdedc3f088bbdbe4bc713c79d02a27f0f61e69"},
+    {file = "pyfakefs-5.3.5.tar.gz", hash = "sha256:7cdc500b35a214cb7a614e1940543acc6650e69a94ac76e30f33c9373bd9cf90"},
+]
+
 [[package]]
 name = "pygments"
 version = "2.17.2"
@@ -679,4 +690,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8"
-content-hash = "f20e29bada880bbb0aad20d2a7ac282d09c3e89d4aac06072cb82a9157f4023a"
+content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d"
diff --git a/pyproject.toml b/pyproject.toml
index f5828a58e..6d0fa4215 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,7 +14,7 @@ packages = [
 ]
 
 [tool.poetry.scripts]
-git-fleximod = "git_fleximod.git-fleximod:main"
+git-fleximod = "git_fleximod.git_fleximod:main"
 fsspec = "fsspec.fuse:main"
 
 [tool.poetry.dependencies]
@@ -24,6 +24,7 @@ sphinx = "^5.0.0"
 fsspec = "^2023.12.2"
 wheel = "^0.42.0"
 pytest = "^8.0.0"
+pyfakefs = "^5.3.5"
 
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"
diff --git a/tests/test_import.py b/tests/test_a_import.py
similarity index 100%
rename from tests/test_import.py
rename to tests/test_a_import.py
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
new file mode 100644
index 000000000..791587628
--- /dev/null
+++ b/tests/test_checkout.py
@@ -0,0 +1,32 @@
+import pytest
+from pathlib import Path
+
+def test_basic_checkout(git_fleximod, test_repo):
+    # Prepare a simple .gitmodules
+    
+    gitmodules_content = """
+    [submodule "test_submodule"]
+        path = modules/test
+        url = https://github.com/ESMCI/mpi-serial.git
+        fxtag = MPIserial_2.4.0
+        fxurl = https://github.com/ESMCI/mpi-serial.git
+        fxrequired = T:T
+    """
+    (test_repo / ".gitmodules").write_text(gitmodules_content)
+    
+    # Run the command
+    result = git_fleximod("checkout")
+
+    # Assertions
+    assert result.returncode == 0
+    assert Path(test_repo / "modules/test").exists()   # Did the submodule directory get created?
+
+    status = git_fleximod("status")
+
+    assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout
+
+    result = git_fleximod("update")
+    assert result.returncode == 0
+
+    status = git_fleximod("status")
+    assert "test_submodule at tag MPIserial_2.4.0" in status.stdout
diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_checkout.py
new file mode 100644
index 000000000..0633802cb
--- /dev/null
+++ b/tests/test_sparse_checkout.py
@@ -0,0 +1,38 @@
+import pytest
+from shutil import rmtree
+from pathlib import Path
+from git_fleximod.gitinterface import GitInterface
+
+def test_sparse_checkout(git_fleximod, test_repo_base):
+    # Prepare a simple .gitmodules
+    gitmodules_content = (test_repo_base / ".gitmodules").read_text() + """
+    [submodule "test_sparse_submodule"]
+        path = modules/sparse_test
+        url = https://github.com/ESMCI/mpi-serial.git
+        fxtag = MPIserial_2.5.0
+        fxurl = https://github.com/ESMCI/mpi-serial.git
+        fxsparse = ../.sparse_file_list
+    """
+    (test_repo_base / ".gitmodules").write_text(gitmodules_content)
+
+    # Add the sparse checkout file
+    sparse_content = """m4
+"""
+    (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content)
+
+    result = git_fleximod("checkout")
+    
+    # Assertions
+    assert result.returncode == 0
+    assert Path(test_repo_base / "modules/sparse_test").exists()   # Did the submodule directory get created?
+    assert Path(test_repo_base /  "modules/sparse_test/m4").exists()   # Did the submodule sparse directory get created?
+    assert not Path(test_repo_base /  "modules/sparse_test/README").exists()   # Did only the submodule sparse directory get created?
+    status = git_fleximod("status test_sparse_submodule")
+
+    assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout
+
+    result = git_fleximod("update")
+    assert result.returncode == 0
+
+    status = git_fleximod("status test_sparse_submodule")
+    assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout

From 8e04947c985c02063c246d20ac5b1ef4ecd18fd4 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 9 Feb 2024 15:51:58 -0700
Subject: [PATCH 090/131] add pytest to workflow

---
 .github/workflows/pytest.yaml | 73 +++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)
 create mode 100644 .github/workflows/pytest.yaml

diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
new file mode 100644
index 000000000..8d2c2cb6e
--- /dev/null
+++ b/.github/workflows/pytest.yaml
@@ -0,0 +1,73 @@
+# Run this job on pushes to `main`, and for pull requests. If you don't specify
+# `branches: [main], then this actions runs _twice_ on pull requests, which is
+# annoying.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and
+      # reference the matrixe python version here.
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow
+      # from installing Poetry every time, which can be slow. Note the use of the Poetry version
+      # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache
+      # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be
+      # mildly cleaner by using an environment variable, but I don't really care.
+      - name: cache poetry install
+        uses: actions/cache@v4
+        with:
+          path: ~/.local
+          key: poetry-1.7.1
+
+      # Install Poetry. You could do this manually, or there are several actions that do this.
+      # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to
+      # Poetry's default install script, which feels correct. I pin the Poetry version here
+      # because Poetry does occasionally change APIs between versions and I don't want my
+      # actions to break if it does.
+      #
+      # The key configuration value here is `virtualenvs-in-project: true`: this creates the
+      # venv as a `.venv` in your testing directory, which allows the next step to easily
+      # cache it.
+      - uses: snok/install-poetry@v1
+        with:
+          version: 1.7.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache
+      # key: if you're using multiple Python versions, or multiple OSes, you'd need to include
+      # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock.
+      - name: cache deps
+        id: cache-deps
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: pydeps-${{ hashFiles('**/poetry.lock') }}
+
+      # Install dependencies. `--no-root` means "install all dependencies but not the project
+      # itself", which is what you want to avoid caching _your_ code. The `if` statement
+      # ensures this only runs on a cache miss.
+      - run: poetry install --no-interaction --no-root
+        if: steps.cache-deps.outputs.cache-hit != 'true'
+
+      # Now install _your_ project. This isn't necessary for many types of projects -- particularly
+      # things like Django apps don't need this. But it's a good idea since it fully-exercises the
+      # pyproject.toml and makes that if you add things like console-scripts at some point that
+      # they'll be installed and working.
+      - run: poetry install --no-interaction
+
+      # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml`
+      # so this line is super-simple. But it could be as complex as you need.
+      - run: poetry run pytest
+

From c1aef37f7b1e0b30399ed20d093a0756e658b25c Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 9 Feb 2024 15:52:42 -0700
Subject: [PATCH 091/131] add pytest to workflow

---
 .github/workflows/pytest.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
index 8d2c2cb6e..4809a2787 100644
--- a/.github/workflows/pytest.yaml
+++ b/.github/workflows/pytest.yaml
@@ -6,6 +6,7 @@ on:
   push:
     branches: [main]
   pull_request:
+    branches: [main]
 
 jobs:
   test:

From e8b6d3cc382785d787ee5fc482a974ef0c1be2b8 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 9 Feb 2024 15:55:32 -0700
Subject: [PATCH 092/131] add pytest fixtures

---
 tests/conftest.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 tests/conftest.py

diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 000000000..1cb059dac
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,51 @@
+import pytest 
+from git_fleximod.gitinterface import GitInterface
+import os
+import subprocess
+import logging
+from pathlib import Path
+
+@pytest.fixture(scope='session')
+def logger():
+    logging.basicConfig(
+        level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()]
+    )
+    logger = logging.getLogger(__name__)
+    return logger
+
+@pytest.fixture
+def test_repo_base(tmp_path, logger):
+    test_dir = tmp_path / "test_repo"
+    test_dir.mkdir()
+    str_path = str(test_dir)
+    gitp = GitInterface(str_path, logger)
+    #subprocess.run(["git", "init"], cwd=test_dir)
+    assert test_dir.joinpath(".git").is_dir()
+    return test_dir
+    
+@pytest.fixture(params=["modules/test"])
+def test_repo(request, test_repo_base, logger):
+    subrepo_path = request.param
+    gitp = GitInterface(str(test_repo_base), logger)
+    gitp.git_operation("submodule", "add", "--depth","1","--name","test_submodule", "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
+    #    subprocess.run(
+    assert test_repo_base.joinpath(".gitmodules").is_file()
+    return test_repo_base
+
+@pytest.fixture
+def git_fleximod(test_repo):
+    def _run_fleximod(args, input=None):
+        cmd = ["git", "fleximod"] + args.split()
+        result = subprocess.run(cmd, cwd=test_repo, input=input, 
+                                stdout=subprocess.PIPE, stderr=subprocess.PIPE, 
+                                text=True)
+        return result
+    return _run_fleximod
+
+    
+@pytest.fixture
+def deinit_submodule(test_repo, logger):
+    def _deinit(submodule_path):
+        gitp = GitInterface(str(test_repo), logger)
+        gitp.git_operation( "submodule", "deinit", "-f", submodule_path) 
+    yield _deinit 

From c1edd2c894d6f28903c70caa2ff5c7a6e892da26 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 11 Feb 2024 10:49:54 -0700
Subject: [PATCH 093/131] all working

---
 README.md                     | 18 +++++----
 git_fleximod/git_fleximod.py  |  7 ++--
 pyproject.toml                |  6 +++
 tests/conftest.py             | 73 +++++++++++++++++++++++++----------
 tests/test_checkout.py        | 54 ++++++++++++++------------
 tests/test_sparse_checkout.py | 52 ++++++++++++-------------
 6 files changed, 127 insertions(+), 83 deletions(-)

diff --git a/README.md b/README.md
index 7c968e70b..259ba8b54 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
 # git-fleximod
 
-Flexible Submodule Management for Git
+Flexible, Enhanced Submodule Management for Git
 
 ## Overview
 
-Git-fleximod is a Python-based tool that extends Git's submodule capabilities, offering additional features for managing submodules in a more flexible and efficient way.
+Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way.
 
 ## Installation
 
@@ -30,10 +30,10 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o
 
     fxtag: Specify a specific tag or branch to checkout for a submodule.
     fxrequired: Mark a submodule's checkout behavior, with allowed values:
-      - T:T: Top-level and required (checked out only when this is the Toplevel module).
-      - T:F: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
-      - I:T: Internal and required (always checked out).
-      - I:F: Internal and optional (checked out with --optional flag).
+      - ToplevelOnlyRequired: Top-level and required (checked out only when this is the Toplevel module).
+      - ToplevelOnlyOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
+      - AlwaysRequired:       Always required (always checked out).
+      - AlwaysOptional:       Always optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
     fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
 
@@ -71,6 +71,7 @@ Example .gitmodules entry:
         path = src/physics/cosp2/src
         url = https://github.com/CFMIP/COSPv2.0
         fxsparse = ../.cosp_sparse_checkout
+        fxrequired = AlwaysRequired
         fxtag = v2.1.4cesm
 ```
 Explanation:
@@ -80,14 +81,15 @@ should be checked out into the directory src/physics/cosp2/src
 relative to the .gitmodules directory.  It should be checked out from
 the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
 described in the file ../.cosp_sparse_checkout relative to the path
-directory.
+directory.  It should be checked out anytime this .gitmodules entry is
+read.
 
 Additional example:
 ```ini, toml
     [submodule "cime"]
         path = cime
         url = https://github.com/jedwards4b/cime
-        fxrequired = T:T
+        fxrequired = ToplevelOnlyRequired
         fxtag = cime6.0.198_rme01
 ```
 
diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 1b6c357b9..df76127b3 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -24,9 +24,9 @@ def commandline_arguments(args=None):
 
     # explicitly listing a component overrides the optional flag
     if options.optional or options.components:
-        fxrequired = ["T:T", "T:F", "I:T"]
+        fxrequired = ["ToplevelOnlyRequired", "ToplevelOnlyOptional", "AlwaysRequired", "AlwaysOptional"]
     else:
-        fxrequired = ["T:T", "I:T"]
+        fxrequired = ["ToplevelOnlyRequired", "AlwaysRequired"]
 
     action = options.action
     if not action:
@@ -66,6 +66,7 @@ def commandline_arguments(args=None):
 def submodule_sparse_checkout(
     root_dir, name, url, path, sparsefile, tag="master", fxhash=None
 ):
+    logger.info("Called sparse_checkout for {}".format(name))
     # first create the module directory
     if not os.path.isdir(os.path.join(root_dir,path)):
         os.makedirs(os.path.join(root_dir,path))
@@ -344,7 +345,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
         url = gitmodules.get(name, "url")
 
         if fxrequired and fxrequired not in requiredlist:
-            if "T:F" == fxrequired:
+            if "Optional" in fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
 
diff --git a/pyproject.toml b/pyproject.toml
index 6d0fa4215..772b15421 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,6 +29,12 @@ pyfakefs = "^5.3.5"
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"
 
+[tool.pytest.ini_options]
+markers = [
+        "skip_after_first: only run on first iteration"
+]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
diff --git a/tests/conftest.py b/tests/conftest.py
index 1cb059dac..d6fec44f3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,39 +13,72 @@ def logger():
     logger = logging.getLogger(__name__)
     return logger
 
+@pytest.fixture(params=[
+    {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """
+    [submodule "test_submodule"]
+    path = modules/test
+    url = https://github.com/ESMCI/mpi-serial.git
+    fxtag = MPIserial_2.4.0
+    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxrequired = ToplevelOnlyRequired
+"""},
+    {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", "gitmodules_content": """
+    [submodule "test_optional"]
+    path = modules/test_optional
+    url = https://github.com/ESMCI/mpi-serial.git
+    fxtag = MPIserial_2.4.0
+    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxrequired = ToplevelOnlyRequired
+"""},
+    {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", "gitmodules_content": """
+    [submodule "test_alwaysoptional"]
+    path = modules/test_alwaysoptional
+    url = https://github.com/ESMCI/mpi-serial.git
+    fxtag = MPIserial_2.3.0
+    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxrequired = AlwaysOptional
+"""},    
+    {"subrepo_path": "modules/test_sparse", "submodule_name": "test_sparse", "gitmodules_content": """
+    [submodule "test_sparse"]
+    path = modules/test_sparse
+    url = https://github.com/ESMCI/mpi-serial.git
+    fxtag = MPIserial_2.5.0
+    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxrequired = AlwaysRequired
+    fxsparse = ../.sparse_file_list
+"""},
+])
+
+def shared_repos(request):
+    return request.param
+
+@pytest.fixture
+def test_repo(shared_repos, test_repo_base, logger): 
+    subrepo_path = shared_repos["subrepo_path"]
+    submodule_name = shared_repos["submodule_name"]
+
+    gitp = GitInterface(str(test_repo_base), logger)
+    gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
+    assert test_repo_base.joinpath(".gitmodules").is_file()
+    return test_repo_base
+
 @pytest.fixture
 def test_repo_base(tmp_path, logger):
-    test_dir = tmp_path / "test_repo"
+    test_dir = tmp_path / "testrepo"
     test_dir.mkdir()
     str_path = str(test_dir)
     gitp = GitInterface(str_path, logger)
-    #subprocess.run(["git", "init"], cwd=test_dir)
     assert test_dir.joinpath(".git").is_dir()
+    (test_dir / "modules").mkdir()
     return test_dir
     
-@pytest.fixture(params=["modules/test"])
-def test_repo(request, test_repo_base, logger):
-    subrepo_path = request.param
-    gitp = GitInterface(str(test_repo_base), logger)
-    gitp.git_operation("submodule", "add", "--depth","1","--name","test_submodule", "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
-    #    subprocess.run(
-    assert test_repo_base.joinpath(".gitmodules").is_file()
-    return test_repo_base
-
 @pytest.fixture
-def git_fleximod(test_repo):
+def git_fleximod(test_repo_base):
     def _run_fleximod(args, input=None):
         cmd = ["git", "fleximod"] + args.split()
-        result = subprocess.run(cmd, cwd=test_repo, input=input, 
+        result = subprocess.run(cmd, cwd=test_repo_base, input=input, 
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 
                                 text=True)
         return result
     return _run_fleximod
 
-    
-@pytest.fixture
-def deinit_submodule(test_repo, logger):
-    def _deinit(submodule_path):
-        gitp = GitInterface(str(test_repo), logger)
-        gitp.git_operation( "submodule", "deinit", "-f", submodule_path) 
-    yield _deinit 
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 791587628..6d17ce18f 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -1,32 +1,38 @@
 import pytest
 from pathlib import Path
 
-def test_basic_checkout(git_fleximod, test_repo):
-    # Prepare a simple .gitmodules
-    
-    gitmodules_content = """
-    [submodule "test_submodule"]
+@pytest.fixture(params=[
+    {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """                                                                                                                                                                                      [submodule "test_submodule"]
         path = modules/test
         url = https://github.com/ESMCI/mpi-serial.git
         fxtag = MPIserial_2.4.0
         fxurl = https://github.com/ESMCI/mpi-serial.git
-        fxrequired = T:T
-    """
-    (test_repo / ".gitmodules").write_text(gitmodules_content)
+        fxrequired = ToplevelOnlyRequired
+"""},
+])
+def test_config(request):                
+    return request.param
+                
+def test_basic_checkout(git_fleximod, test_repo, test_config):
+    # Prepare a simple .gitmodules
+    gm = test_config['gitmodules_content']
+    file_path = (test_repo / ".gitmodules")
+    if not file_path.exists():
+        file_path.write_text(gm)
     
-    # Run the command
-    result = git_fleximod("checkout")
-
-    # Assertions
-    assert result.returncode == 0
-    assert Path(test_repo / "modules/test").exists()   # Did the submodule directory get created?
-
-    status = git_fleximod("status")
-
-    assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout
-
-    result = git_fleximod("update")
-    assert result.returncode == 0
-
-    status = git_fleximod("status")
-    assert "test_submodule at tag MPIserial_2.4.0" in status.stdout
+        # Run the command
+        result = git_fleximod("checkout test_submodule")
+        
+        # Assertions
+        assert result.returncode == 0
+        assert Path(test_repo / "modules/test").exists()   # Did the submodule directory get created?
+        
+        status = git_fleximod("status")
+        
+        assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout
+        
+        result = git_fleximod("update")
+        assert result.returncode == 0
+        
+        status = git_fleximod("status")
+        assert "test_submodule at tag MPIserial_2.4.0" in status.stdout
diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_checkout.py
index 0633802cb..7276e18d3 100644
--- a/tests/test_sparse_checkout.py
+++ b/tests/test_sparse_checkout.py
@@ -3,36 +3,32 @@
 from pathlib import Path
 from git_fleximod.gitinterface import GitInterface
 
-def test_sparse_checkout(git_fleximod, test_repo_base):
-    # Prepare a simple .gitmodules
-    gitmodules_content = (test_repo_base / ".gitmodules").read_text() + """
-    [submodule "test_sparse_submodule"]
-        path = modules/sparse_test
-        url = https://github.com/ESMCI/mpi-serial.git
-        fxtag = MPIserial_2.5.0
-        fxurl = https://github.com/ESMCI/mpi-serial.git
-        fxsparse = ../.sparse_file_list
-    """
-    (test_repo_base / ".gitmodules").write_text(gitmodules_content)
+def test_sparse_checkout(shared_repos, git_fleximod, test_repo_base):
+    repo_name = shared_repos["submodule_name"]
+    if repo_name == "test_sparse":
+        gm = shared_repos["gitmodules_content"]
+        (test_repo_base / ".gitmodules").write_text(gm)
 
-    # Add the sparse checkout file
-    sparse_content = """m4
+        # Add the sparse checkout file
+        sparse_content = """m4
 """
-    (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content)
 
-    result = git_fleximod("checkout")
-    
-    # Assertions
-    assert result.returncode == 0
-    assert Path(test_repo_base / "modules/sparse_test").exists()   # Did the submodule directory get created?
-    assert Path(test_repo_base /  "modules/sparse_test/m4").exists()   # Did the submodule sparse directory get created?
-    assert not Path(test_repo_base /  "modules/sparse_test/README").exists()   # Did only the submodule sparse directory get created?
-    status = git_fleximod("status test_sparse_submodule")
-
-    assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout
+        (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content)
 
-    result = git_fleximod("update")
-    assert result.returncode == 0
+        result = git_fleximod(f"checkout {repo_name}")
+    
+        # Assertions
+        assert result.returncode == 0
+        assert Path(test_repo_base / f"modules/{repo_name}").exists()   # Did the submodule directory get created?
+        assert Path(test_repo_base /  f"modules/{repo_name}/m4").exists()   # Did the submodule sparse directory get created?
+        assert not Path(test_repo_base /  f"modules/{repo_name}/README").exists()   # Did only the submodule sparse directory get created?
+        status = git_fleximod(f"status {repo_name}")
+        
+        assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout
+        
+        result = git_fleximod(f"update {repo_name}")
+        assert result.returncode == 0
+        
+        status = git_fleximod(f"status {repo_name}")
+        assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout
 
-    status = git_fleximod("status test_sparse_submodule")
-    assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout

From 03156218c7ddf4387fd5a8b74b8550ddae7d08da Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 11 Feb 2024 11:47:20 -0700
Subject: [PATCH 094/131] clean up and combine tests

---
 tests/conftest.py             | 50 +++++++++++++++++++++++-----------
 tests/test_checkout.py        | 51 +++++++++++++++--------------------
 tests/test_sparse_checkout.py | 34 -----------------------
 3 files changed, 57 insertions(+), 78 deletions(-)
 delete mode 100644 tests/test_sparse_checkout.py

diff --git a/tests/conftest.py b/tests/conftest.py
index d6fec44f3..ffff68956 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -14,7 +14,11 @@ def logger():
     return logger
 
 @pytest.fixture(params=[
-    {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """
+    {"subrepo_path": "modules/test",
+     "submodule_name": "test_submodule",
+     "status1" : "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status2" : "test_submodule at tag MPIserial_2.4.0",
+     "gitmodules_content" : """
     [submodule "test_submodule"]
     path = modules/test
     url = https://github.com/ESMCI/mpi-serial.git
@@ -22,7 +26,11 @@ def logger():
     fxurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = ToplevelOnlyRequired
 """},
-    {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", "gitmodules_content": """
+    {"subrepo_path": "modules/test_optional",
+     "submodule_name": "test_optional",
+     "status1" : "test_optional d82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status2" : "test_optional at tag MPIserial_2.4.0",
+     "gitmodules_content": """
     [submodule "test_optional"]
     path = modules/test_optional
     url = https://github.com/ESMCI/mpi-serial.git
@@ -30,7 +38,11 @@ def logger():
     fxurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = ToplevelOnlyRequired
 """},
-    {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", "gitmodules_content": """
+    {"subrepo_path": "modules/test_alwaysoptional",
+     "submodule_name": "test_alwaysoptional",
+     "status1" : "test_alwaysoptional d82ce7c is out of sync with .gitmodules MPIserial_2.3.0",
+     "status2" : "test_alwaysoptional at tag MPIserial_2.3.0",
+     "gitmodules_content": """
     [submodule "test_alwaysoptional"]
     path = modules/test_alwaysoptional
     url = https://github.com/ESMCI/mpi-serial.git
@@ -38,7 +50,11 @@ def logger():
     fxurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = AlwaysOptional
 """},    
-    {"subrepo_path": "modules/test_sparse", "submodule_name": "test_sparse", "gitmodules_content": """
+    {"subrepo_path": "modules/test_sparse",
+     "submodule_name": "test_sparse",
+     "status1" : "test_sparse at tag MPIserial_2.5.0",
+     "status2" : "test_sparse at tag MPIserial_2.5.0",
+     "gitmodules_content": """
     [submodule "test_sparse"]
     path = modules/test_sparse
     url = https://github.com/ESMCI/mpi-serial.git
@@ -53,30 +69,34 @@ def shared_repos(request):
     return request.param
 
 @pytest.fixture
-def test_repo(shared_repos, test_repo_base, logger): 
+def test_repo(shared_repos, tmp_path, logger):
     subrepo_path = shared_repos["subrepo_path"]
     submodule_name = shared_repos["submodule_name"]
-
-    gitp = GitInterface(str(test_repo_base), logger)
-    gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
-    assert test_repo_base.joinpath(".gitmodules").is_file()
-    return test_repo_base
-
-@pytest.fixture
-def test_repo_base(tmp_path, logger):
     test_dir = tmp_path / "testrepo"
     test_dir.mkdir()
     str_path = str(test_dir)
     gitp = GitInterface(str_path, logger)
     assert test_dir.joinpath(".git").is_dir()
     (test_dir / "modules").mkdir()
+    if "sparse" in submodule_name:
+        (test_dir / subrepo_path).mkdir()
+        # Add the sparse checkout file
+        sparse_content = """m4
+"""
+        (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content)
+    else:
+        gitp = GitInterface(str(test_dir), logger)
+        gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
+        assert test_dir.joinpath(".gitmodules").is_file()
+        
+        
     return test_dir
     
 @pytest.fixture
-def git_fleximod(test_repo_base):
+def git_fleximod(test_repo):
     def _run_fleximod(args, input=None):
         cmd = ["git", "fleximod"] + args.split()
-        result = subprocess.run(cmd, cwd=test_repo_base, input=input, 
+        result = subprocess.run(cmd, cwd=test_repo, input=input, 
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 
                                 text=True)
         return result
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 6d17ce18f..38bc5f9b2 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -1,38 +1,31 @@
 import pytest
 from pathlib import Path
-
-@pytest.fixture(params=[
-    {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """                                                                                                                                                                                      [submodule "test_submodule"]
-        path = modules/test
-        url = https://github.com/ESMCI/mpi-serial.git
-        fxtag = MPIserial_2.4.0
-        fxurl = https://github.com/ESMCI/mpi-serial.git
-        fxrequired = ToplevelOnlyRequired
-"""},
-])
-def test_config(request):                
-    return request.param
                 
-def test_basic_checkout(git_fleximod, test_repo, test_config):
+def test_basic_checkout(git_fleximod, test_repo, shared_repos):
     # Prepare a simple .gitmodules
-    gm = test_config['gitmodules_content']
+    gm = shared_repos['gitmodules_content']
     file_path = (test_repo / ".gitmodules")
-    if not file_path.exists():
-        file_path.write_text(gm)
+    repo_name = shared_repos["submodule_name"]
+    repo_path = shared_repos["subrepo_path"]
+
+    file_path.write_text(gm)
     
-        # Run the command
-        result = git_fleximod("checkout test_submodule")
-        
-        # Assertions
-        assert result.returncode == 0
-        assert Path(test_repo / "modules/test").exists()   # Did the submodule directory get created?
-        
-        status = git_fleximod("status")
+    # Run the command
+    result = git_fleximod(f"checkout {repo_name}")
+    
+    # Assertions
+    assert result.returncode == 0
+    assert Path(test_repo / repo_path).exists()   # Did the submodule directory get created?
+    if "sparse" in repo_name:
+        assert Path(test_repo /  f"{repo_path}/m4").exists()   # Did the submodule sparse directory get created?
+        assert not Path(test_repo /  f"{repo_path}/README").exists()   # Did only the submodule sparse directory get created?
+    
+    status = git_fleximod(f"status {repo_name}")
         
-        assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout
+    assert shared_repos["status1"] in status.stdout
         
-        result = git_fleximod("update")
-        assert result.returncode == 0
+    result = git_fleximod(f"update {repo_name}")
+    assert result.returncode == 0
         
-        status = git_fleximod("status")
-        assert "test_submodule at tag MPIserial_2.4.0" in status.stdout
+    status = git_fleximod(f"status {repo_name}")
+    assert shared_repos["status2"] in status.stdout
diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_checkout.py
deleted file mode 100644
index 7276e18d3..000000000
--- a/tests/test_sparse_checkout.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import pytest
-from shutil import rmtree
-from pathlib import Path
-from git_fleximod.gitinterface import GitInterface
-
-def test_sparse_checkout(shared_repos, git_fleximod, test_repo_base):
-    repo_name = shared_repos["submodule_name"]
-    if repo_name == "test_sparse":
-        gm = shared_repos["gitmodules_content"]
-        (test_repo_base / ".gitmodules").write_text(gm)
-
-        # Add the sparse checkout file
-        sparse_content = """m4
-"""
-
-        (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content)
-
-        result = git_fleximod(f"checkout {repo_name}")
-    
-        # Assertions
-        assert result.returncode == 0
-        assert Path(test_repo_base / f"modules/{repo_name}").exists()   # Did the submodule directory get created?
-        assert Path(test_repo_base /  f"modules/{repo_name}/m4").exists()   # Did the submodule sparse directory get created?
-        assert not Path(test_repo_base /  f"modules/{repo_name}/README").exists()   # Did only the submodule sparse directory get created?
-        status = git_fleximod(f"status {repo_name}")
-        
-        assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout
-        
-        result = git_fleximod(f"update {repo_name}")
-        assert result.returncode == 0
-        
-        status = git_fleximod(f"status {repo_name}")
-        assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout
-

From 9602eb0b777c3e174027bc9ea121da907879bac6 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 11 Feb 2024 14:43:38 -0700
Subject: [PATCH 095/131] all tests passing

---
 tests/conftest.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index ffff68956..a6ba85089 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -16,8 +16,10 @@ def logger():
 @pytest.fixture(params=[
     {"subrepo_path": "modules/test",
      "submodule_name": "test_submodule",
-     "status1" : "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
      "status2" : "test_submodule at tag MPIserial_2.4.0",
+     "status3" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status4" : "test_submodule at tag MPIserial_2.4.0",
      "gitmodules_content" : """
     [submodule "test_submodule"]
     path = modules/test
@@ -28,20 +30,24 @@ def logger():
 """},
     {"subrepo_path": "modules/test_optional",
      "submodule_name": "test_optional",
-     "status1" : "test_optional d82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
      "status2" : "test_optional at tag MPIserial_2.4.0",
+     "status3" : "test_optional not checked out, aligned at tag MPIserial_2.4.0",
+     "status4" : "test_optional at tag MPIserial_2.4.0",
      "gitmodules_content": """
     [submodule "test_optional"]
     path = modules/test_optional
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.4.0
     fxurl = https://github.com/ESMCI/mpi-serial.git
-    fxrequired = ToplevelOnlyRequired
+    fxrequired = ToplevelOnlyOptional
 """},
     {"subrepo_path": "modules/test_alwaysoptional",
      "submodule_name": "test_alwaysoptional",
-     "status1" : "test_alwaysoptional d82ce7c is out of sync with .gitmodules MPIserial_2.3.0",
+     "status1" : "test_alwaysoptional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.3.0",
      "status2" : "test_alwaysoptional at tag MPIserial_2.3.0",
+     "status3" : "test_alwaysoptional not checked out, aligned at tag MPIserial_2.3.0",
+     "status4" : "test_alwaysoptional at tag MPIserial_2.3.0",
      "gitmodules_content": """
     [submodule "test_alwaysoptional"]
     path = modules/test_alwaysoptional
@@ -54,6 +60,8 @@ def logger():
      "submodule_name": "test_sparse",
      "status1" : "test_sparse at tag MPIserial_2.5.0",
      "status2" : "test_sparse at tag MPIserial_2.5.0",
+     "status3" : "test_sparse at tag MPIserial_2.5.0",
+     "status4" : "test_sparse at tag MPIserial_2.5.0",
      "gitmodules_content": """
     [submodule "test_sparse"]
     path = modules/test_sparse
@@ -84,13 +92,18 @@ def test_repo(shared_repos, tmp_path, logger):
         sparse_content = """m4
 """
         (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content)
+        gitp.git_operation("add","modules/.sparse_file_list")
     else:
         gitp = GitInterface(str(test_dir), logger)
         gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path)
         assert test_dir.joinpath(".gitmodules").is_file()
+        gitp.git_operation("add",subrepo_path)
+    gitp.git_operation("commit","-a","-m","\"add submod\"")
+    test_dir2 = tmp_path / "testrepo2"
+    gitp.git_operation("clone",test_dir,test_dir2)
+    return test_dir2    
         
-        
-    return test_dir
+
     
 @pytest.fixture
 def git_fleximod(test_repo):

From b5c4515d0e9d1a48de659f96da99e6a8f3c20ed5 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 11 Feb 2024 15:52:38 -0700
Subject: [PATCH 096/131] thought it was working??

---
 tests/test_required.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 tests/test_required.py

diff --git a/tests/test_required.py b/tests/test_required.py
new file mode 100644
index 000000000..0997db69f
--- /dev/null
+++ b/tests/test_required.py
@@ -0,0 +1,33 @@
+import pytest
+from pathlib import Path
+                
+def test_required(git_fleximod, test_repo, shared_repos):
+    file_path = (test_repo / ".gitmodules")
+    gm = shared_repos["gitmodules_content"]
+    repo_name = shared_repos["submodule_name"]
+
+    status = git_fleximod(f"status {repo_name}")
+    print(f"stdout is {status.stdout}")
+    if file_path.exists():
+        with file_path.open("r") as f:
+            gitmodules_content = f.read()
+            # add the entry if it does not exist
+        if repo_name not in gitmodules_content:
+            file_path.write_text(gitmodules_content+gm)
+        # or if it is incomplete
+        elif gm not in gitmodules_content:
+                file_path.write_text(gm)
+    else:
+        file_path.write_text(gm)
+    status = git_fleximod(f"status {repo_name}")
+    result = git_fleximod("checkout")
+    assert result.returncode == 0
+    status = git_fleximod(f"status {repo_name}")
+    assert shared_repos["status3"] in status.stdout
+    if "not checked out" in status.stdout:
+        status = git_fleximod(f"checkout {repo_name}")
+        assert result.returncode == 0
+    status = git_fleximod(f"update {repo_name}")
+    assert result.returncode == 0
+    status = git_fleximod(f"status {repo_name}")
+    assert shared_repos["status4"] in status.stdout

From a3ce6116065af2e8e86bd3d7d6de06bc215a6e58 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 13 Feb 2024 15:50:55 -0700
Subject: [PATCH 097/131] test_complex is still a work in progress

---
 tests/conftest.py      | 18 +++++++++++++-----
 tests/test_checkout.py |  2 +-
 tests/test_complex.py  | 35 +++++++++++++++++++++++++++++++++++
 tests/test_required.py | 11 ++++-------
 4 files changed, 53 insertions(+), 13 deletions(-)
 create mode 100644 tests/test_complex.py

diff --git a/tests/conftest.py b/tests/conftest.py
index a6ba85089..9bd6a758b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,7 +13,7 @@ def logger():
     logger = logging.getLogger(__name__)
     return logger
 
-@pytest.fixture(params=[
+all_repos=[
     {"subrepo_path": "modules/test",
      "submodule_name": "test_submodule",
      "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
@@ -71,11 +71,21 @@ def logger():
     fxrequired = AlwaysRequired
     fxsparse = ../.sparse_file_list
 """},
-])
+]
+@pytest.fixture(params=all_repos)
 
 def shared_repos(request):
     return request.param
 
+@pytest.fixture
+def get_all_repos():
+    return all_repos
+
+def write_sparse_checkout_file(fp):
+    sparse_content = """m4
+"""
+    fp.write_text(sparse_content)
+
 @pytest.fixture
 def test_repo(shared_repos, tmp_path, logger):
     subrepo_path = shared_repos["subrepo_path"]
@@ -89,9 +99,7 @@ def test_repo(shared_repos, tmp_path, logger):
     if "sparse" in submodule_name:
         (test_dir / subrepo_path).mkdir()
         # Add the sparse checkout file
-        sparse_content = """m4
-"""
-        (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content)
+        write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list")
         gitp.git_operation("add","modules/.sparse_file_list")
     else:
         gitp = GitInterface(str(test_dir), logger)
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 38bc5f9b2..fd8ffc199 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -14,7 +14,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos):
     result = git_fleximod(f"checkout {repo_name}")
     
     # Assertions
-    assert result.returncode == 0
+    assert result.returncode == 0 
     assert Path(test_repo / repo_path).exists()   # Did the submodule directory get created?
     if "sparse" in repo_name:
         assert Path(test_repo /  f"{repo_path}/m4").exists()   # Did the submodule sparse directory get created?
diff --git a/tests/test_complex.py b/tests/test_complex.py
new file mode 100644
index 000000000..628a92d83
--- /dev/null
+++ b/tests/test_complex.py
@@ -0,0 +1,35 @@
+import pytest
+from pathlib import Path
+from git_fleximod.gitinterface import GitInterface
+
+def test_complex_checkout(git_fleximod, get_all_repos, test_repo, request, logger):
+    gitp = None
+    for repo in get_all_repos:
+        repo_name = repo["submodule_name"]
+        gm = repo["gitmodules_content"]
+        if "shared_repos0" in request.node.name:
+            if not gitp:
+                gitp = GitInterface(str(test_repo), logger)
+            file_path = (test_repo / ".gitmodules")
+            if file_path.exists():
+                with file_path.open("r") as f:
+                    gitmodules_content = f.read()
+                    print(f"content={gitmodules_content}")
+                    print(f"gm={gm}")
+                    # add the entry if it does not exist
+                    if repo_name not in gitmodules_content:
+                        file_path.write_text(gitmodules_content+gm)
+                    # or if it is incomplete
+                    elif gm not in gitmodules_content:
+                        file_path.write_text(gm)
+                        
+            else:
+                file_path.write_text(gm)
+            if "sparse" in repo_name:
+                print(f"writing sparse_file_list in {test_repo}")
+                write_sparse_checkout_file(test_repo / "modules" / ".sparse_file_list")
+                gitp.git_operation("add","modules/.sparse_file_list")
+            gitp.git_operation("commit","-a","-m","\"add submod\"")
+
+            
+    assert(False)
diff --git a/tests/test_required.py b/tests/test_required.py
index 0997db69f..b4d99c564 100644
--- a/tests/test_required.py
+++ b/tests/test_required.py
@@ -5,17 +5,14 @@ def test_required(git_fleximod, test_repo, shared_repos):
     file_path = (test_repo / ".gitmodules")
     gm = shared_repos["gitmodules_content"]
     repo_name = shared_repos["submodule_name"]
-
-    status = git_fleximod(f"status {repo_name}")
-    print(f"stdout is {status.stdout}")
     if file_path.exists():
         with file_path.open("r") as f:
             gitmodules_content = f.read()
             # add the entry if it does not exist
-        if repo_name not in gitmodules_content:
-            file_path.write_text(gitmodules_content+gm)
-        # or if it is incomplete
-        elif gm not in gitmodules_content:
+            if repo_name not in gitmodules_content:
+                file_path.write_text(gitmodules_content+gm)
+            # or if it is incomplete
+            elif gm not in gitmodules_content:
                 file_path.write_text(gm)
     else:
         file_path.write_text(gm)

From 8af16188fe7c531f3e651000505c06751784e988 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Tue, 13 Feb 2024 16:08:33 -0700
Subject: [PATCH 098/131] update should checkout if needed

---
 git_fleximod/git_fleximod.py                  | 34 +++++++++++++++++--
 ...arse_checkout.py => test_sparse_update.py} |  7 +---
 tests/{test_checkout.py => test_update.py}    |  7 +---
 3 files changed, 33 insertions(+), 15 deletions(-)
 rename tests/{test_sparse_checkout.py => test_sparse_update.py} (86%)
 rename tests/{test_checkout.py => test_update.py} (82%)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 1b6c357b9..593a8b3b2 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -275,7 +275,7 @@ def submodules_status(gitmodules, root_dir):
     return testfails, localmods
 
 
-def submodules_update(gitmodules, root_dir, force):
+def submodules_update(gitmodules, root_dir, requiredlist, force):
     _, localmods = submodules_status(gitmodules, root_dir)
     print("")
     if localmods and not force:
@@ -289,6 +289,34 @@ def submodules_update(gitmodules, root_dir, force):
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info("name={} path={} url={} fxtag={}".format(name, path, url, fxtag))
+        if not os.path.exists(os.path.join(path, ".git")):
+            fxrequired = gitmodules.get(name, "fxrequired")
+            fxsparse = gitmodules.get(name, "fxsparse")
+
+            if fxrequired and fxrequired not in requiredlist:
+                if "T:F" == fxrequired:
+                    print("Skipping optional component {}".format(name))
+                continue
+
+            if fxsparse:
+                logger.debug(
+                    "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format(
+                        root_dir, name, url, path, fxsparse, fxtag
+                    )
+                )
+                submodule_sparse_checkout(
+                    root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
+                )
+            else:
+                logger.debug(
+                    "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
+                )
+                
+                single_submodule_checkout(
+                    root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash
+                )
+
+
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
@@ -326,7 +354,7 @@ def submodules_update(gitmodules, root_dir, force):
                 else:
                     print(f"{name:>20} up to date.")
 
-
+# checkout is done by update if required so this function may be depricated
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
     _, localmods = submodules_status(gitmodules, root_dir)
     print("")
@@ -424,7 +452,7 @@ def main():
         sys.exit("No submodule components found")
     retval = 0
     if action == "update":
-        submodules_update(gitmodules, root_dir, force)
+        submodules_update(gitmodules, root_dir, fxrequired, force)
     elif action == "checkout":
         submodules_checkout(gitmodules, root_dir, fxrequired, force)
     elif action == "status":
diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_update.py
similarity index 86%
rename from tests/test_sparse_checkout.py
rename to tests/test_sparse_update.py
index 0633802cb..1e6ee83ba 100644
--- a/tests/test_sparse_checkout.py
+++ b/tests/test_sparse_update.py
@@ -20,7 +20,7 @@ def test_sparse_checkout(git_fleximod, test_repo_base):
 """
     (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content)
 
-    result = git_fleximod("checkout")
+    result = git_fleximod("update")
     
     # Assertions
     assert result.returncode == 0
@@ -31,8 +31,3 @@ def test_sparse_checkout(git_fleximod, test_repo_base):
 
     assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout
 
-    result = git_fleximod("update")
-    assert result.returncode == 0
-
-    status = git_fleximod("status test_sparse_submodule")
-    assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout
diff --git a/tests/test_checkout.py b/tests/test_update.py
similarity index 82%
rename from tests/test_checkout.py
rename to tests/test_update.py
index 791587628..baaea7603 100644
--- a/tests/test_checkout.py
+++ b/tests/test_update.py
@@ -15,7 +15,7 @@ def test_basic_checkout(git_fleximod, test_repo):
     (test_repo / ".gitmodules").write_text(gitmodules_content)
     
     # Run the command
-    result = git_fleximod("checkout")
+    result = git_fleximod("update")
 
     # Assertions
     assert result.returncode == 0
@@ -23,10 +23,5 @@ def test_basic_checkout(git_fleximod, test_repo):
 
     status = git_fleximod("status")
 
-    assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout
-
-    result = git_fleximod("update")
-    assert result.returncode == 0
-
     status = git_fleximod("status")
     assert "test_submodule at tag MPIserial_2.4.0" in status.stdout

From 7c47c05ffbd4d329e94ada655c9a467e8f968585 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 14 Feb 2024 09:18:43 -0700
Subject: [PATCH 099/131] restructure of tests working

---
 tests/conftest.py        | 20 +++++++++++++++-----
 tests/test_b_update.py   |  4 ++--
 tests/test_c_required.py | 12 ++++--------
 tests/test_d_complex.py  | 34 +++-------------------------------
 4 files changed, 24 insertions(+), 46 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 9bd6a758b..fd3678f9e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -18,7 +18,7 @@ def logger():
      "submodule_name": "test_submodule",
      "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
      "status2" : "test_submodule at tag MPIserial_2.4.0",
-     "status3" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
+     "status3" : "test_submodule at tag MPIserial_2.4.0",
      "status4" : "test_submodule at tag MPIserial_2.4.0",
      "gitmodules_content" : """
     [submodule "test_submodule"]
@@ -84,6 +84,7 @@ def get_all_repos():
 def write_sparse_checkout_file(fp):
     sparse_content = """m4
 """
+    print(f"writing sparse_file_list \n")
     fp.write_text(sparse_content)
 
 @pytest.fixture
@@ -110,14 +111,23 @@ def test_repo(shared_repos, tmp_path, logger):
     test_dir2 = tmp_path / "testrepo2"
     gitp.git_operation("clone",test_dir,test_dir2)
     return test_dir2    
-        
 
+
+@pytest.fixture
+def complex_repo(tmp_path, logger):
+    test_dir = tmp_path / "testcomplex"
+    test_dir.mkdir()
+    str_path = str(test_dir)
+    gitp = GitInterface(str_path, logger)
+    gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test")
+    gitp.git_operation("fetch", "origin", "main")
+    return test_dir
     
 @pytest.fixture
-def git_fleximod(test_repo):
-    def _run_fleximod(args, input=None):
+def git_fleximod():
+    def _run_fleximod(path, args, input=None):
         cmd = ["git", "fleximod"] + args.split()
-        result = subprocess.run(cmd, cwd=test_repo, input=input, 
+        result = subprocess.run(cmd, cwd=path, input=input, 
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 
                                 text=True)
         return result
diff --git a/tests/test_b_update.py b/tests/test_b_update.py
index e03daa397..159f1cfae 100644
--- a/tests/test_b_update.py
+++ b/tests/test_b_update.py
@@ -11,7 +11,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos):
     file_path.write_text(gm)
     
     # Run the command
-    result = git_fleximod(f"update {repo_name}")
+    result = git_fleximod(test_repo, f"update {repo_name}")
 
     # Assertions
     assert result.returncode == 0 
@@ -20,7 +20,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos):
         assert Path(test_repo /  f"{repo_path}/m4").exists()   # Did the submodule sparse directory get created?
         assert not Path(test_repo /  f"{repo_path}/README").exists()   # Did only the submodule sparse directory get created?
     
-    status = git_fleximod(f"status {repo_name}")
+    status = git_fleximod(test_repo, f"status {repo_name}")
         
     assert shared_repos["status2"] in status.stdout
         
diff --git a/tests/test_c_required.py b/tests/test_c_required.py
index b4d99c564..d6cc040b8 100644
--- a/tests/test_c_required.py
+++ b/tests/test_c_required.py
@@ -16,15 +16,11 @@ def test_required(git_fleximod, test_repo, shared_repos):
                 file_path.write_text(gm)
     else:
         file_path.write_text(gm)
-    status = git_fleximod(f"status {repo_name}")
-    result = git_fleximod("checkout")
+    result = git_fleximod(test_repo, "update")
     assert result.returncode == 0
-    status = git_fleximod(f"status {repo_name}")
+    status = git_fleximod(test_repo, f"status {repo_name}")
     assert shared_repos["status3"] in status.stdout
-    if "not checked out" in status.stdout:
-        status = git_fleximod(f"checkout {repo_name}")
-        assert result.returncode == 0
-    status = git_fleximod(f"update {repo_name}")
+    status = git_fleximod(test_repo, f"update {repo_name}")
     assert result.returncode == 0
-    status = git_fleximod(f"status {repo_name}")
+    status = git_fleximod(test_repo, f"status {repo_name}")
     assert shared_repos["status4"] in status.stdout
diff --git a/tests/test_d_complex.py b/tests/test_d_complex.py
index 628a92d83..2983b0193 100644
--- a/tests/test_d_complex.py
+++ b/tests/test_d_complex.py
@@ -2,34 +2,6 @@
 from pathlib import Path
 from git_fleximod.gitinterface import GitInterface
 
-def test_complex_checkout(git_fleximod, get_all_repos, test_repo, request, logger):
-    gitp = None
-    for repo in get_all_repos:
-        repo_name = repo["submodule_name"]
-        gm = repo["gitmodules_content"]
-        if "shared_repos0" in request.node.name:
-            if not gitp:
-                gitp = GitInterface(str(test_repo), logger)
-            file_path = (test_repo / ".gitmodules")
-            if file_path.exists():
-                with file_path.open("r") as f:
-                    gitmodules_content = f.read()
-                    print(f"content={gitmodules_content}")
-                    print(f"gm={gm}")
-                    # add the entry if it does not exist
-                    if repo_name not in gitmodules_content:
-                        file_path.write_text(gitmodules_content+gm)
-                    # or if it is incomplete
-                    elif gm not in gitmodules_content:
-                        file_path.write_text(gm)
-                        
-            else:
-                file_path.write_text(gm)
-            if "sparse" in repo_name:
-                print(f"writing sparse_file_list in {test_repo}")
-                write_sparse_checkout_file(test_repo / "modules" / ".sparse_file_list")
-                gitp.git_operation("add","modules/.sparse_file_list")
-            gitp.git_operation("commit","-a","-m","\"add submod\"")
-
-            
-    assert(False)
+def test_complex_checkout(git_fleximod, complex_repo, logger):
+    status = git_fleximod(complex_repo, "status")
+    print(status)

From e6d4838ebeee5a7ce408aa098d9cfab0cd34d405 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 16 Feb 2024 17:40:26 -0700
Subject: [PATCH 100/131] all tests are passing and complete

---
 README.md                    |   6 +-
 git_fleximod/git_fleximod.py | 167 +++++++++++++++++++----------------
 tests/conftest.py            |  13 +--
 tests/test_c_required.py     |   4 +
 tests/test_d_complex.py      |  62 ++++++++++++-
 5 files changed, 167 insertions(+), 85 deletions(-)

diff --git a/README.md b/README.md
index 259ba8b54..94d85c415 100644
--- a/README.md
+++ b/README.md
@@ -30,8 +30,8 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
 
     fxtag: Specify a specific tag or branch to checkout for a submodule.
     fxrequired: Mark a submodule's checkout behavior, with allowed values:
-      - ToplevelOnlyRequired: Top-level and required (checked out only when this is the Toplevel module).
-      - ToplevelOnlyOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
+      - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module).
+      - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
       - AlwaysRequired:       Always required (always checked out).
       - AlwaysOptional:       Always optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
@@ -89,7 +89,7 @@ Additional example:
     [submodule "cime"]
         path = cime
         url = https://github.com/jedwards4b/cime
-        fxrequired = ToplevelOnlyRequired
+        fxrequired = ToplevelRequired
         fxtag = cime6.0.198_rme01
 ```
 
diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index f81e71381..08f31fa77 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -12,7 +12,8 @@
 
 # logger variable is global
 logger = None
-
+def fxrequired_allowed_values():
+    return ['ToplevelRequired', 'ToplevelOptional', 'AlwaysRequired', 'AlwaysOptional']
 
 def commandline_arguments(args=None):
     parser = cli.get_parser()
@@ -24,9 +25,9 @@ def commandline_arguments(args=None):
 
     # explicitly listing a component overrides the optional flag
     if options.optional or options.components:
-        fxrequired = ["ToplevelOnlyRequired", "ToplevelOnlyOptional", "AlwaysRequired", "AlwaysOptional"]
+        fxrequired = ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"]
     else:
-        fxrequired = ["ToplevelOnlyRequired", "AlwaysRequired"]
+        fxrequired = ["ToplevelRequired", "AlwaysRequired"]
 
     action = options.action
     if not action:
@@ -67,12 +68,16 @@ def submodule_sparse_checkout(
     root_dir, name, url, path, sparsefile, tag="master", fxhash=None
 ):
     logger.info("Called sparse_checkout for {}".format(name))
+    rgit = GitInterface(root_dir, logger)
+    superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+    if superroot:
+        gitroot = superroot
+    else:
+        gitroot = root_dir
+    assert(os.path.isdir(os.path.join(gitroot,".git")))
     # first create the module directory
     if not os.path.isdir(os.path.join(root_dir,path)):
         os.makedirs(os.path.join(root_dir,path))
-    # Check first if the module is already defined
-    # and the sparse-checkout file exists
-    git = GitInterface(root_dir, logger)
 
     # initialize a new git repo and set the sparse checkout flag
     sprep_repo = os.path.join(root_dir, path)
@@ -92,36 +97,39 @@ def submodule_sparse_checkout(
     sprepo_git.config_set_value("core", "sparseCheckout", "true")
 
     # set the repository remote
-    sprepo_git.git_operation("remote", "add", "origin", url)
 
-    superroot = git.git_operation("rev-parse", "--show-superproject-working-tree")
-    if os.path.isfile(os.path.join(root_dir, ".git")):
+    logger.info("Setting remote origin in {}/{}".format(root_dir,path))
+    status = sprepo_git.git_operation("remote", "-v")
+    if url not in status:
+        sprepo_git.git_operation("remote", "add", "origin", url)
+    
+    topgit = os.path.join(gitroot,".git")
+    
+    if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
-            gitpath = os.path.abspath(os.path.join(root_dir, f.read().split()[1]))
-        topgit = os.path.abspath(os.path.join(gitpath, "modules"))
+            gitpath = os.path.relpath(os.path.join(root_dir, f.read().split()[1]), start=os.path.join(root_dir,path))
+        topgit = os.path.join(gitpath, "modules")
     else:
-        topgit = os.path.abspath(os.path.join(root_dir, ".git", "modules"))
-
-    if not os.path.isdir(topgit):
-        os.makedirs(topgit)
-    topgit = os.path.join(topgit, name)
-    logger.debug(
-        "root_dir is {} topgit is {} superroot is {}".format(
-            root_dir, topgit, superroot
-        )
-    )
-
+        topgit = os.path.relpath(os.path.join(root_dir, ".git", "modules"), start=os.path.join(root_dir,path))
+    
+    with utils.pushd(sprep_repo):
+        if not os.path.isdir(topgit):
+            os.makedirs(topgit)
+    topgit += os.sep + name
+    
     if os.path.isdir(os.path.join(root_dir, path, ".git")):
-        shutil.move(os.path.join(root_dir, path, ".git"), topgit)
-        with open(os.path.join(root_dir, path, ".git"), "w") as f:
-            f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir, path)))
-
-    gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
-    if os.path.isfile(gitsparse):
-        logger.warning("submodule {} is already initialized".format(name))
-        return
-
-    shutil.copy(os.path.join(root_dir, path, sparsefile), gitsparse)
+        with utils.pushd(sprep_repo):
+            shutil.move(".git", topgit)
+            with open(".git", "w") as f:
+                f.write("gitdir: " + os.path.relpath(topgit))
+            #    assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
+            gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
+        if os.path.isfile(gitsparse):
+            logger.warning("submodule {} is already initialized {}".format(name, topgit))
+            return
+
+        with utils.pushd(sprep_repo):
+            shutil.copy(sparsefile, gitsparse)
 
     # Finally checkout the repo
     if fxhash:
@@ -132,28 +140,31 @@ def submodule_sparse_checkout(
         sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
         sprepo_git.git_operation("checkout", tag)
         print(f"Successfully checked out {name:>20} at {tag}")
-
+    rgit.config_set_value(f'submodule "{name}"',"active","true")
+    rgit.config_set_value(f'submodule "{name}"',"url",url)
 
 def single_submodule_checkout(
-    root, name, path, url=None, tag=None, force=False, fxhash=None
+        root, name, path, url=None, tag=None, force=False, fxhash=None, optional=False
 ):
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
-    if os.path.exists(os.path.join(repodir, ".git")):
-        logger.info("Submodule {} already checked out".format(name))
-        return
+    logger.info("Checkout {} into {}/{}".format(name,root,path))
     # if url is provided update to the new url
     tmpurl = None
+    repo_exists = False
+    if os.path.exists(os.path.join(repodir, ".git")):
+        logger.info("Submodule {} already checked out".format(name))
+        repo_exists = True
 
     # Look for a .gitmodules file in the newly checkedout repo
-    if url:
+    if not repo_exists and url:
         # ssh urls cause problems for those who dont have git accounts with ssh keys defined
         # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was
         # opened with a GitModules object we don't need to worry about restoring the file here
         # it will be done by the GitModules class
         if url.startswith("git@"):
             tmpurl = url
-            url = url.replace("git@github.com:", "https://github.com")
+            url = url.replace("git@github.com:", "https://github.com/")
             git.git_operation("clone", url, path)
             smgit = GitInterface(repodir, logger)
             if not tag and not fxhash:
@@ -170,26 +181,26 @@ def single_submodule_checkout(
                     if line.startswith("gitdir: "):
                         rootdotgit = line[8:].rstrip()
 
-            newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", path))
-            if not os.path.isdir(os.path.join(newpath, os.pardir)):
-                os.makedirs(os.path.abspath(os.path.join(newpath, os.pardir)))
-
+            newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name))
             shutil.move(os.path.join(repodir, ".git"), newpath)
             with open(os.path.join(repodir, ".git"), "w") as f:
-                f.write("gitdir: " + newpath)
+                f.write("gitdir: " + os.path.relpath(newpath, start=repodir))
 
-    if not tmpurl:
+    if not repo_exists or not tmpurl:
         logger.debug(git.git_operation("submodule", "update", "--init", "--", path))
 
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name} {repodir} {url}")
         gitmodules = GitModules(logger, confpath=repodir)
-        submodules_checkout(gitmodules, repodir, ["I:T"], force=force)
+        requiredlist = ["AlwaysRequired"]
+        if optional:
+            requiredlist.append("AlwaysOptional")
+        submodules_checkout(gitmodules, repodir, requiredlist, force=force)
     if os.path.exists(os.path.join(repodir, ".git")):
-        print(f"Successfully checked out {name}")
+        print(f"Successfully checked out {name} {repodir}")
     else:
-        utils.fatal_error(f"Failed to checkout {name}")
+        utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}")
 
     if tmpurl:
         print(git.git_operation("restore", ".gitmodules"))
@@ -289,33 +300,37 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
         fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-        logger.info("name={} path={} url={} fxtag={}".format(name, path, url, fxtag))
-        if not os.path.exists(os.path.join(path, ".git")):
-            fxrequired = gitmodules.get(name, "fxrequired")
-            fxsparse = gitmodules.get(name, "fxsparse")
-
-            if fxrequired and fxrequired not in requiredlist:
-                if "T:F" == fxrequired:
-                    print("Skipping optional component {}".format(name))
-                continue
-
-            if fxsparse:
-                logger.debug(
-                    "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format(
-                        root_dir, name, url, path, fxsparse, fxtag
-                    )
-                )
-                submodule_sparse_checkout(
-                    root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
-                )
-            else:
-                logger.debug(
-                    "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
+        logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist))
+#        if not os.path.exists(os.path.join(root_dir,path, ".git")):
+        fxrequired = gitmodules.get(name, "fxrequired")
+        assert(fxrequired in fxrequired_allowed_values())
+        rgit = GitInterface(root_dir, logger)
+        superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+            
+        fxsparse = gitmodules.get(name, "fxsparse")
+            
+        if fxrequired and (superroot and "Toplevel" in fxrequired) or fxrequired not in requiredlist:
+            if "ToplevelOptional" == fxrequired:
+                print("Skipping optional component {}".format(name))
+            continue
+        if fxsparse:
+            logger.debug(
+                "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format(
+                    root_dir, name, url, path, fxsparse, fxtag
                 )
+            )
+            submodule_sparse_checkout(
+                root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
+            )
+        else:
+            logger.info(
+                "Calling submodule_checkout({},{},{},{})".format(root_dir, name, path,url)
+            )
                 
-                single_submodule_checkout(
-                    root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash
-                )
+            single_submodule_checkout(
+                root_dir, name, path, url=url, tag=fxtag, force=force,
+                fxhash=fxhash, optional=("AlwaysOptional" in requiredlist)
+            )
 
 
         if os.path.exists(os.path.join(path, ".git")):
@@ -371,7 +386,6 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
         fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-
         if fxrequired and fxrequired not in requiredlist:
             if "Optional" in fxrequired:
                 print("Skipping optional component {}".format(name))
@@ -392,7 +406,8 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
             )
 
             single_submodule_checkout(
-                root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash
+                root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash,
+                optional = "AlwaysOptional" in requiredlist
             )
 
 
@@ -441,7 +456,7 @@ def main():
             )
 
         root_dir = os.path.dirname(file_path)
-    logger.info("root_dir is {}".format(root_dir))
+    logger.info("root_dir is {} includelist={} excludelist={}".format(root_dir, includelist, excludelist))
     gitmodules = GitModules(
         logger,
         confpath=root_dir,
diff --git a/tests/conftest.py b/tests/conftest.py
index fd3678f9e..b88fa61d2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -26,7 +26,7 @@ def logger():
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.4.0
     fxurl = https://github.com/ESMCI/mpi-serial.git
-    fxrequired = ToplevelOnlyRequired
+    fxrequired = ToplevelRequired
 """},
     {"subrepo_path": "modules/test_optional",
      "submodule_name": "test_optional",
@@ -35,12 +35,12 @@ def logger():
      "status3" : "test_optional not checked out, aligned at tag MPIserial_2.4.0",
      "status4" : "test_optional at tag MPIserial_2.4.0",
      "gitmodules_content": """
-    [submodule "test_optional"]
+     [submodule "test_optional"]
     path = modules/test_optional
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.4.0
     fxurl = https://github.com/ESMCI/mpi-serial.git
-    fxrequired = ToplevelOnlyOptional
+    fxrequired = ToplevelOptional
 """},
     {"subrepo_path": "modules/test_alwaysoptional",
      "submodule_name": "test_alwaysoptional",
@@ -84,7 +84,6 @@ def get_all_repos():
 def write_sparse_checkout_file(fp):
     sparse_content = """m4
 """
-    print(f"writing sparse_file_list \n")
     fp.write_text(sparse_content)
 
 @pytest.fixture
@@ -119,8 +118,9 @@ def complex_repo(tmp_path, logger):
     test_dir.mkdir()
     str_path = str(test_dir)
     gitp = GitInterface(str_path, logger)
-    gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test")
+    gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2")
     gitp.git_operation("fetch", "origin", "main")
+    gitp.git_operation("checkout", "main")
     return test_dir
     
 @pytest.fixture
@@ -130,6 +130,9 @@ def _run_fleximod(path, args, input=None):
         result = subprocess.run(cmd, cwd=path, input=input, 
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 
                                 text=True)
+        if result.returncode:
+            print(result.stdout)
+            print(result.stderr)
         return result
     return _run_fleximod
 
diff --git a/tests/test_c_required.py b/tests/test_c_required.py
index d6cc040b8..89ab8d294 100644
--- a/tests/test_c_required.py
+++ b/tests/test_c_required.py
@@ -20,6 +20,10 @@ def test_required(git_fleximod, test_repo, shared_repos):
     assert result.returncode == 0
     status = git_fleximod(test_repo, f"status {repo_name}")
     assert shared_repos["status3"] in status.stdout
+    status = git_fleximod(test_repo, f"update --optional")
+    assert result.returncode == 0
+    status = git_fleximod(test_repo, f"status {repo_name}")
+    assert shared_repos["status4"] in status.stdout
     status = git_fleximod(test_repo, f"update {repo_name}")
     assert result.returncode == 0
     status = git_fleximod(test_repo, f"status {repo_name}")
diff --git a/tests/test_d_complex.py b/tests/test_d_complex.py
index 2983b0193..fdce51627 100644
--- a/tests/test_d_complex.py
+++ b/tests/test_d_complex.py
@@ -4,4 +4,64 @@
 
 def test_complex_checkout(git_fleximod, complex_repo, logger):
     status = git_fleximod(complex_repo, "status")
-    print(status)
+    assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout)
+    assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout)
+    assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout)
+    assert("Complex not checked out, aligned at tag testtag01" in status.stdout)
+    assert("AlwaysOptional not checked out, aligned at tag MPIserial_2.3.0" in status.stdout)
+    
+    # This should checkout and update test_submodule and complex_sub
+    result = git_fleximod(complex_repo, "update")    
+    assert result.returncode == 0
+
+    status = git_fleximod(complex_repo, "status")
+    assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout)
+    assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout)
+    assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout)
+    assert("Complex at tag testtag01" in status.stdout)
+
+    # now check the complex_sub
+    root = (complex_repo / "modules" / "complex")
+    assert(not (root / "libraries" / "gptl" / ".git").exists())
+    assert(not (root / "libraries" / "mpi-serial" / ".git").exists())
+    assert((root / "modules" / "mpi-serial" / ".git").exists())
+    assert(not (root / "modules" / "mpi-serial2" / ".git").exists())
+    assert((root / "modules" / "mpi-sparse" / ".git").exists())
+    assert((root / "modules" / "mpi-sparse" / "m4").exists())
+    assert(not (root / "modules" / "mpi-sparse" / "README").exists())
+
+    # update a single optional submodule
+
+    result = git_fleximod(complex_repo, "update ToplevelOptional")
+    assert result.returncode == 0
+
+    status = git_fleximod(complex_repo, "status")
+    assert("ToplevelOptional at tag v5.3.2" in status.stdout)
+    assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout)
+    assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout)
+    assert("Complex at tag testtag01" in status.stdout)
+    assert("AlwaysOptional not checked out, aligned at tag MPIserial_2.3.0" in status.stdout)
+
+
+    # Finally update optional
+    result = git_fleximod(complex_repo, "update --optional")
+    assert result.returncode == 0
+
+    status = git_fleximod(complex_repo, "status")
+    assert("ToplevelOptional at tag v5.3.2" in status.stdout)
+    assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout)
+    assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout)
+    assert("Complex at tag testtag01" in status.stdout)
+    assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout)
+    
+    # now check the complex_sub
+    root = (complex_repo / "modules" / "complex" )
+    assert(not (root / "libraries" / "gptl" / ".git").exists())
+    assert(not (root / "libraries" / "mpi-serial" / ".git").exists())
+    assert((root / "modules" / "mpi-serial" / ".git").exists())
+    assert((root / "modules" / "mpi-serial2" / ".git").exists())
+    assert((root / "modules" / "mpi-sparse" / ".git").exists())
+    assert((root / "modules" / "mpi-sparse" / "m4").exists())
+    assert(not (root / "modules" / "mpi-sparse" / "README").exists())
+
+

From 194d845ad2131c7ce50e1f5a14fc5476799c87fb Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 17 Feb 2024 08:08:57 -0700
Subject: [PATCH 101/131] add git user info to tests

---
 .github/workflows/pytest.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
index 4809a2787..0868dd9a3 100644
--- a/.github/workflows/pytest.yaml
+++ b/.github/workflows/pytest.yaml
@@ -70,5 +70,8 @@ jobs:
 
       # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml`
       # so this line is super-simple. But it could be as complex as you need.
-      - run: poetry run pytest
+      - run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
+          poetry run pytest
 

From d64c5c6dd311364cc98ffc5ca39be425437e377c Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 17 Feb 2024 08:14:37 -0700
Subject: [PATCH 102/131] remove checkout user option

---
 git_fleximod/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 719b15620..80dbd58d5 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -25,11 +25,11 @@ def get_parser():
     #
     # user options
     #
-    choices = ["update", "checkout", "status", "test"]
+    choices = ["update", "status", "test"]
     parser.add_argument(
         "action",
         choices=choices,
-        default="checkout",
+        default="update",
         help=f"Subcommand of git-fleximod, choices are {choices[:-1]}",
     )
 

From 6bac7946376f7ae85511b4ed02ad4881f9e0440d Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sat, 17 Feb 2024 09:43:54 -0700
Subject: [PATCH 103/131] remove fxhash, putting hash in tag works fine

---
 git_fleximod/git_fleximod.py | 87 +++++++++++-------------------------
 tests/conftest.py            | 10 ++---
 2 files changed, 32 insertions(+), 65 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 08f31fa77..7a4470e24 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -31,7 +31,7 @@ def commandline_arguments(args=None):
 
     action = options.action
     if not action:
-        action = "checkout"
+        action = "update"
     handlers = [logging.StreamHandler()]
 
     if options.debug:
@@ -65,7 +65,7 @@ def commandline_arguments(args=None):
 
 
 def submodule_sparse_checkout(
-    root_dir, name, url, path, sparsefile, tag="master", fxhash=None
+    root_dir, name, url, path, sparsefile, tag="master"
 ):
     logger.info("Called sparse_checkout for {}".format(name))
     rgit = GitInterface(root_dir, logger)
@@ -132,19 +132,14 @@ def submodule_sparse_checkout(
             shutil.copy(sparsefile, gitsparse)
 
     # Finally checkout the repo
-    if fxhash:
-        sprepo_git.git_operation("fetch", "origin", "--tags")
-        sprepo_git.git_operation("checkout", fxhash)
-        print(f"Successfully checked out {name:>20} at {fxhash}")
-    else:
-        sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
-        sprepo_git.git_operation("checkout", tag)
-        print(f"Successfully checked out {name:>20} at {tag}")
+    sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
+    sprepo_git.git_operation("checkout", tag)
+    print(f"Successfully checked out {name:>20} at {tag}")
     rgit.config_set_value(f'submodule "{name}"',"active","true")
     rgit.config_set_value(f'submodule "{name}"',"url",url)
 
 def single_submodule_checkout(
-        root, name, path, url=None, tag=None, force=False, fxhash=None, optional=False
+        root, name, path, url=None, tag=None, force=False, optional=False
 ):
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
@@ -167,12 +162,9 @@ def single_submodule_checkout(
             url = url.replace("git@github.com:", "https://github.com/")
             git.git_operation("clone", url, path)
             smgit = GitInterface(repodir, logger)
-            if not tag and not fxhash:
+            if not tag:
                 tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
-            if fxhash:
-                smgit.git_operation("checkout", fxhash)
-            else:
-                smgit.git_operation("checkout", tag)
+            smgit.git_operation("checkout", tag)
             # Now need to move the .git dir to the submodule location
             rootdotgit = os.path.join(root, ".git")
             if os.path.isfile(rootdotgit):
@@ -214,9 +206,6 @@ def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
-        fxhash = gitmodules.get(name, "fxhash")
-        if tag and fxhash:
-            utils.fatal_error(f"{name:>20} cannot have both fxtag and fxhash")
         if not path:
             utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
@@ -234,48 +223,33 @@ def submodules_status(gitmodules, root_dir):
             if tag and tag == atag:
                 print(f"e {name:>20} not checked out, aligned at tag {tag}")
             elif tag:
-                print(
-                    f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}"
-                )
-                testfails += 1
-            elif fxhash:
-                n = len(fxhash)
-                smhash = rootgit.git_operation(
-                    "ls-tree", "--object-only", f"--abbrev={n}", "HEAD", path
-                )
-                if smhash == fxhash:
-                    print(f"  {name:>20} not checked out, aligned at hash {fxhash}")
+                ahash = rootgit.git_operation("submodule", "status", "{}".format(path)).rstrip()
+                ahash = ahash[1:len(tag)+1]
+                if tag == ahash:
+                    print(f"e {name:>20} not checked out, aligned at hash {ahash}")
                 else:
                     print(
-                        f"s {name:>20} not checked out, out of sync at hash {smhash}, expected hash is {fxhash}"
+                        f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}"
                     )
                     testfails += 1
             else:
-                print(f"e {name:>20} has no fxtag nor fxhash defined in .gitmodules")
+                print(f"e {name:>20} has no fxtag defined in .gitmodules")
                 testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 ahash = git.git_operation("status").partition("\n")[0].split()[-1]
-                if tag and atag != tag:
-                    print(f"s {name:>20} {atag} is out of sync with .gitmodules {tag}")
-                    testfails += 1
-                elif tag:
+                if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
-                elif fxhash:
-                    rootgit = GitInterface(root_dir, logger)
-                    n = len(fxhash)
-                    if ahash.startswith(fxhash):
-                        print(f"  {name:>20} at hash {fxhash}")
-                    else:
-                        print(
-                            f"s {name:>20} {ahash} is out of sync with .gitmodules {fxhash}"
-                        )
-                        testfails += 1
+                elif tag and ahash == tag:
+                    print(f"  {name:>20} at hash {ahash}")
+                elif tag:
+                    print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}")
+                    testfails += 1
                 else:
                     print(
-                        f"e {name:>20} has no fxtag nor fxhash defined in .gitmodules, module at {atag}"
+                        f"e {name:>20} has no fxtag defined in .gitmodules, module at {atag}"
                     )
                     testfails += 1
 
@@ -297,7 +271,6 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
         return
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
-        fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist))
@@ -320,7 +293,7 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 )
             )
             submodule_sparse_checkout(
-                root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
+                root_dir, name, url, path, fxsparse, tag=fxtag
             )
         else:
             logger.info(
@@ -329,7 +302,7 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 
             single_submodule_checkout(
                 root_dir, name, path, url=url, tag=fxtag, force=force,
-                fxhash=fxhash, optional=("AlwaysOptional" in requiredlist)
+                optional=("AlwaysOptional" in requiredlist)
             )
 
 
@@ -362,11 +335,8 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 if fxtag and fxtag != atag:
                     print(f"{name:>20} updated to {fxtag}")
                     git.git_operation("checkout", fxtag)
-                elif fxhash:
-                    print(f"{name:>20} updated to {fxhash}")
-                    git.git_operation("checkout", fxhash)
-                elif not (fxtag or fxhash):
-                    print(f"No fxtag nor fxhash found for submodule {name:>20}")
+                elif not fxtag:
+                    print(f"No fxtag found for submodule {name:>20}")
                 else:
                     print(f"{name:>20} up to date.")
 
@@ -383,7 +353,6 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
         fxrequired = gitmodules.get(name, "fxrequired")
         fxsparse = gitmodules.get(name, "fxsparse")
         fxtag = gitmodules.get(name, "fxtag")
-        fxhash = gitmodules.get(name, "fxhash")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         if fxrequired and fxrequired not in requiredlist:
@@ -398,7 +367,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
                 )
             )
             submodule_sparse_checkout(
-                root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash
+                root_dir, name, url, path, fxsparse, tag=fxtag
             )
         else:
             logger.debug(
@@ -406,7 +375,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
             )
 
             single_submodule_checkout(
-                root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash,
+                root_dir, name, path, url=url, tag=fxtag, force=force,
                 optional = "AlwaysOptional" in requiredlist
             )
 
@@ -469,8 +438,6 @@ def main():
     retval = 0
     if action == "update":
         submodules_update(gitmodules, root_dir, fxrequired, force)
-    elif action == "checkout":
-        submodules_checkout(gitmodules, root_dir, fxrequired, force)
     elif action == "status":
         submodules_status(gitmodules, root_dir)
     elif action == "test":
diff --git a/tests/conftest.py b/tests/conftest.py
index b88fa61d2..5fed907c4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -44,15 +44,15 @@ def logger():
 """},
     {"subrepo_path": "modules/test_alwaysoptional",
      "submodule_name": "test_alwaysoptional",
-     "status1" : "test_alwaysoptional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.3.0",
-     "status2" : "test_alwaysoptional at tag MPIserial_2.3.0",
-     "status3" : "test_alwaysoptional not checked out, aligned at tag MPIserial_2.3.0",
-     "status4" : "test_alwaysoptional at tag MPIserial_2.3.0",
+     "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c",
+     "status2" : "test_alwaysoptional at hash e5cf35c",
+     "status3" : "test_alwaysoptional not checked out, out of sync at tag MPIserial_2.3.0",
+     "status4" : "test_alwaysoptional at hash e5cf35c",
      "gitmodules_content": """
     [submodule "test_alwaysoptional"]
     path = modules/test_alwaysoptional
     url = https://github.com/ESMCI/mpi-serial.git
-    fxtag = MPIserial_2.3.0
+    fxtag = e5cf35c
     fxurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = AlwaysOptional
 """},    

From 678bae800ca818a0dd458dbc39334007da8f973d Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Sun, 18 Feb 2024 10:09:05 -0700
Subject: [PATCH 104/131] update version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 772b15421..e8c7bd58b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.6.2"
+version = "0.7.0"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]

From 8d0ed18d3a7f30048aa4b302eb607e27bf1d45ff Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 22 Feb 2024 09:25:59 -0700
Subject: [PATCH 105/131] add translater from manage_externals

---
 git_fleximod/gitinterface.py |  22 ++--
 git_fleximod/gitmodules.py   |  39 +++---
 git_fleximod/metoflexi.py    | 242 +++++++++++++++++++++++++++++++++++
 pyproject.toml               |   1 +
 4 files changed, 281 insertions(+), 23 deletions(-)
 create mode 100755 git_fleximod/metoflexi.py

diff --git a/git_fleximod/gitinterface.py b/git_fleximod/gitinterface.py
index 4d4c4f4b9..203c50030 100644
--- a/git_fleximod/gitinterface.py
+++ b/git_fleximod/gitinterface.py
@@ -1,25 +1,31 @@
 import os
 import sys
 from . import utils
+from pathlib import Path
 
 class GitInterface:
     def __init__(self, repo_path, logger):
         logger.debug("Initialize GitInterface for {}".format(repo_path))
-        self.repo_path = repo_path
+        if isinstance(repo_path, str):
+            self.repo_path = Path(repo_path).resolve()
+        elif isinstance(repo_path, Path):
+            self.repo_path = repo_path.resolve()
+        else:
+            raise TypeError("repo_path must be a str or Path object")
         self.logger = logger
         try:
             import git
 
             self._use_module = True
             try:
-                self.repo = git.Repo(repo_path)  # Initialize GitPython repo
+                self.repo = git.Repo(str(self.repo_path))  # Initialize GitPython repo
             except git.exc.InvalidGitRepositoryError:
                 self.git = git
                 self._init_git_repo()
             msg = "Using GitPython interface to git"
         except ImportError:
             self._use_module = False
-            if not os.path.exists(os.path.join(repo_path, ".git")):
+            if not (repo_path / ".git").exists():
                 self._init_git_repo()
             msg = "Using shell interface to git"
         self.logger.info(msg)
@@ -32,13 +38,13 @@ def _git_command(self, operation, *args):
             except Exception as e:
                 sys.exit(e)
         else:
-            return ["git", "-C", self.repo_path, operation] + list(args)
+            return ["git", "-C", str(self.repo_path), operation] + list(args)
 
     def _init_git_repo(self):
         if self._use_module:
-            self.repo = self.git.Repo.init(self.repo_path)
+            self.repo = self.git.Repo.init(str(self.repo_path))
         else:
-            command = ("git", "-C", self.repo_path, "init")
+            command = ("git", "-C", str(self.repo_path), "init")
             utils.execute_subprocess(command)
 
     # pylint: disable=unused-argument
@@ -58,7 +64,7 @@ def config_get_value(self, section, name):
             config = self.repo.config_reader()
             return config.get_value(section, name)
         else:
-            cmd = ("git", "-C", self.repo_path, "config", "--get", f"{section}.{name}")
+            cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}")
             output = utils.execute_subprocess(cmd, output_to_caller=True)
             return output.strip()
 
@@ -68,6 +74,6 @@ def config_set_value(self, section, name, value):
                 writer.set_value(section, name, value)
             writer.release()  # Ensure changes are saved
         else:
-            cmd = ("git", "-C", self.repo_path, "config", f"{section}.{name}", value)
+            cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value)
             self.logger.info(cmd)
             utils.execute_subprocess(cmd, output_to_caller=True)
diff --git a/git_fleximod/gitmodules.py b/git_fleximod/gitmodules.py
index ae0ebe121..68c82d066 100644
--- a/git_fleximod/gitmodules.py
+++ b/git_fleximod/gitmodules.py
@@ -1,14 +1,14 @@
-import os
 import shutil
-from configparser import ConfigParser
+from pathlib import Path
+from configparser import RawConfigParser, ConfigParser
 from .lstripreader import LstripReader
 
 
-class GitModules(ConfigParser):
+class GitModules(RawConfigParser):
     def __init__(
         self,
         logger,
-        confpath=os.getcwd(),
+        confpath=Path.cwd(),
         conffile=".gitmodules",
         includelist=None,
         excludelist=None,
@@ -25,25 +25,32 @@ def __init__(
                 confpath, conffile, includelist, excludelist
             )
         )
-        ConfigParser.__init__(self)
-        self.conf_file = os.path.join(confpath, conffile)
-        # first create a backup of this file to be restored on deletion of the object
-        shutil.copy(self.conf_file, self.conf_file + ".save")
-        self.read_file(LstripReader(self.conf_file), source=conffile)
+        super().__init__()
+        self.conf_file = (Path(confpath) / Path(conffile))
+        if self.conf_file.exists():
+            self.read_file(LstripReader(str(self.conf_file)), source=conffile)
         self.includelist = includelist
         self.excludelist = excludelist
+        self.isdirty = False
+        
+    def reload(self):
+        self.clear()
+        if self.conf_file.exists():
+            self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file)
 
+        
     def set(self, name, option, value):
         """
         Sets a configuration value for a specific submodule:
         Ensures the appropriate section exists for the submodule.
         Calls the parent class's set method to store the value.
         """
+        self.isdirty = True
         self.logger.debug("set called {} {} {}".format(name, option, value))
         section = f'submodule "{name}"'
         if not self.has_section(section):
             self.add_section(section)
-        ConfigParser.set(self, section, option, str(value))
+        super().set(section, option, str(value))
 
     # pylint: disable=redefined-builtin, arguments-differ
     def get(self, name, option, raw=False, vars=None, fallback=None):
@@ -62,12 +69,14 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
             return None
 
     def save(self):
-        print("Called gitmodules save, not expected")
-        #        self.write(open(self.conf_file, "w"))
-
+        if self.isdirty:
+            self.logger.info("Writing {}".format(self.conf_file))
+            with open(self.conf_file, "w") as fd:
+                self.write(fd)
+        self.isdirty = False
+        
     def __del__(self):
-        self.logger.debug("Destroying GitModules object")
-        shutil.move(self.conf_file + ".save", self.conf_file)
+        self.save()
 
     def sections(self):
         """Strip the submodule part out of section and just use the name"""
diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py
new file mode 100755
index 000000000..1d56bc9ad
--- /dev/null
+++ b/git_fleximod/metoflexi.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+from configparser import ConfigParser
+import sys
+import shutil
+from pathlib import Path
+import argparse
+import logging
+from git_fleximod.gitinterface import GitInterface
+from git_fleximod.gitmodules import GitModules
+from git_fleximod import utils
+
+logger = None
+
+def find_root_dir(filename=".git"):
+    d = Path.cwd()
+    root = Path(d.root)
+    while d != root:
+        attempt = d / filename
+        if attempt.is_dir():
+            return attempt
+        d = d.parent
+    return None
+
+
+def get_parser():
+    description = """
+    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
+    """
+    parser = argparse.ArgumentParser(
+        description=description, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument('-e', '--externals', nargs='?',
+                        default='Externals.cfg',
+                        help='The externals description filename. '
+                        'Default: %(default)s.')
+
+    parser.add_argument(
+        "-C",
+        "--path",
+        default=find_root_dir(),
+        help="Toplevel repository directory.  Defaults to top git directory relative to current.",
+    )
+
+    parser.add_argument(
+        "-g",
+        "--gitmodules",
+        nargs="?",
+        default=".gitmodules",
+        help="The submodule description filename. " "Default: %(default)s.",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Output additional information to "
+        "the screen and log file. This flag can be "
+        "used up to two times, increasing the "
+        "verbosity level each time.",
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        default=False,
+        help="DEVELOPER: output additional debugging "
+        "information to the screen and log file.",
+    )
+
+    return parser
+
+def commandline_arguments(args=None):
+    parser = get_parser()
+
+    options = parser.parse_args(args)
+    handlers = [logging.StreamHandler()]
+
+    if options.debug:
+        try:
+            open("fleximod.log", "w")
+        except PermissionError:
+            sys.exit("ABORT: Could not write file fleximod.log")
+        level = logging.DEBUG
+        handlers.append(logging.FileHandler("fleximod.log"))
+    elif options.verbose:
+        level = logging.INFO
+    else:
+        level = logging.WARNING
+    # Configure the root logger
+    logging.basicConfig(
+        level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers
+    )
+
+    return(
+        options.path,
+        options.gitmodules,
+        options.externals
+        )
+
+class ExternalRepoTranslator:
+    """
+    Translates external repositories configured in an INI-style externals file.
+    """
+
+    def __init__(self, rootpath, gitmodules, externals):
+        self.rootpath = rootpath
+        if gitmodules:
+            self.gitmodules = GitModules(logger, confpath=rootpath)
+        self.externals = (rootpath / Path(externals)).resolve()
+        print(f"Translating {self.externals}")
+        self.git = GitInterface(rootpath, logger)
+        
+#    def __del__(self):
+#        if (self.rootpath / "save.gitignore"):
+            
+
+    def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, protocol):
+        """
+        Translates a single repository based on configuration details.
+
+        Args:
+            rootpath (str): Root path of the main repository.
+            gitmodules (str): Path to the .gitmodules file.
+            tag (str): The tag to use for the external repository.
+            url (str): The URL of the external repository.
+            path (str): The relative path within the main repository for the external repository.
+            efile (str): The external file or file containing submodules.
+            hash_ (str): The commit hash to checkout (if applicable).
+            sparse (str):  Boolean indicating whether to use sparse checkout (if applicable).
+            protocol (str): The protocol to use (e.g., 'git', 'http').
+        """
+        assert protocol != "svn", "SVN protocol is not currently supported"
+        print(f"Translating repository {section}")
+        if efile:
+            file_path = Path(path) / Path(efile)
+            newroot = (self.rootpath / file_path).parent.resolve()
+            if not newroot.exists():
+                newroot.mkdir(parents=True)
+            logger.info("Newroot is {}".format(newroot))
+            newt = ExternalRepoTranslator(newroot, ".gitmodules", efile)
+            newt.translate_repo()
+        if protocol == "externals_only":
+            if tag:
+                self.gitmodules.set(section, "fxtag", tag)
+            if hash_:
+                self.gitmodules.set(section, "fxtag", hash_)
+            
+            self.gitmodules.set(section, "fxurl", url)
+            if sparse:
+                self.gitmodules.set(section, "fxsparse", sparse)
+            self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
+
+            return
+        
+        newpath = (self.rootpath / Path(path))
+        if newpath.exists():
+            shutil.rmtree(newpath)
+        logger.info("Creating directory {}".format(newpath))
+        newpath.mkdir(parents=True)
+        if tag:
+            logger.info("cloning {}".format(section))
+            try:
+                self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path)
+            except:
+                self.git.git_operation("clone", url, path)
+                with utils.pushd(newpath):
+                    ngit = GitInterface(newpath, logger)
+                    ngit.git_operation("checkout", tag)
+
+        if (newpath / ".gitignore").exists():
+            logger.info("Moving .gitignore file in {}".format(newpath))
+            (newpath / ".gitignore").rename((newpath / "save.gitignore"))
+            
+        if hash_:
+            self.git.git_operation("clone", url, path)
+            git = GitInterface(newpath, logger)
+            git.git_operation("fetch", "origin")
+            git.git_operation("checkout", hash_)
+        if sparse:
+            print("setting as sparse submodule {}".format(section))
+            sparsefile = (newpath / Path(sparse))
+            newfile = (newpath / ".git" / "info" / "sparse-checkout")
+            print(f"sparsefile {sparsefile} newfile {newfile}")
+            shutil.copy(sparsefile, newfile)
+        logger.info("adding submodule {}".format(section))
+        self.gitmodules.save()
+        self.git.git_operation("submodule", "add", "-f", "--name", section, url, path)
+        self.git.git_operation("submodule","absorbgitdirs")
+        self.gitmodules.reload()
+        if tag:
+            self.gitmodules.set(section, "fxtag", tag)
+        if hash_:
+           self.gitmodules.set(section, "fxtag", hash_)
+            
+        self.gitmodules.set(section, "fxurl", url)
+        if sparse:
+            self.gitmodules.set(section, "fxsparse", sparse)
+        self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
+    
+        
+    def translate_repo(self):
+        """
+        Translates external repositories defined within an external file.
+
+        Args:
+            rootpath (str): Root path of the main repository.
+            gitmodules (str): Path to the .gitmodules file.
+            external_file (str): The path to the external file containing repository definitions.
+        """
+        econfig = ConfigParser()
+        econfig.read((self.rootpath / Path(self.externals)))
+
+        for section in econfig.sections():
+            if section == "externals_description":
+                logger.info("skipping section {}".format(section))
+                return
+            logger.info("Translating section {}".format(section))
+            tag = econfig.get(section, "tag", raw=False, fallback=None)
+            url = econfig.get(section, "repo_url", raw=False, fallback=None)
+            path = econfig.get(section, "local_path", raw=False, fallback=None)
+            efile = econfig.get(section, "externals", raw=False, fallback=None)
+            hash_ = econfig.get(section, "hash", raw=False, fallback=None)
+            sparse = econfig.get(section, "sparse", raw=False, fallback=None)
+            protocol = econfig.get(section, "protocol", raw=False, fallback=None)
+
+            self.translate_single_repo(section, tag, url, path, efile, hash_, sparse, protocol)
+
+
+
+def _main():
+    rootpath, gitmodules, externals = commandline_arguments()
+    global logger
+    logger = logging.getLogger(__name__)
+    with utils.pushd(rootpath):
+        t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals)
+        logger.info("Translating {}".format(rootpath))
+        t.translate_repo()
+
+        
+if __name__ == "__main__":
+    sys.exit(_main())
diff --git a/pyproject.toml b/pyproject.toml
index e8c7bd58b..212b014d9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,7 @@ packages = [
 
 [tool.poetry.scripts]
 git-fleximod = "git_fleximod.git_fleximod:main"
+me2flexi = "git_fleximod.metoflexi:_main"
 fsspec = "fsspec.fuse:main"
 
 [tool.poetry.dependencies]

From 91df5a0ea5f8601c01c0462cf7e9a8aca84f2aa3 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Wed, 28 Feb 2024 16:00:42 -0700
Subject: [PATCH 106/131] update README and improve me2flexi

---
 README.md                    | 5 ++---
 git_fleximod/git_fleximod.py | 5 +++--
 git_fleximod/metoflexi.py    | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 94d85c415..c17c5b6f7 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,6 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
     Basic Usage:
         git fleximod <command> [options]
     Available Commands:
-        checkout: Checkout submodules according to git submodule hash configuration.
         status: Display the status of submodules.
         update: Update submodules to the tag indicated in .gitmodules variable fxtag.
         test: Make sure that fxtags and submodule hashes are consistant,
@@ -55,9 +54,9 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
 
 Here are some common usage examples:
 
-Checkout submodules, including optional ones:
+Update all submodules, including optional ones:
 ```bash
-  git fleximod checkout --optional
+  git fleximod update --optional
 ```
 
 Updating a specific submodule to the fxtag indicated in .gitmodules:
diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 7a4470e24..1f60b1822 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -132,8 +132,9 @@ def submodule_sparse_checkout(
             shutil.copy(sparsefile, gitsparse)
 
     # Finally checkout the repo
-    sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags")
+    sprepo_git.git_operation("fetch", "origin", "--tags")
     sprepo_git.git_operation("checkout", tag)
+        
     print(f"Successfully checked out {name:>20} at {tag}")
     rgit.config_set_value(f'submodule "{name}"',"active","true")
     rgit.config_set_value(f'submodule "{name}"',"url",url)
@@ -242,7 +243,7 @@ def submodules_status(gitmodules, root_dir):
                 ahash = git.git_operation("status").partition("\n")[0].split()[-1]
                 if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
-                elif tag and ahash == tag:
+                elif tag and ahash[:len(tag)] == tag:
                     print(f"  {name:>20} at hash {ahash}")
                 elif tag:
                     print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}")
diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py
index 1d56bc9ad..b607ad925 100755
--- a/git_fleximod/metoflexi.py
+++ b/git_fleximod/metoflexi.py
@@ -168,9 +168,9 @@ def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, p
                     ngit = GitInterface(newpath, logger)
                     ngit.git_operation("checkout", tag)
 
-        if (newpath / ".gitignore").exists():
-            logger.info("Moving .gitignore file in {}".format(newpath))
-            (newpath / ".gitignore").rename((newpath / "save.gitignore"))
+#        if (newpath / ".gitignore").exists():
+#            logger.info("Moving .gitignore file in {}".format(newpath))
+#            (newpath / ".gitignore").rename((newpath / "save.gitignore"))
             
         if hash_:
             self.git.git_operation("clone", url, path)

From b7914f26c22704b0aa984bad0978eeb7d88e55e0 Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 29 Feb 2024 15:25:23 -0700
Subject: [PATCH 107/131] correcting version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 212b014d9..cbbe83a40 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.0"
+version = "0.6.2"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]

From ff3fcb9587fabfe28e924b2249bcee89bb68d97c Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Thu, 29 Feb 2024 15:25:38 -0700
Subject: [PATCH 108/131] Bump to 0.7.0

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 80dbd58d5..f9cd67cff 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.6.2"
+__version__ = "0.7.0"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index cbbe83a40..212b014d9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.6.2"
+version = "0.7.0"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index 583cef45a..568226dfa 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.6.2"
+current = "0.7.0"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From fd53786233fd742ec7959908cc8a543dd5315d8f Mon Sep 17 00:00:00 2001
From: James Edwards <jedwards@ucar.edu>
Date: Fri, 1 Mar 2024 07:09:45 -0700
Subject: [PATCH 109/131] code cleanup

---
 git_fleximod/git_fleximod.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 1f60b1822..8f2b1199e 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -184,14 +184,14 @@ def single_submodule_checkout(
 
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
-        print(f"Recursively checking out submodules of {name} {repodir} {url}")
+        print(f"Recursively checking out submodules of {name}")
         gitmodules = GitModules(logger, confpath=repodir)
         requiredlist = ["AlwaysRequired"]
         if optional:
             requiredlist.append("AlwaysOptional")
         submodules_checkout(gitmodules, repodir, requiredlist, force=force)
     if os.path.exists(os.path.join(repodir, ".git")):
-        print(f"Successfully checked out {name} {repodir}")
+        print(f"Successfully checked out {name:>20}")
     else:
         utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}")
 
@@ -207,6 +207,8 @@ def submodules_status(gitmodules, root_dir):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
+        required = gitmodules.get(name, "fxrequired")
+        level = required and "Toplevel" in required 
         if not path:
             utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
@@ -217,6 +219,8 @@ def submodules_status(gitmodules, root_dir):
             url = gitmodules.get(name, "url")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
+            if level:
+                continue
             for htag in tags.split("\n"):
                 if tag and tag in htag:
                     atag = (htag.split()[1])[10:]

From 0c26ad0ce55a4d080ef211662e9f0d544c4ceada Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 14 Mar 2024 12:43:23 -0600
Subject: [PATCH 110/131] update cleanup

---
 git_fleximod/git_fleximod.py | 12 +++---
 git_fleximod/metoflexi.py    | 84 +++++++++++++++++-------------------
 2 files changed, 46 insertions(+), 50 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 8f2b1199e..69301e72d 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -201,7 +201,7 @@ def single_submodule_checkout(
     return
 
 
-def submodules_status(gitmodules, root_dir):
+def submodules_status(gitmodules, root_dir, toplevel=False):
     testfails = 0
     localmods = 0
     for name in gitmodules.sections():
@@ -219,7 +219,7 @@ def submodules_status(gitmodules, root_dir):
             url = gitmodules.get(name, "url")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
-            if level:
+            if not toplevel and level:
                 continue
             for htag in tags.split("\n"):
                 if tag and tag in htag:
@@ -268,18 +268,20 @@ def submodules_status(gitmodules, root_dir):
 
 def submodules_update(gitmodules, root_dir, requiredlist, force):
     _, localmods = submodules_status(gitmodules, root_dir)
-    print("")
+
     if localmods and not force:
         print(
             "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
         )
         return
+    elif not localmods:
+        return
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist))
-#        if not os.path.exists(os.path.join(root_dir,path, ".git")):
+        #        if not os.path.exists(os.path.join(root_dir,path, ".git")):
         fxrequired = gitmodules.get(name, "fxrequired")
         assert(fxrequired in fxrequired_allowed_values())
         rgit = GitInterface(root_dir, logger)
@@ -444,7 +446,7 @@ def main():
     if action == "update":
         submodules_update(gitmodules, root_dir, fxrequired, force)
     elif action == "status":
-        submodules_status(gitmodules, root_dir)
+        submodules_status(gitmodules, root_dir, toplevel=True)
     elif action == "test":
         retval = submodules_test(gitmodules, root_dir)
     else:
diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py
index b607ad925..1a4f84213 100755
--- a/git_fleximod/metoflexi.py
+++ b/git_fleximod/metoflexi.py
@@ -150,53 +150,47 @@ def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, p
             if sparse:
                 self.gitmodules.set(section, "fxsparse", sparse)
             self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
-
-            return
-        
-        newpath = (self.rootpath / Path(path))
-        if newpath.exists():
-            shutil.rmtree(newpath)
-        logger.info("Creating directory {}".format(newpath))
-        newpath.mkdir(parents=True)
-        if tag:
-            logger.info("cloning {}".format(section))
-            try:
-                self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path)
-            except:
+        else:
+            newpath = (self.rootpath / Path(path))
+            if newpath.exists():
+                shutil.rmtree(newpath)
+                logger.info("Creating directory {}".format(newpath))
+            newpath.mkdir(parents=True)
+            if tag:
+                logger.info("cloning {}".format(section))
+                try:
+                    self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path)
+                except:
+                    self.git.git_operation("clone", url, path)
+                    with utils.pushd(newpath):
+                        ngit = GitInterface(newpath, logger)
+                        ngit.git_operation("checkout", tag)
+            if hash_:
                 self.git.git_operation("clone", url, path)
-                with utils.pushd(newpath):
-                    ngit = GitInterface(newpath, logger)
-                    ngit.git_operation("checkout", tag)
-
-#        if (newpath / ".gitignore").exists():
-#            logger.info("Moving .gitignore file in {}".format(newpath))
-#            (newpath / ".gitignore").rename((newpath / "save.gitignore"))
-            
-        if hash_:
-            self.git.git_operation("clone", url, path)
-            git = GitInterface(newpath, logger)
-            git.git_operation("fetch", "origin")
-            git.git_operation("checkout", hash_)
-        if sparse:
-            print("setting as sparse submodule {}".format(section))
-            sparsefile = (newpath / Path(sparse))
-            newfile = (newpath / ".git" / "info" / "sparse-checkout")
-            print(f"sparsefile {sparsefile} newfile {newfile}")
-            shutil.copy(sparsefile, newfile)
-        logger.info("adding submodule {}".format(section))
-        self.gitmodules.save()
-        self.git.git_operation("submodule", "add", "-f", "--name", section, url, path)
-        self.git.git_operation("submodule","absorbgitdirs")
-        self.gitmodules.reload()
-        if tag:
-            self.gitmodules.set(section, "fxtag", tag)
-        if hash_:
-           self.gitmodules.set(section, "fxtag", hash_)
+                git = GitInterface(newpath, logger)
+                git.git_operation("fetch", "origin")
+                git.git_operation("checkout", hash_)
+            if sparse:
+                print("setting as sparse submodule {}".format(section))
+                sparsefile = (newpath / Path(sparse))
+                newfile = (newpath / ".git" / "info" / "sparse-checkout")
+                print(f"sparsefile {sparsefile} newfile {newfile}")
+                shutil.copy(sparsefile, newfile)
+        
+            logger.info("adding submodule {}".format(section))        
+            self.gitmodules.save()
+            self.git.git_operation("submodule", "add", "-f", "--name", section, url, path)
+            self.git.git_operation("submodule","absorbgitdirs")
+            self.gitmodules.reload()
+            if tag:
+                self.gitmodules.set(section, "fxtag", tag)
+            if hash_:
+                self.gitmodules.set(section, "fxtag", hash_)
             
-        self.gitmodules.set(section, "fxurl", url)
-        if sparse:
-            self.gitmodules.set(section, "fxsparse", sparse)
-        self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
+            self.gitmodules.set(section, "fxurl", url)
+            if sparse:
+                self.gitmodules.set(section, "fxsparse", sparse)
+            self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
     
         
     def translate_repo(self):

From 17dc6d3a0abbc7a6f69c79197f6892501b7e2e8d Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 14 Mar 2024 12:45:58 -0600
Subject: [PATCH 111/131] update cleanup

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index c17c5b6f7..f8afb40b3 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,8 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
       - AlwaysOptional:       Always optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
     fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
+           **NOTE** the fxurl variable is only used to identify the official project repository and should not be
+           changed by users.  Use the url variable to change to a fork if desired.
 
 ## Sparse Checkouts
 

From a56b7f7cec1372dec15962865123f9d14f8fd4bf Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 14 Mar 2024 14:33:45 -0600
Subject: [PATCH 112/131] all tests passing

---
 git_fleximod/git_fleximod.py | 52 +++++++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 69301e72d..db788c148 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -67,6 +67,24 @@ def commandline_arguments(args=None):
 def submodule_sparse_checkout(
     root_dir, name, url, path, sparsefile, tag="master"
 ):
+    """
+    This function performs a sparse checkout of a git submodule.  It does so by first creating the .git/info/sparse-checkout fileq
+    in the submodule and then checking out the desired tag.  If the submodule is already checked out, it will not be checked out again.
+    Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed.  This is important
+    because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time 
+    and disk space consuming.
+    
+    Parameters:
+    root_dir (str): The root directory for the git operation.
+    name (str): The name of the submodule.
+    url (str): The URL of the submodule.
+    path (str): The path to the submodule.
+    sparsefile (str): The sparse file for the submodule.
+    tag (str, optional): The tag to checkout. Defaults to "master".
+
+    Returns:
+    None
+    """
     logger.info("Called sparse_checkout for {}".format(name))
     rgit = GitInterface(root_dir, logger)
     superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
@@ -140,8 +158,24 @@ def submodule_sparse_checkout(
     rgit.config_set_value(f'submodule "{name}"',"url",url)
 
 def single_submodule_checkout(
-        root, name, path, url=None, tag=None, force=False, optional=False
+    root, name, path, url=None, tag=None, force=False, optional=False
 ):
+    """
+    This function checks out a single git submodule.
+
+    Parameters:
+    root (str): The root directory for the git operation.
+    name (str): The name of the submodule.
+    path (str): The path to the submodule.
+    url (str, optional): The URL of the submodule. Defaults to None.
+    tag (str, optional): The tag to checkout. Defaults to None.
+    force (bool, optional): If set to True, forces the checkout operation. Defaults to False.
+    optional (bool, optional): If set to True, the submodule is considered optional. Defaults to False.
+
+    Returns:
+    None
+    """
+    # function implementation...
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
     logger.info("Checkout {} into {}/{}".format(name,root,path))
@@ -204,6 +238,7 @@ def single_submodule_checkout(
 def submodules_status(gitmodules, root_dir, toplevel=False):
     testfails = 0
     localmods = 0
+    needsupdate = 0
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
@@ -219,6 +254,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
             url = gitmodules.get(name, "url")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
+            needsupdate += 1
             if not toplevel and level:
                 continue
             for htag in tags.split("\n"):
@@ -252,6 +288,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                 elif tag:
                     print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}")
                     testfails += 1
+                    needsupdate += 1
                 else:
                     print(
                         f"e {name:>20} has no fxtag defined in .gitmodules, module at {atag}"
@@ -263,19 +300,20 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                     localmods = localmods + 1
                     print("M" + textwrap.indent(status, "                      "))
 
-    return testfails, localmods
+    return testfails, localmods, needsupdate
 
 
 def submodules_update(gitmodules, root_dir, requiredlist, force):
-    _, localmods = submodules_status(gitmodules, root_dir)
+    _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
 
     if localmods and not force:
         print(
             "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
         )
         return
-    elif not localmods:
+    if needsupdate == 0:
         return
+    
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
@@ -349,13 +387,15 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
 
 # checkout is done by update if required so this function may be depricated
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
-    _, localmods = submodules_status(gitmodules, root_dir)
+    _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
     print("")
     if localmods and not force:
         print(
             "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
         )
         return
+    if not needsupdate:
+        return
     for name in gitmodules.sections():
         fxrequired = gitmodules.get(name, "fxrequired")
         fxsparse = gitmodules.get(name, "fxsparse")
@@ -389,7 +429,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
 
 def submodules_test(gitmodules, root_dir):
     # First check that fxtags are present and in sync with submodule hashes
-    testfails, localmods = submodules_status(gitmodules, root_dir)
+    testfails, localmods, _ = submodules_status(gitmodules, root_dir)
     print("")
     # Then make sure that urls are consistant with fxurls (not forks and not ssh)
     # and that sparse checkout files exist

From c31e17f4fbb17aaffd8afdd1475b19dcfb074409 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 14 Mar 2024 14:48:01 -0600
Subject: [PATCH 113/131] more cleanup and comments

---
 git_fleximod/git_fleximod.py | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index db788c148..89b0a6e03 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -387,8 +387,21 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
 
 # checkout is done by update if required so this function may be depricated
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
-    _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
+    """
+    This function checks out all git submodules based on the provided parameters.
+
+    Parameters:
+    gitmodules (ConfigParser): The gitmodules configuration.
+    root_dir (str): The root directory for the git operation.
+    requiredlist (list): The list of required modules.
+    force (bool, optional): If set to True, forces the checkout operation. Defaults to False.
+
+    Returns:
+    None
+    """
+    # function implementation...
     print("")
+    _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
     if localmods and not force:
         print(
             "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
@@ -426,10 +439,23 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
                 optional = "AlwaysOptional" in requiredlist
             )
 
-
 def submodules_test(gitmodules, root_dir):
+    """
+    This function tests the git submodules based on the provided parameters.
+
+    It first checks that fxtags are present and in sync with submodule hashes.
+    Then it ensures that urls are consistent with fxurls (not forks and not ssh)
+    and that sparse checkout files exist.
+
+    Parameters:
+    gitmodules (ConfigParser): The gitmodules configuration.
+    root_dir (str): The root directory for the git operation.
+
+    Returns:
+    int: The number of test failures.
+    """
     # First check that fxtags are present and in sync with submodule hashes
-    testfails, localmods, _ = submodules_status(gitmodules, root_dir)
+    testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir)
     print("")
     # Then make sure that urls are consistant with fxurls (not forks and not ssh)
     # and that sparse checkout files exist
@@ -444,7 +470,7 @@ def submodules_test(gitmodules, root_dir):
         if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
             print(f"{name:>20} sparse checkout file {fxsparse} not found")
             testfails += 1
-    return testfails + localmods
+    return testfails + localmods + needsupdate
 
 
 def main():

From 82428601f52b15b27f67031ad093d91554d09d3e Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 19 Mar 2024 13:00:01 -0600
Subject: [PATCH 114/131] fix py3.11 issue in gitinterface

---
 git_fleximod/gitinterface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git_fleximod/gitinterface.py b/git_fleximod/gitinterface.py
index 203c50030..93ae38ecd 100644
--- a/git_fleximod/gitinterface.py
+++ b/git_fleximod/gitinterface.py
@@ -25,7 +25,7 @@ def __init__(self, repo_path, logger):
             msg = "Using GitPython interface to git"
         except ImportError:
             self._use_module = False
-            if not (repo_path / ".git").exists():
+            if not (self.repo_path / ".git").exists():
                 self._init_git_repo()
             msg = "Using shell interface to git"
         self.logger.info(msg)

From 3fba38563fd5a9ceec5d0abcd210ae90676b6309 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 28 Mar 2024 13:09:57 -0600
Subject: [PATCH 115/131] explain M flag

---
 git_fleximod/git_fleximod.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 89b0a6e03..ebb719d4b 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -307,9 +307,7 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
     _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
 
     if localmods and not force:
-        print(
-            "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
-        )
+        local_mods_output()
         return
     if needsupdate == 0:
         return
@@ -385,6 +383,22 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 else:
                     print(f"{name:>20} up to date.")
 
+def local_mods_output():
+    text = '''\
+    The submodules labeled with 'M' above are not in a clean state.
+    The following are options for how to proceed:
+    (1) Go into each submodule which is not in a clean state and issue a 'git status'
+        Either revert or commit your changes so that the submodule is in a clean state.
+    (2) use the --force option to git-fleximod
+    (3) you can name the particular submodules to update using the git-fleximod command line
+    (4) As a last resort you can remove the submodule (via 'rm -fr [directory]')
+        then rerun git-fleximod update.
+'''
+    print(text)
+    
+    
+
+                    
 # checkout is done by update if required so this function may be depricated
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
     """
@@ -403,9 +417,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
     print("")
     _, localmods, needsupdate = submodules_status(gitmodules, root_dir)
     if localmods and not force:
-        print(
-            "Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
-        )
+        local_mods_output()
         return
     if not needsupdate:
         return
@@ -512,7 +524,11 @@ def main():
     if action == "update":
         submodules_update(gitmodules, root_dir, fxrequired, force)
     elif action == "status":
-        submodules_status(gitmodules, root_dir, toplevel=True)
+        tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
+        if tfails + lmods + updates > 0:
+            print(f"    testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n")
+            if lmods > 0:
+                local_mods_output()                
     elif action == "test":
         retval = submodules_test(gitmodules, root_dir)
     else:

From b4e07beb2fef9e4c62514a9b7fe4b86e99e043b2 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 28 Mar 2024 13:37:26 -0600
Subject: [PATCH 116/131] fixes issue with ssh access to github (should not be
 required)

---
 git_fleximod/git_fleximod.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index ebb719d4b..a0d3a7613 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -185,7 +185,6 @@ def single_submodule_checkout(
     if os.path.exists(os.path.join(repodir, ".git")):
         logger.info("Submodule {} already checked out".format(name))
         repo_exists = True
-
     # Look for a .gitmodules file in the newly checkedout repo
     if not repo_exists and url:
         # ssh urls cause problems for those who dont have git accounts with ssh keys defined
@@ -209,7 +208,11 @@ def single_submodule_checkout(
                         rootdotgit = line[8:].rstrip()
 
             newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name))
-            shutil.move(os.path.join(repodir, ".git"), newpath)
+            if os.path.exists(newpath):
+                shutil.rmtree(os.path.join(repodir,".git"))
+            else:
+                shutil.move(os.path.join(repodir, ".git"), newpath)
+
             with open(os.path.join(repodir, ".git"), "w") as f:
                 f.write("gitdir: " + os.path.relpath(newpath, start=repodir))
 
@@ -252,6 +255,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
             rootgit = GitInterface(root_dir, logger)
             # submodule commands use path, not name
             url = gitmodules.get(name, "url")
+            url = url.replace("git@github.com:", "https://github.com/")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
             needsupdate += 1
@@ -445,7 +449,6 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
-
             single_submodule_checkout(
                 root_dir, name, path, url=url, tag=fxtag, force=force,
                 optional = "AlwaysOptional" in requiredlist

From 3a9201dce1a482de86d7a00e6958cd8ce310b4ef Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 28 Mar 2024 13:59:06 -0600
Subject: [PATCH 117/131] better error handling for not a tag

---
 git_fleximod/git_fleximod.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index a0d3a7613..8e479fc30 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -227,9 +227,7 @@ def single_submodule_checkout(
         if optional:
             requiredlist.append("AlwaysOptional")
         submodules_checkout(gitmodules, repodir, requiredlist, force=force)
-    if os.path.exists(os.path.join(repodir, ".git")):
-        print(f"Successfully checked out {name:>20}")
-    else:
+    if not os.path.exists(os.path.join(repodir, ".git")):
         utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}")
 
     if tmpurl:
@@ -380,8 +378,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                     git.git_operation("fetch", newremote, "--tags")
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
                 if fxtag and fxtag != atag:
-                    print(f"{name:>20} updated to {fxtag}")
-                    git.git_operation("checkout", fxtag)
+                    try:
+                        git.git_operation("checkout", fxtag)
+                        print(f"{name:>20} updated to {fxtag}")
+                    except Exception as error:
+                        print(error)
                 elif not fxtag:
                     print(f"No fxtag found for submodule {name:>20}")
                 else:

From ad6b84ee19be4983561fbfff5d9d9abe24fda74e Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 28 Mar 2024 14:15:45 -0600
Subject: [PATCH 118/131] Bump to 0.7.3

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index f9cd67cff..169a1b338 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.7.0"
+__version__ = "0.7.3"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index 212b014d9..bb4d89a30 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.0"
+version = "0.7.3"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index 568226dfa..7338278fb 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.7.0"
+current = "0.7.3"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From fd0ce7ad12f73b7932b98c6531b127767a7a2a19 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 28 Mar 2024 14:40:25 -0600
Subject: [PATCH 119/131] set python 3.7 as min version requirement

---
 git_fleximod/git_fleximod.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 8e479fc30..3c0ae0764 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python
 import sys
+MIN_PYTHON = (3, 7)
+if sys.version_info < MIN_PYTHON:
+    sys.exit("Python %s.%s or later is required."%MIN_PYTHON)
+
 import os
 import shutil
 import logging

From ca419c61c810d57f0f7b4ab0de165ad30e28360d Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Fri, 29 Mar 2024 14:53:16 -0600
Subject: [PATCH 120/131] rename fxurl

---
 git_fleximod/git_fleximod.py | 165 ++++++++++++++++++++++-------------
 1 file changed, 105 insertions(+), 60 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 3c0ae0764..e4e18ee25 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 import sys
+
 MIN_PYTHON = (3, 7)
 if sys.version_info < MIN_PYTHON:
-    sys.exit("Python %s.%s or later is required."%MIN_PYTHON)
+    sys.exit("Python %s.%s or later is required." % MIN_PYTHON)
 
 import os
 import shutil
@@ -16,8 +17,11 @@
 
 # logger variable is global
 logger = None
+
+
 def fxrequired_allowed_values():
-    return ['ToplevelRequired', 'ToplevelOptional', 'AlwaysRequired', 'AlwaysOptional']
+    return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"]
+
 
 def commandline_arguments(args=None):
     parser = cli.get_parser()
@@ -29,7 +33,12 @@ def commandline_arguments(args=None):
 
     # explicitly listing a component overrides the optional flag
     if options.optional or options.components:
-        fxrequired = ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"]
+        fxrequired = [
+            "ToplevelRequired",
+            "ToplevelOptional",
+            "AlwaysRequired",
+            "AlwaysOptional",
+        ]
     else:
         fxrequired = ["ToplevelRequired", "AlwaysRequired"]
 
@@ -68,16 +77,14 @@ def commandline_arguments(args=None):
     )
 
 
-def submodule_sparse_checkout(
-    root_dir, name, url, path, sparsefile, tag="master"
-):
+def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"):
     """
     This function performs a sparse checkout of a git submodule.  It does so by first creating the .git/info/sparse-checkout fileq
     in the submodule and then checking out the desired tag.  If the submodule is already checked out, it will not be checked out again.
     Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed.  This is important
-    because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time 
+    because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time
     and disk space consuming.
-    
+
     Parameters:
     root_dir (str): The root directory for the git operation.
     name (str): The name of the submodule.
@@ -96,10 +103,10 @@ def submodule_sparse_checkout(
         gitroot = superroot
     else:
         gitroot = root_dir
-    assert(os.path.isdir(os.path.join(gitroot,".git")))
+    assert os.path.isdir(os.path.join(gitroot, ".git"))
     # first create the module directory
-    if not os.path.isdir(os.path.join(root_dir,path)):
-        os.makedirs(os.path.join(root_dir,path))
+    if not os.path.isdir(os.path.join(root_dir, path)):
+        os.makedirs(os.path.join(root_dir, path))
 
     # initialize a new git repo and set the sparse checkout flag
     sprep_repo = os.path.join(root_dir, path)
@@ -120,25 +127,31 @@ def submodule_sparse_checkout(
 
     # set the repository remote
 
-    logger.info("Setting remote origin in {}/{}".format(root_dir,path))
+    logger.info("Setting remote origin in {}/{}".format(root_dir, path))
     status = sprepo_git.git_operation("remote", "-v")
     if url not in status:
         sprepo_git.git_operation("remote", "add", "origin", url)
-    
-    topgit = os.path.join(gitroot,".git")
-    
+
+    topgit = os.path.join(gitroot, ".git")
+
     if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")):
         with open(os.path.join(root_dir, ".git")) as f:
-            gitpath = os.path.relpath(os.path.join(root_dir, f.read().split()[1]), start=os.path.join(root_dir,path))
+            gitpath = os.path.relpath(
+                os.path.join(root_dir, f.read().split()[1]),
+                start=os.path.join(root_dir, path),
+            )
         topgit = os.path.join(gitpath, "modules")
     else:
-        topgit = os.path.relpath(os.path.join(root_dir, ".git", "modules"), start=os.path.join(root_dir,path))
-    
+        topgit = os.path.relpath(
+            os.path.join(root_dir, ".git", "modules"),
+            start=os.path.join(root_dir, path),
+        )
+
     with utils.pushd(sprep_repo):
         if not os.path.isdir(topgit):
             os.makedirs(topgit)
     topgit += os.sep + name
-    
+
     if os.path.isdir(os.path.join(root_dir, path, ".git")):
         with utils.pushd(sprep_repo):
             shutil.move(".git", topgit)
@@ -147,7 +160,9 @@ def submodule_sparse_checkout(
             #    assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
             gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
         if os.path.isfile(gitsparse):
-            logger.warning("submodule {} is already initialized {}".format(name, topgit))
+            logger.warning(
+                "submodule {} is already initialized {}".format(name, topgit)
+            )
             return
 
         with utils.pushd(sprep_repo):
@@ -156,10 +171,11 @@ def submodule_sparse_checkout(
     # Finally checkout the repo
     sprepo_git.git_operation("fetch", "origin", "--tags")
     sprepo_git.git_operation("checkout", tag)
-        
+
     print(f"Successfully checked out {name:>20} at {tag}")
-    rgit.config_set_value(f'submodule "{name}"',"active","true")
-    rgit.config_set_value(f'submodule "{name}"',"url",url)
+    rgit.config_set_value(f'submodule "{name}"', "active", "true")
+    rgit.config_set_value(f'submodule "{name}"', "url", url)
+
 
 def single_submodule_checkout(
     root, name, path, url=None, tag=None, force=False, optional=False
@@ -182,7 +198,7 @@ def single_submodule_checkout(
     # function implementation...
     git = GitInterface(root, logger)
     repodir = os.path.join(root, path)
-    logger.info("Checkout {} into {}/{}".format(name,root,path))
+    logger.info("Checkout {} into {}/{}".format(name, root, path))
     # if url is provided update to the new url
     tmpurl = None
     repo_exists = False
@@ -213,7 +229,7 @@ def single_submodule_checkout(
 
             newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name))
             if os.path.exists(newpath):
-                shutil.rmtree(os.path.join(repodir,".git"))
+                shutil.rmtree(os.path.join(repodir, ".git"))
             else:
                 shutil.move(os.path.join(repodir, ".git"), newpath)
 
@@ -232,7 +248,9 @@ def single_submodule_checkout(
             requiredlist.append("AlwaysOptional")
         submodules_checkout(gitmodules, repodir, requiredlist, force=force)
     if not os.path.exists(os.path.join(repodir, ".git")):
-        utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}")
+        utils.fatal_error(
+            f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}"
+        )
 
     if tmpurl:
         print(git.git_operation("restore", ".gitmodules"))
@@ -248,7 +266,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
         required = gitmodules.get(name, "fxrequired")
-        level = required and "Toplevel" in required 
+        level = required and "Toplevel" in required
         if not path:
             utils.fatal_error("No path found in .gitmodules for {}".format(name))
         newpath = os.path.join(root_dir, path)
@@ -270,8 +288,10 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
             if tag and tag == atag:
                 print(f"e {name:>20} not checked out, aligned at tag {tag}")
             elif tag:
-                ahash = rootgit.git_operation("submodule", "status", "{}".format(path)).rstrip()
-                ahash = ahash[1:len(tag)+1]
+                ahash = rootgit.git_operation(
+                    "submodule", "status", "{}".format(path)
+                ).rstrip()
+                ahash = ahash[1 : len(tag) + 1]
                 if tag == ahash:
                     print(f"e {name:>20} not checked out, aligned at hash {ahash}")
                 else:
@@ -289,10 +309,12 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                 ahash = git.git_operation("status").partition("\n")[0].split()[-1]
                 if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
-                elif tag and ahash[:len(tag)] == tag:
+                elif tag and ahash[: len(tag)] == tag:
                     print(f"  {name:>20} at hash {ahash}")
                 elif tag:
-                    print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}")
+                    print(
+                        f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}"
+                    )
                     testfails += 1
                     needsupdate += 1
                 else:
@@ -317,21 +339,29 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
         return
     if needsupdate == 0:
         return
-    
+
     for name in gitmodules.sections():
         fxtag = gitmodules.get(name, "fxtag")
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
-        logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist))
+        logger.info(
+            "name={} path={} url={} fxtag={} requiredlist={}".format(
+                name, os.path.join(root_dir, path), url, fxtag, requiredlist
+            )
+        )
         #        if not os.path.exists(os.path.join(root_dir,path, ".git")):
         fxrequired = gitmodules.get(name, "fxrequired")
-        assert(fxrequired in fxrequired_allowed_values())
+        assert fxrequired in fxrequired_allowed_values()
         rgit = GitInterface(root_dir, logger)
         superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
-            
+
         fxsparse = gitmodules.get(name, "fxsparse")
-            
-        if fxrequired and (superroot and "Toplevel" in fxrequired) or fxrequired not in requiredlist:
+
+        if (
+            fxrequired
+            and (superroot and "Toplevel" in fxrequired)
+            or fxrequired not in requiredlist
+        ):
             if "ToplevelOptional" == fxrequired:
                 print("Skipping optional component {}".format(name))
             continue
@@ -341,20 +371,24 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                     root_dir, name, url, path, fxsparse, fxtag
                 )
             )
-            submodule_sparse_checkout(
-                root_dir, name, url, path, fxsparse, tag=fxtag
-            )
+            submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag)
         else:
             logger.info(
-                "Calling submodule_checkout({},{},{},{})".format(root_dir, name, path,url)
+                "Calling submodule_checkout({},{},{},{})".format(
+                    root_dir, name, path, url
+                )
             )
-                
+
             single_submodule_checkout(
-                root_dir, name, path, url=url, tag=fxtag, force=force,
-                optional=("AlwaysOptional" in requiredlist)
+                root_dir,
+                name,
+                path,
+                url=url,
+                tag=fxtag,
+                force=force,
+                optional=("AlwaysOptional" in requiredlist),
             )
 
-
         if os.path.exists(os.path.join(path, ".git")):
             submoddir = os.path.join(root_dir, path)
             with utils.pushd(submoddir):
@@ -392,8 +426,9 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 else:
                     print(f"{name:>20} up to date.")
 
+
 def local_mods_output():
-    text = '''\
+    text = """\
     The submodules labeled with 'M' above are not in a clean state.
     The following are options for how to proceed:
     (1) Go into each submodule which is not in a clean state and issue a 'git status'
@@ -402,12 +437,10 @@ def local_mods_output():
     (3) you can name the particular submodules to update using the git-fleximod command line
     (4) As a last resort you can remove the submodule (via 'rm -fr [directory]')
         then rerun git-fleximod update.
-'''
+"""
     print(text)
-    
-    
 
-                    
+
 # checkout is done by update if required so this function may be depricated
 def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
     """
@@ -447,18 +480,22 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
                     root_dir, name, url, path, fxsparse, fxtag
                 )
             )
-            submodule_sparse_checkout(
-                root_dir, name, url, path, fxsparse, tag=fxtag
-            )
+            submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag)
         else:
             logger.debug(
                 "Calling submodule_checkout({},{},{})".format(root_dir, name, path)
             )
             single_submodule_checkout(
-                root_dir, name, path, url=url, tag=fxtag, force=force,
-                optional = "AlwaysOptional" in requiredlist
+                root_dir,
+                name,
+                path,
+                url=url,
+                tag=fxtag,
+                force=force,
+                optional="AlwaysOptional" in requiredlist,
             )
 
+
 def submodules_test(gitmodules, root_dir):
     """
     This function tests the git submodules based on the provided parameters.
@@ -481,10 +518,12 @@ def submodules_test(gitmodules, root_dir):
     # and that sparse checkout files exist
     for name in gitmodules.sections():
         url = gitmodules.get(name, "url")
-        fxurl = gitmodules.get(name, "fxurl")
+        fxurl = gitmodules.get(name, "fxDONOTMODIFYurl")
         fxsparse = gitmodules.get(name, "fxsparse")
         path = gitmodules.get(name, "path")
-        if not fxurl or url != fxurl:
+        fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl
+        url = url[:-4] if url.endswith(".git") else url
+        if not fxurl or url.lower() != fxurl.lower():
             print(f"{name:>20} url {url} not in sync with required {fxurl}")
             testfails += 1
         if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
@@ -518,7 +557,11 @@ def main():
             )
 
         root_dir = os.path.dirname(file_path)
-    logger.info("root_dir is {} includelist={} excludelist={}".format(root_dir, includelist, excludelist))
+    logger.info(
+        "root_dir is {} includelist={} excludelist={}".format(
+            root_dir, includelist, excludelist
+        )
+    )
     gitmodules = GitModules(
         logger,
         confpath=root_dir,
@@ -534,9 +577,11 @@ def main():
     elif action == "status":
         tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
         if tfails + lmods + updates > 0:
-            print(f"    testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n")
+            print(
+                f"    testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n"
+            )
             if lmods > 0:
-                local_mods_output()                
+                local_mods_output()
     elif action == "test":
         retval = submodules_test(gitmodules, root_dir)
     else:

From 152bad98fecc769308c782db5c80d95e51b33f61 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Wed, 3 Apr 2024 09:05:06 -0600
Subject: [PATCH 121/131] fix tests on izumi

---
 git_fleximod/git_fleximod.py | 4 ++--
 git_fleximod/utils.py        | 2 +-
 tests/conftest.py            | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index e4e18ee25..7957001b8 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -100,9 +100,9 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     rgit = GitInterface(root_dir, logger)
     superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     if superroot:
-        gitroot = superroot
+        gitroot = superroot.strip()
     else:
-        gitroot = root_dir
+        gitroot = root_dir.strip()
     assert os.path.isdir(os.path.join(gitroot, ".git"))
     # first create the module directory
     if not os.path.isdir(os.path.join(root_dir, path)):
diff --git a/git_fleximod/utils.py b/git_fleximod/utils.py
index f0753367e..7cc1de38c 100644
--- a/git_fleximod/utils.py
+++ b/git_fleximod/utils.py
@@ -272,7 +272,7 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False)
     cwd = os.getcwd()
     msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd)
     logging.info(msg)
-    commands_str = " ".join(commands)
+    commands_str = " ".join(str(element) for element in commands)
     logging.info(commands_str)
     return_to_caller = status_to_caller or output_to_caller
     status = -1
diff --git a/tests/conftest.py b/tests/conftest.py
index 5fed907c4..942a0efb9 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -25,7 +25,7 @@ def logger():
     path = modules/test
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.4.0
-    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = ToplevelRequired
 """},
     {"subrepo_path": "modules/test_optional",
@@ -39,7 +39,7 @@ def logger():
     path = modules/test_optional
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.4.0
-    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = ToplevelOptional
 """},
     {"subrepo_path": "modules/test_alwaysoptional",
@@ -53,7 +53,7 @@ def logger():
     path = modules/test_alwaysoptional
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = e5cf35c
-    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = AlwaysOptional
 """},    
     {"subrepo_path": "modules/test_sparse",
@@ -67,7 +67,7 @@ def logger():
     path = modules/test_sparse
     url = https://github.com/ESMCI/mpi-serial.git
     fxtag = MPIserial_2.5.0
-    fxurl = https://github.com/ESMCI/mpi-serial.git
+    fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
     fxrequired = AlwaysRequired
     fxsparse = ../.sparse_file_list
 """},

From 5a1e5f64b813208fb6c62dfdab6e466a5b4ef8d6 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Wed, 3 Apr 2024 09:50:57 -0600
Subject: [PATCH 122/131] fix issue in metoflexi

---
 git_fleximod/metoflexi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py
index 1a4f84213..cc347db2d 100755
--- a/git_fleximod/metoflexi.py
+++ b/git_fleximod/metoflexi.py
@@ -17,7 +17,7 @@ def find_root_dir(filename=".git"):
     while d != root:
         attempt = d / filename
         if attempt.is_dir():
-            return attempt
+            return d
         d = d.parent
     return None
 
@@ -146,7 +146,7 @@ def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, p
             if hash_:
                 self.gitmodules.set(section, "fxtag", hash_)
             
-            self.gitmodules.set(section, "fxurl", url)
+            self.gitmodules.set(section, "fxDONOTUSEurl", url)
             if sparse:
                 self.gitmodules.set(section, "fxsparse", sparse)
             self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
@@ -187,7 +187,7 @@ def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, p
             if hash_:
                 self.gitmodules.set(section, "fxtag", hash_)
             
-            self.gitmodules.set(section, "fxurl", url)
+            self.gitmodules.set(section, "fxDONOTUSEurl", url)
             if sparse:
                 self.gitmodules.set(section, "fxsparse", sparse)
             self.gitmodules.set(section, "fxrequired", "ToplevelRequired")

From 5398ef43fc6cdacdc894b7e6567bc9fb28b26da6 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 23 Apr 2024 15:18:24 -0600
Subject: [PATCH 123/131] if submodule does not exist add it

---
 git_fleximod/git_fleximod.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 7957001b8..adf575f9c 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -235,10 +235,13 @@ def single_submodule_checkout(
 
             with open(os.path.join(repodir, ".git"), "w") as f:
                 f.write("gitdir: " + os.path.relpath(newpath, start=repodir))
+                
+    if not os.path.exists(repodir):
+        git.git_operation("submodule", "add", "--name", name, "--", url, path) 
 
     if not repo_exists or not tmpurl:
-        logger.debug(git.git_operation("submodule", "update", "--init", "--", path))
-
+        git.git_operation("submodule", "update", "--init", "--", path)
+            
     if os.path.exists(os.path.join(repodir, ".gitmodules")):
         # recursively handle this checkout
         print(f"Recursively checking out submodules of {name}")
@@ -311,6 +314,8 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                     print(f"  {name:>20} at tag {tag}")
                 elif tag and ahash[: len(tag)] == tag:
                     print(f"  {name:>20} at hash {ahash}")
+                elif atag == ahash:
+                    print(f"  {name:>20} at hash {ahash}")
                 elif tag:
                     print(
                         f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}"

From 46a74ac7f75b5d086fb1f3b3ffb117ae76cc46d4 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 23 Apr 2024 16:44:14 -0600
Subject: [PATCH 124/131] create parent if it doesnt exist

---
 git_fleximod/git_fleximod.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index adf575f9c..103cc82a5 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -237,6 +237,9 @@ def single_submodule_checkout(
                 f.write("gitdir: " + os.path.relpath(newpath, start=repodir))
                 
     if not os.path.exists(repodir):
+        parent = os.path.dirname(repodir)
+        if not os.path.isdir(parent):
+            os.makedirs(parent)
         git.git_operation("submodule", "add", "--name", name, "--", url, path) 
 
     if not repo_exists or not tmpurl:

From eb932b015072ba3ec8cc7fadf5a387058f79fc4c Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 23 Apr 2024 16:49:47 -0600
Subject: [PATCH 125/131] Bump to 0.7.4

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 169a1b338..1fb959dad 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 import argparse
 
-__version__ = "0.7.3"
+__version__ = "0.7.4"
 
 def find_root_dir(filename=".git"):
     d = Path.cwd()
diff --git a/pyproject.toml b/pyproject.toml
index bb4d89a30..2484552e4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.3"
+version = "0.7.4"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index 7338278fb..d4b8eaee1 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.7.3"
+current = "0.7.4"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From 94121295e57a1d6e713b8e3ba84261d9e9cfebe6 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Tue, 14 May 2024 13:48:01 -0600
Subject: [PATCH 126/131] update documentation

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f8afb40b3..d1ef632f2 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
         status: Display the status of submodules.
         update: Update submodules to the tag indicated in .gitmodules variable fxtag.
         test: Make sure that fxtags and submodule hashes are consistant,
-              make sure that official urls (as defined by fxurl) are set
+              make sure that official urls (as defined by fxDONOTUSEurl) are set
               make sure that fxtags are defined for all submodules
     Additional Options:
         See git fleximod --help for more details.
@@ -34,8 +34,8 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
       - AlwaysRequired:       Always required (always checked out).
       - AlwaysOptional:       Always optional (checked out with --optional flag).
     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
-    fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
-           **NOTE** the fxurl variable is only used to identify the official project repository and should not be
+    fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
+           **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be
            changed by users.  Use the url variable to change to a fork if desired.
 
 ## Sparse Checkouts

From 4aa073c3482ad30b6952834ccc5eb99778f88c09 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Fri, 31 May 2024 09:56:18 -0600
Subject: [PATCH 127/131] update find_root_dir to work with git worktree

---
 git_fleximod/cli.py          | 24 +++++++++++++++++-------
 git_fleximod/git_fleximod.py | 10 +++++++---
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 1fb959dad..3eb90870d 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -1,17 +1,27 @@
 from pathlib import Path
 import argparse
+from git_fleximod import utils
 
 __version__ = "0.7.4"
 
-def find_root_dir(filename=".git"):
+def find_root_dir(filename=".gitmodules"):
+    """ finds the highest directory in tree
+    which contains a file called filename """
     d = Path.cwd()
     root = Path(d.root)
-    while d != root:
-        attempt = d / filename
-        if attempt.is_dir():
-            return attempt
-        d = d.parent
-    return None
+    dirlist = []
+    dl = d
+    while dl != root:
+        dirlist.append(dl)
+        dl = dl.parent
+    dirlist.append(root)
+    dirlist.reverse()
+
+    for dl in dirlist:
+        attempt = dl / filename
+        if attempt.is_file():
+            return dl
+    utils.fatal_error("No .gitmodules found in directory tree")
 
 
 def get_parser():
diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index 103cc82a5..f080513a5 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -312,7 +312,11 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
-                ahash = git.git_operation("status").partition("\n")[0].split()[-1]
+                part =  git.git_operation("status").partition("\n")[0]
+                # fake hash to initialize
+                ahash = "xxxx"
+                if part:
+                    ahash = part.split()[-1]
                 if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
                 elif tag and ahash[: len(tag)] == tag:
@@ -554,8 +558,8 @@ def main():
     global logger
     logger = logging.getLogger(__name__)
 
-    logger.info("action is {}".format(action))
-
+    logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name))
+    
     if not os.path.isfile(os.path.join(root_dir, file_name)):
         file_path = utils.find_upwards(root_dir, file_name)
 

From 1fddaac99d83852380738c44ca33f615898123ca Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Fri, 31 May 2024 10:09:31 -0600
Subject: [PATCH 128/131] return str not Path

---
 License             | 2 +-
 README.md           | 2 --
 git_fleximod/cli.py | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/License b/License
index 2c6fe768c..88bc22515 100644
--- a/License
+++ b/License
@@ -1,4 +1,4 @@
-Copyright 2024 National Center for Atmospheric Sciences (NCAR)
+Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR)
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
diff --git a/README.md b/README.md
index d1ef632f2..53917da40 100644
--- a/README.md
+++ b/README.md
@@ -8,8 +8,6 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
 
 ## Installation
 
-#TODO    Install using pip:
-#        pip install git-fleximod
   If you choose to locate git-fleximod in your path you can access it via command: git fleximod
 
 ## Usage
diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 3eb90870d..d24d07c59 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -20,7 +20,7 @@ def find_root_dir(filename=".gitmodules"):
     for dl in dirlist:
         attempt = dl / filename
         if attempt.is_file():
-            return dl
+            return str(dl)
     utils.fatal_error("No .gitmodules found in directory tree")
 
 

From a354b0528228ac89ce103b3b42b2a203fe495ba5 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Fri, 31 May 2024 10:14:11 -0600
Subject: [PATCH 129/131] Bump to 0.7.5

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index d24d07c59..4c3fb1a8f 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -2,7 +2,7 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.7.4"
+__version__ = "0.7.5"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
diff --git a/pyproject.toml b/pyproject.toml
index 2484552e4..ac1684ea0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.4"
+version = "0.7.5"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index d4b8eaee1..e644fc4d2 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.7.4"
+current = "0.7.5"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

From 00b7b38174360d20b9c2bdeeb8d09f1ac0656fca Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Sat, 1 Jun 2024 11:41:49 -0600
Subject: [PATCH 130/131] some fixes when modifying url

---
 git_fleximod/git_fleximod.py | 49 ++++++++++++++++++++----------------
 git_fleximod/gitmodules.py   |  2 +-
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py
index f080513a5..a003c0da8 100755
--- a/git_fleximod/git_fleximod.py
+++ b/git_fleximod/git_fleximod.py
@@ -263,6 +263,21 @@ def single_submodule_checkout(
 
     return
 
+def add_remote(git, url):
+    remotes = git.git_operation("remote", "-v")
+    newremote = "newremote.00"
+    if url in remotes:
+        for line in remotes:
+            if url in line and "fetch" in line:
+                newremote = line.split()[0]
+                break
+    else:
+        i = 0
+        while "newremote" in remotes:
+            i = i + 1
+            newremote = f"newremote.{i:02d}"
+        git.git_operation("remote", "add", newremote, url)
+    return newremote
 
 def submodules_status(gitmodules, root_dir, toplevel=False):
     testfails = 0
@@ -271,6 +286,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
+        url = gitmodules.get(name, "url")
         required = gitmodules.get(name, "fxrequired")
         level = required and "Toplevel" in required
         if not path:
@@ -280,7 +296,6 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
         if not os.path.exists(os.path.join(newpath, ".git")):
             rootgit = GitInterface(root_dir, logger)
             # submodule commands use path, not name
-            url = gitmodules.get(name, "url")
             url = url.replace("git@github.com:", "https://github.com/")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
             atag = None
@@ -312,11 +327,11 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
-                part =  git.git_operation("status").partition("\n")[0]
-                # fake hash to initialize
-                ahash = "xxxx"
-                if part:
-                    ahash = part.split()[-1]
+                ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
+                rurl = git.git_operation("ls-remote","--get-url").rstrip()
+                if rurl != url:
+                    remote = add_remote(git, url)
+                    git.git_operation("fetch", remote)
                 if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
                 elif tag and ahash[: len(tag)] == tag:
@@ -335,7 +350,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                     )
                     testfails += 1
 
-                status = git.git_operation("status", "--ignore-submodules")
+                status = git.git_operation("status", "--ignore-submodules", "-uno")
                 if "nothing to commit" not in status:
                     localmods = localmods + 1
                     print("M" + textwrap.indent(status, "                      "))
@@ -357,11 +372,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info(
-            "name={} path={} url={} fxtag={} requiredlist={}".format(
+            "name={} path={} url={} fxtag={} requiredlist={} ".format(
                 name, os.path.join(root_dir, path), url, fxtag, requiredlist
             )
         )
-        #        if not os.path.exists(os.path.join(root_dir,path, ".git")):
+
         fxrequired = gitmodules.get(name, "fxrequired")
         assert fxrequired in fxrequired_allowed_values()
         rgit = GitInterface(root_dir, logger)
@@ -409,19 +424,7 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 upstream = git.git_operation("ls-remote", "--get-url").rstrip()
                 newremote = "origin"
                 if upstream != url:
-                    # TODO - this needs to be a unique name
-                    remotes = git.git_operation("remote", "-v")
-                    if url in remotes:
-                        for line in remotes:
-                            if url in line and "fetch" in line:
-                                newremote = line.split()[0]
-                                break
-                    else:
-                        i = 0
-                        while newremote in remotes:
-                            i = i + 1
-                            newremote = f"newremote.{i:02d}"
-                        git.git_operation("remote", "add", newremote, url)
+                    add_remote(git, url)
 
                 tags = git.git_operation("tag", "-l")
                 if fxtag and fxtag not in tags:
@@ -439,6 +442,8 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                     print(f"{name:>20} up to date.")
 
 
+
+
 def local_mods_output():
     text = """\
     The submodules labeled with 'M' above are not in a clean state.
diff --git a/git_fleximod/gitmodules.py b/git_fleximod/gitmodules.py
index 68c82d066..7e4e05394 100644
--- a/git_fleximod/gitmodules.py
+++ b/git_fleximod/gitmodules.py
@@ -59,7 +59,7 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
         Uses the parent class's get method to access the value.
         Handles potential errors if the section or option doesn't exist.
         """
-        self.logger.debug("get called {} {}".format(name, option))
+        self.logger.debug("git get called {} {}".format(name, option))
         section = f'submodule "{name}"'
         try:
             return ConfigParser.get(

From a34070be02d10071b544be3839f6c9ed79662665 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Sat, 1 Jun 2024 11:46:24 -0600
Subject: [PATCH 131/131] Bump to 0.7.6

---
 git_fleximod/cli.py | 2 +-
 pyproject.toml      | 2 +-
 tbump.toml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py
index 4c3fb1a8f..7d09abd83 100644
--- a/git_fleximod/cli.py
+++ b/git_fleximod/cli.py
@@ -2,7 +2,7 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.7.5"
+__version__ = "0.7.6"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
diff --git a/pyproject.toml b/pyproject.toml
index ac1684ea0..3c70c6c9c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.5"
+version = "0.7.6"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <jedwards@ucar.edu>"]
 maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
diff --git a/tbump.toml b/tbump.toml
index e644fc4d2..3854682df 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.7.5"
+current = "0.7.6"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before