Update handling of updated configuration files

tests/test_quickstart.py

@@ -1,62 +1,66 @@
 # flake8: noqa
 import os
 import subprocess
-import tempfile
+
+import pytest
 
 from wellies.quickstart import main
 
 
-def quickstart():
-    suite_dir = tempfile.mkdtemp(prefix=f"test_suite_")
-    deploy_dir = tempfile.mkdtemp(prefix=f"deploy_")
+@pytest.fixture 
+def quickstart(tmpdir):
+    suite_dir = os.path.join(tmpdir, "suite")
+    deploy_dir = os.path.join(tmpdir, "deploy")
+
     main(
         [
             f"{suite_dir}",
             "-p",
             "test",
             "--deploy_root",
-            deploy_dir,
+            str(deploy_dir),
             "--output_root",
             "/my/output/root",
         ]
     )
-    print(suite_dir)
-    print(os.listdir(suite_dir))
-    print(os.listdir(os.path.join(suite_dir, "configs")))
+    # print(suite_dir)
+    # print(os.listdir(suite_dir))
+    # print(os.listdir(os.path.join(suite_dir, "configs")))
 
     return suite_dir, deploy_dir
 
 
-def test_quickstart():
-    suite_dir, deploy_dir = quickstart()
+def test_quickstart(quickstart):
+    suite_dir, deploy_dir = quickstart
     assert os.path.exists(os.path.join(suite_dir, "deploy.py"))
 
 
-def test_quickstart_help():
-    suite_dir, deploy_dir = quickstart()
+def test_quickstart_help(quickstart):
+    suite_dir, deploy_dir = quickstart
 
     # test help
     subprocess.check_call([f"{suite_dir}/deploy.py", "--help"])
 
 
-def test_quickstart_deploy():
-    suite_dir, deploy_dir = quickstart()
+def test_quickstart_deploy(quickstart):
+    suite_dir, deploy_dir = quickstart
 
     # test deployment
     subprocess.check_call(
         [
             f"{suite_dir}/deploy.py",
             os.path.join(suite_dir, "configs", "config.yaml"),
+            os.path.join(suite_dir, "configs", "host.yaml"),
             "-y",
         ]
     )
 
     def_file = os.path.join(deploy_dir, "test", "test.def")
     assert os.path.exists(def_file)
 
-
-def test_quickstart_tools():
-    suite_dir, deploy_dir = quickstart()
+@pytest.mark.xfail(reason="We require a host.yaml for suite build")
+def test_quickstart_tools_nohost_fails(quickstart):
+    suite_dir, deploy_dir = quickstart
 
     # test deployment
     subprocess.check_call(
@@ -70,20 +74,3 @@ def test_quickstart_tools():
 
     def_file = os.path.join(deploy_dir, "test", "test.def")
     assert os.path.exists(def_file)
-
-
-def test_quickstart_exec_context():
-    suite_dir, deploy_dir = quickstart()
-
-    # test deployment
-    subprocess.check_call(
-        [
-            f"{suite_dir}/deploy.py",
-            os.path.join(suite_dir, "configs", "config.yaml"),
-            os.path.join(suite_dir, "configs", "execution_contexts.yaml"),
-            "-y",
-        ]
-    )
-
-    def_file = os.path.join(deploy_dir, "test", "test.def")
-    assert os.path.exists(def_file)

wellies/__init__.py

@@ -5,7 +5,7 @@
     get_parser,
     get_user_globals,
     overwrite_entries,
-    parse_execution_contexts,
+    parse_submit_arguments,
     parse_yaml_files,
     substitute_variables,
 )

wellies/config.py

@@ -4,6 +4,7 @@
 from collections import abc
 from datetime import datetime, timedelta
 from string import Formatter
+from typing import Optional
 
 import yaml
 
@@ -70,7 +71,9 @@ def get_parser() -> ArgumentParser:
     return parser
 
 
-def parse_yaml_files(config_files, set_variables=None, global_vars=None):
+def parse_yaml_files(
+    config_files: list, set_variables=None, global_vars=None
+) -> dict:
     """
     Concatenates the config dictionaries and check for duplicates
     Override values in files with entries given on set_variables.
@@ -188,7 +191,7 @@ def parse(self, format_string):
                 yield literal_text, field_name, format_spec, conversion
 
 
-def check_environment_variables_substitution(value):
+def check_environment_variables_substitution(value: str) -> None:
     """Checks if environment variables are defined in the keys to format
     and make sure they exist in the environment.
 
@@ -210,19 +213,24 @@ def check_environment_variables_substitution(value):
                 raise ValueError(f"Environment variable {key} is not set")
 
 
-def substitute_variables(options, globals=None):
-    """parse base configuration file using the keys on that same file to
+def substitute_variables(
+    options: dict, globals: Optional[dict] = None
+) -> dict:
+    """Parse base configuration file using the keys on that same file to
     string format other values.
     Replaced variables will always be of type string.
 
     Parameters
     ----------
         options : dict
-            execution_contexts configuration dictionnary (from yaml file)
+            submit_arguments configuration dictionary (from yaml file)
         globals: dict, default=None
             A dictionary with globals key-value pairs to use on the string
             format substitution to be performed on the file content.
             New local assignments will take prevalence.
+    Returns
+    -------
+        dict: A dictionary with all variables substituted.
 
     :Attention: Does not support Lists
 
@@ -259,13 +267,12 @@ def update(tree, newMapping=None):
     global_subs = get_user_globals()
     if globals is not None:
         global_subs.update(globals)
+
     return update(options, global_subs)
 
 
-def parse_execution_contexts(options):
-    """parse execution context configuration file.
-    Expects a global `execution_contexts` mapping with other mappings
-    that define submit_arguments to be used on [pyflow.Task][]
+def parse_submit_arguments(options: dict) -> tuple:
+    """Parse submission arguments to be used on [pyflow.Task][]
     definitions.
 
     If a `defaults` mapping is defined it will be used as a default
@@ -276,11 +283,11 @@ def parse_execution_contexts(options):
     Parameters
     ----------
     options : dict
-        execution_contexts configuration dictionnary (from yaml file)
+        submit_arguments configuration dictionary (from yaml file)
 
     Returns
     -------
-    execution_contexts, submit_arguments_defaults : dict, dict
+    submit_arguments, submit_arguments_defaults : dict, dict
         Return parsed options as two dictionaries. The base execution contexts
         and a second one with the defaults options in a compatible format
         to be passed to a pyflow.Node variables argument.

wellies/hosts.py

@@ -43,7 +43,7 @@ def get_host(
         host = pf.LocalHost(
             hostname, user=user, ecflow_path=ecflow_path, **kwargs
         )
-        print("running on host: " + str(hostname))
+        print("Running on host: " + str(hostname))
     else:
         host = pf.TroikaHost(
             "%HOST%",
@@ -53,6 +53,6 @@ def get_host(
             ecflow_path=ecflow_path,
             **kwargs,
         )
-        print("submitting jobs using troika on host: " + str(hostname))
+        print("Submitting jobs using troika on host: " + str(hostname))
 
     return host

wellies/quickstart.py

@@ -21,8 +21,6 @@
     "author": pw_user.pw_gecos,
     "output_root": "{SCRATCH}",
     "deploy_root": "{PERM}/pyflow",
-    "job_out": "%ECF_HOME%",
-    "workdir": "$TMPDIR",
 }
 
 
@@ -50,7 +48,15 @@ def render_from_file():
 
 
 def start_project(options: Dict, overwrite: bool = False) -> None:
-    """Generate project basic structure based on options."""
+    """Generate project basic structure based on options.
+
+    Args:
+        options (dict): project options
+        overwrite (bool, optional): overwrite existing files.
+        Defaults to False.
+    Returns:
+        None
+    """
 
     def write_file(fpath: str, content: str) -> None:
         if overwrite or not path.isfile(fpath):
@@ -72,8 +78,6 @@ def write_file(fpath: str, content: str) -> None:
     out_root = path.join(out_root, project)
     options["output_root"] = out_root
 
-    options.setdefault("lib_dir", path.join("{output_root}", "local"))
-    options.setdefault("data_dir", path.join("{output_root}", "data"))
     options.setdefault(
         "deploy_dir", path.join(options["deploy_root"], project)
     )
@@ -107,8 +111,8 @@ def write_file(fpath: str, content: str) -> None:
         renderer.render("config.yaml_t", options),
     )
     write_file(
-        path.join(config_dir, "execution_contexts.yaml"),
-        renderer.render("execution_contexts.yaml_t", options),
+        path.join(config_dir, "host.yaml"),
+        renderer.render("host.yaml_t", options),
     )
     write_file(
         path.join(config_dir, "tools.yaml"),

wellies/templates/Makefile_t

@@ -4,6 +4,6 @@ WELLIES_VERSION = {{ version }}
 user:
 	module load wellies/$(WELLIES_VERSION); \
 	module list; \
-	./deploy.py configs/config.yaml configs/data.yaml configs/execution_contexts.yaml configs/tools.yaml ${ARGS}
+	./deploy.py configs/config.yaml configs/data.yaml configs/host.yaml configs/tools.yaml ${ARGS}
 
 default: user

wellies/templates/config.yaml_t

@@ -14,16 +14,19 @@
 
 # -------- Suite deployment configuration -------------------------------------
 # host options
-suite_name: "{{ project }}"
-hostname: "{{ host }}"
-user: "{{ user }}"
-
-# server options
-deploy_dir: "{{ deploy_dir }}"
-job_out: "{{ job_out }}"
-workdir: "{{ workdir }}"
-
-output_root: "{{ output_root }}"
+# suite_name: "{{ project }}"
+name: "{{ project }}"
+
+ecflow_server:
+  hostname: "{{ host }}"
+  user: "{{ user }}"
+  deploy_dir: "{{ deploy_dir }}"
+
+# -------- Global variables ------------------------------------
+ecflow_variables:
+  OUTPUT_ROOT: "{{ output_root }}"
+  LIB_DIR: "%OUTPUT_ROOT%/local"
+  DATA_DIR: "%OUTPUT_ROOT%/data"
 
 # ---------- Specific optionals -----------------------------------------------
 # add your project's specific variables and options here

wellies/templates/host.yaml_t

@@ -0,0 +1,13 @@
+host:
+  hostname: "{{ host }}"
+  user: "{{ user }}"
+  job_out: "%ECF_HOME%"
+  workdir: "$TMPDIR"
+  submit_arguments:
+    defaults:
+      queue: nf
+      memory_per_task: 8Gb
+      job_name: "%FAMILY1:NOT_DEF%_%TASK%"
+      tmpdir: 20Gb
+    sequential:
+      tasks: 1