Prepare 0 7 5 release (#54)

* rework scripts such that it uses the jinja package instead of jinja cli

* refactor script

* Prepare for upcoming changes in the config schema

change command to generate the cli and config schemas

* Remove old intermediate .yaml files

Remove `.qmd` from the file names in the `config_pages_settings.yaml`. It's now added in the script.

* Tweak empty lines in config pages jinja template

* Add environmentVariables to the config_pages_settings.yaml

* Update a bunch of automatically generated file changes

Mostly adding the default value, some type string improvements or clearing an extra empty line
Adds the new test --setup command documentation too
config.info, functionality.argumentGroup, environment variables are documented

* Use flat list of config schema data instead of map of map

Actually now a list of list instead of map of map of list
Get title information from the `__this__` parameter
Add some minor exceptions in the title that was previously handled by manually defining map keys

* Prepare & verify removal of removed parameter documentation

Also assuming any mention of 'VDSL3' is supposed to be removed.

* Copy static reference pages from viash

- Remove all subfolders in reference
- Copy static pages
- Rebuild cli & config pages

* Don't display command arguments if they are hidden

We're not (yet?) telling the --colorize and --loglevel options to the user so don't display it in the documentation

* refactor guide, add vdsl3 reference docs

* Update the pages for the Viash 0.7.5 release.

* Add vdsl3 references (#61)

* fixes to vdsl3 references

* fix link

---------

Co-authored-by: Robrecht Cannoodt <rcannood@gmail.com>

_src/automation/config_pages_settings.yaml

@@ -1,63 +1,65 @@
 structure:
-  'config/config': ./index.qmd
-  'config/project': ../project/index.qmd
+  'Config': ./index
+  'Project': ../project/index
+  'Info': ./info
+  'EnvironmentVariables': ./environmentVariables
 
-  'functionality/functionality': ./functionality/index.qmd
-  'functionality/author': ./functionality/author.qmd
-  'functionality/computationalRequirements': ./functionality/computationalRequirements.qmd
+  'Functionality': ./functionality/index
+  'Author': ./functionality/author
+  'ComputationalRequirements': ./functionality/computationalRequirements
+  'ArgumentGroup': ./functionality/argumentGroup
 
-  'platforms/platform': ./platforms/index.qmd
-  'platforms/nativePlatform': ./platforms/native/index.qmd
-  'platforms/dockerPlatform': ./platforms/docker/index.qmd
-  'platforms/nextflowVdsl3Platform': ./platforms/nextflow/index.qmd
-  'platforms/nextflowLegacyPlatform': ./platforms/nextflowLegacy/index.qmd
+  'Platform': ./platforms/index
+  'NativePlatform': ./platforms/native/index
+  'DockerPlatform': ./platforms/docker/index
+  'NextflowPlatform': ./platforms/nextflow/index
 
-  'arguments/argument': ./functionality/arguments/index.qmd
-  'arguments/boolean_false': ./functionality/arguments/boolean_false.qmd
-  'arguments/boolean': ./functionality/arguments/boolean.qmd
-  'arguments/boolean_true': ./functionality/arguments/boolean_true.qmd
-  'arguments/double': ./functionality/arguments/double.qmd
-  'arguments/file': ./functionality/arguments/file.qmd
-  'arguments/integer': ./functionality/arguments/integer.qmd
-  'arguments/long': ./functionality/arguments/long.qmd
-  'arguments/string': ./functionality/arguments/string.qmd
+  'Argument': ./functionality/arguments/index
+  'BooleanFalseArgument': ./functionality/arguments/boolean_false
+  'BooleanArgument': ./functionality/arguments/boolean
+  'BooleanTrueArgument': ./functionality/arguments/boolean_true
+  'DoubleArgument': ./functionality/arguments/double
+  'FileArgument': ./functionality/arguments/file
+  'IntegerArgument': ./functionality/arguments/integer
+  'LongArgument': ./functionality/arguments/long
+  'StringArgument': ./functionality/arguments/string
 
-  'requirements/requirements': ./platforms/docker/setup/index.qmd
-  'requirements/apkRequirements': ./platforms/docker/setup/apkRequirements.qmd
-  'requirements/aptRequirements': ./platforms/docker/setup/aptRequirements.qmd
-  'requirements/dockerRequirements': ./platforms/docker/setup/dockerRequirements.qmd
-  'requirements/javascriptRequirements': ./platforms/docker/setup/javascriptRequirements.qmd
-  'requirements/pythonRequirements': ./platforms/docker/setup/pythonRequirements.qmd
-  'requirements/rRequirements': ./platforms/docker/setup/rRequirements.qmd
-  'requirements/rubyRequirements': ./platforms/docker/setup/rubyRequirements.qmd
-  'requirements/yumRequirements': ./platforms/docker/setup/yumRequirements.qmd
+  'Requirements': ./platforms/docker/setup/index
+  'ApkRequirements': ./platforms/docker/setup/apkRequirements
+  'AptRequirements': ./platforms/docker/setup/aptRequirements
+  'DockerRequirements': ./platforms/docker/setup/dockerRequirements
+  'JavaScriptRequirements': ./platforms/docker/setup/javascriptRequirements
+  'PythonRequirements': ./platforms/docker/setup/pythonRequirements
+  'RRequirements': ./platforms/docker/setup/rRequirements
+  'RubyRequirements': ./platforms/docker/setup/rubyRequirements
+  'YumRequirements': ./platforms/docker/setup/yumRequirements
 
-  'resources/resource': ./functionality/resources/index.qmd
-  'resources/cSharpScript': ./functionality/resources/cSharpScript.qmd
-  'resources/pythonScript': ./functionality/resources/pythonScript.qmd
-  'resources/plainFile': ./functionality/resources/plainFile.qmd
-  'resources/bashScript': ./functionality/resources/bashScript.qmd
-  'resources/scalaScript': ./functionality/resources/scalaScript.qmd
-  'resources/nextflowScript': ./functionality/resources/nextflowScript.qmd
-  'resources/rScript': ./functionality/resources/rScript.qmd
-  'resources/executable': ./functionality/resources/executable.qmd
-  'resources/javaScriptScript': ./functionality/resources/javaScriptScript.qmd
+  'Resource': ./functionality/resources/index
+  'CSharpScript': ./functionality/resources/cSharpScript
+  'PythonScript': ./functionality/resources/pythonScript
+  'PlainFile': ./functionality/resources/plainFile
+  'BashScript': ./functionality/resources/bashScript
+  'ScalaScript': ./functionality/resources/scalaScript
+  'NextflowScript': ./functionality/resources/nextflowScript
+  'RScript': ./functionality/resources/rScript
+  'Executable': ./functionality/resources/executable
+  'JavaScriptScript': ./functionality/resources/javaScriptScript
 
-  'nextflowParameters/nextflowDirectives': ./platforms/nextflow/directives.qmd
-  'nextflowParameters/nextflowAuto': ./platforms/nextflow/auto.qmd
-  'nextflowParameters/nextflowConfig': ./platforms/nextflow/config.qmd
+  'NextflowDirectives': ./platforms/nextflow/directives
+  'NextflowAuto': ./platforms/nextflow/auto
+  'NextflowConfig': ./platforms/nextflow/config
 
 order:
-  'config/config': 20
-  'config/project': 30
+  'Config': 20
+  'Project': 30
 
-  'functionality/functionality': 10
-  'platforms/platform': 20
+  'Functionality': 10
+  'Platform': 20
+  'Info': 30
 
-  'platforms/nativePlatform': 10
-  'platforms/dockerPlatform': 20
-  'platforms/nextflowVdsl3Platform': 30
-  'platforms/nextflowLegacyPlatform': 40
+  'NativePlatform': 10
+  'DockerPlatform': 20
+  'NextflowPlatform': 30
 
 keywords:
   mykeyword: /reference/config.html

_src/automation/generate_reference_cli_pages/script.py

@@ -1,23 +1,29 @@
+import json, re, yaml
+from pathlib import Path
+import jinja2
 
 ## VIASH START
-# The following code has been auto-generated by Viash.
 par = {
-  'input': r'/path/to/file',
-  'output': r'reference/cli'
+  'input': 'reference/cli_schema_export.json',
+  'output': 'reference/cli'
+}
+meta = {
+	"resources_dir": "_src/automation/generate_reference_cli_pages"
 }
 ## VIASH END
-import subprocess, json, re, yaml
-from pathlib import Path
 
+output = Path(par["output"])
 keyword_regex = r"\@\[(.*?)\]\((.*?)\)"
 
 template_file = Path(meta['resources_dir'], "template_cli_page.j2.qmd")
 config_file = Path(meta['resources_dir'], 'config_pages_settings.yaml')
 
+with open(template_file) as f:
+	template = jinja2.Template(f.read())
+
 with open(config_file, 'r') as infile:
 	config_pages_settings = yaml.safe_load(infile)
 
-
 def generate_pages():
 	""" Load the generated JSON file and creates pages for every command entry. """
 
@@ -26,7 +32,7 @@ def generate_pages():
 
 	for entry in viash_json:
 		# create_page(entry["name"], entry)
-		if "bannerCommand" in entry:		
+		if "bannerCommand" in entry:
 			create_page(entry["name"], entry)
 		else:
 			for subcommand in entry['subcommands']:
@@ -47,22 +53,14 @@ def create_page(name, json_data):
 				if 'descr' in arg:
 					arg['descr'] = replace_keywords(arg["descr"])
 
-	render_jinja_page(par['output'], name.replace(' ', '_'), page_data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
-	""" Write data to yaml file and run jinja. """
-
-	full_path = Path(folder, filename)
-	base_dir = full_path.parent
-	yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
-	qmd_file = full_path.with_suffix('.qmd')
-
-	base_dir.mkdir(parents=True, exist_ok=True)	
-
-	with open(yaml_file, 'w') as outfile:
-		yaml.safe_dump(data, outfile, default_flow_style=False)
+	qmd_file = output / f"{name.replace(' ', '_')}.qmd"
+	render_jinja_page(qmd_file, page_data)
 
-	subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+	"""Run jinja on data and write results to file."""
+	path.parent.mkdir(parents=True, exist_ok=True)	
+	content = template.render(**data)
+	path.write_text(content)
 
 def replace_keywords(text: str) -> str:
 	""" Finds all keywords in the format @[keyword](text) and returns the replacements based on the keywords settings. """

_src/automation/generate_reference_cli_pages/template_cli_page.j2.qmd

@@ -17,7 +17,7 @@ search: true
 
 | Argument | Description | Type |
 |-|:----|-:
-{% for arg in d.opts|sort(attribute="name") -%}
+{% for arg in d.opts|sort(attribute="name") if not arg.hidden -%}
 
 {% if arg.name == "config" %}| `{{arg.name}}`{%- else -%}| `--{{arg.name}}`{% endif -%}
 {%- if arg.short is defined %}, `-{{arg.short}}`{% endif %} | {{arg.descr|replace("$", "\\$")|replace("\n", " ")}}

_src/automation/generate_reference_config_pages/script.py

@@ -1,54 +1,62 @@
+import json, re, yaml
+from pathlib import Path
+import jinja2
 
 ## VIASH START
-# The following code has been auto-generated by Viash.
 par = {
-  'input': r'/path/to/file',
-  'output': r'reference/config'
+  'input': 'reference/config_schema_export.json',
+  'output': 'reference/cli'
+}
+meta = {
+	"resources_dir": "_src/automation/generate_reference_config_pages"
 }
 ## VIASH END
-import subprocess, json, re, yaml
-from pathlib import Path
 
+output = Path(par["output"])
 keyword_regex = r"\@\[(.*?)\]\((.*?)\)"
 
 template_file = Path(meta['resources_dir'], "template_config_page.j2.qmd")
 config_file = Path(meta['resources_dir'], 'config_pages_settings.yaml')
 
+with open(template_file) as f:
+	template = jinja2.Template(f.read())
+
 with open(config_file, 'r') as infile:
 	config_pages_settings = yaml.safe_load(infile)
 
-
 def read_json_entries():
 	""" Load the generated JSON file, split into logical page chunks and generate pages. """
 
 	with open(par['input'], 'r') as infile:
 		viash_json = json.load(infile)
 
-	for topic, topic_json in viash_json.items():
-		if isinstance(topic_json, dict):
-			for subtopic, subtopic_json in topic_json.items():
-				generate_page(topic, subtopic, subtopic_json)
-		else:
-			generate_page(".", topic, topic_json)
+	for topic_json in viash_json:
+		generate_page(topic_json)
 
-def generate_page(topic: str, subtopic: str, json_data: list):
+def generate_page(json_data: list):
 	""" Receives JSON data, does some minor data manipulation and writes to yaml & qmd. """
 
-	# if topic == 'arguments': # Keep title of argument pages as-is
-	# 	title = subtopic
-	# else:
-	title = re.sub(r"(\w)([A-Z])", r"\1 \2", subtopic).title() # split words and capitalize
-
+	this_parameter = {}
 	# Fix description markdown keywords to links
 	for d in json_data:
 		if d['name'] == '__merge__':
 			d['name'] = '`__merge__' # Bump __merge__ behind __this__ when sorted
+		if d['name'] == '__this__':
+			this_parameter = d
 		if 'description' in d:
 			d['description'] = replace_keywords(d["description"])
 
-	page_data = {"topic": topic, "title": title, "data": json_data}
+	topic = this_parameter['type']
 
-	filename = f"{topic}/{subtopic}"
+	# split words and capitalize
+	# do some extra substitutions to clean things up a bit
+	title = re.sub(r"(\w)([A-Z])", r"\1 \2", topic).title() \
+		.replace("Java Script", "JavaScript") \
+		.replace("C Sharp", "C#") \
+		.replace(" Argument", "")
+	filename = topic
+
+	page_data = {"title": title, "data": json_data}
 
 	try:
 		page_data['order'] = config_pages_settings['order'][filename]
@@ -59,23 +67,16 @@ def generate_page(topic: str, subtopic: str, json_data: list):
 		filename = config_pages_settings['structure'][filename]
 	except KeyError:
 		print(f"Could not find {filename} in the config pages settings structure")
-
-	render_jinja_page(par['output'], filename, page_data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
-	""" Write data to yaml file and run jinja. """
-
-	full_path = Path(folder, filename)
-	base_dir = full_path.parent
-	yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
-	qmd_file = full_path.with_suffix('.qmd')
 
-	base_dir.mkdir(parents=True, exist_ok=True)	
+	qmd_file = output / (filename + ".qmd")
+
+	render_jinja_page(qmd_file, page_data)
 
-	with open(yaml_file, 'w') as outfile:
-		yaml.safe_dump(data, outfile, default_flow_style=False)
-
-	subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+	"""Run jinja on data and write results to file."""
+	path.parent.mkdir(parents=True, exist_ok=True)	
+	content = template.render(**data)
+	path.write_text(content)
 
 def replace_keywords(text: str) -> str:
 	""" Finds all keywords in the format @[keyword](text) and returns the replacements based on the keywords settings. """

_src/automation/generate_reference_config_pages/template_config_page.j2.qmd

@@ -9,12 +9,15 @@ order: {{order}}
 
 {% if not d.name == "__this__" %}
 ## {{d.name|replace("`__merge__", "\_\_merge\_\_")}}{# Replace fixes __merge__ being rendered bold and coming before __this__ when sorted #}
-{% if "Option of " in d.type %}
-**Type**: `{{d.type|replace("Option of ", "")}}`
-{% elif "OneOrMore of " in d.type %}
-**Type**: `{{d.type|replace("OneOrMore of ", "")}}` / `List of {{d.type|replace("OneOrMore of ", "")}}`
+{% if "Option of " in d.niceType %}
+**Type**: `{{d.niceType|replace("Option of ", "")}}`
+{% elif "OneOrMore of " in d.niceType %}
+**Type**: `{{d.niceType|replace("OneOrMore of ", "")}}` / `List of {{d.niceType|replace("OneOrMore of ", "")}}`
 {% else %}
-**Type**: `{{d.type}}`
+**Type**: `{{d.niceType}}`
+{% endif %}
+{%- if d.default is defined %}
+**Default**: `{{d.default}}`
 {% endif %}
 {% else %}
 {% endif -%}

_src/automation/generate_version_blog_pages/script.py

@@ -1,15 +1,23 @@
-import subprocess, re, yaml
+import re
 from pathlib import Path
+import jinja2
 
 ## VIASH START
 par = {
-    'input': "_src/automation/generate_version_blog_pages/test_changelog.md",
-    'output': "output/"
+    'input': '../viash/CHANGELOG.md',
+    'output': 'blog/posts'
+}
+meta = {
+    "resources_dir": "_src/automation/generate_version_blog_pages"
 }
 ## VIASH END
 
+output = Path(par["output"])
 template_file = Path(meta['resources_dir'], "template_blog_page.j2.qmd")
 
+with open(template_file) as f:
+	template = jinja2.Template(f.read())
+
 def bump_md_line(string: str) -> str:
     """ if header, bump header up one level """
     if re.match(r"^#+ .+$", string):
@@ -43,22 +51,14 @@ def handle_section(lines: list[str]):
         "changes": "".join(bump_markdown_headers(lines))
     }
 
-    render_jinja_page(par['output'], f'viash-{version}/index.qmd', data)
-
-def render_jinja_page(folder: str, filename: str, data: dict):
-	""" Write data to yaml file and run jinja. """
-
-	full_path = Path(folder, filename)
-	base_dir = full_path.parent
-	yaml_file = Path(base_dir, "_" + full_path.name).with_suffix('.yaml')
-	qmd_file = full_path.with_suffix('.qmd')
-
-	base_dir.mkdir(parents=True, exist_ok=True)	
-
-	with open(yaml_file, 'w') as outfile:
-		yaml.safe_dump(data, outfile, default_flow_style=False)
+    qmd_file = output / f'viash-{version}/index.qmd'
+    render_jinja_page(qmd_file, data)
 
-	subprocess.run(["jinja2", template_file, yaml_file, "-o", qmd_file])
+def render_jinja_page(path: Path, data: dict):
+	"""Run jinja on data and write results to file."""
+	path.parent.mkdir(parents=True, exist_ok=True)	
+	content = template.render(**data)
+	path.write_text(content)
 
 def load_log(changelog_path: str):
     """ Load changelog and split into sections """