From 07539570828c6a3d207b03bf9c31683fcff490b9 Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Fri, 3 Feb 2023 17:54:31 -0800 Subject: [PATCH 1/8] build: Incremented version number. --- nexport/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nexport/__init__.py b/nexport/__init__.py index 7c31ec0..dc75a61 100644 --- a/nexport/__init__.py +++ b/nexport/__init__.py @@ -1,5 +1,5 @@ # Dunder attributes -__version__ = "v0.4.0" # update setup.py +__version__ = "v0.4.1" # update setup.py __author__ = "Jordan Welsman" # Import submodules so submodule functions diff --git a/setup.py b/setup.py index 5709047..1dbddd5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup # Arguments -version = "0.4.0" # update __init__.py +version = "0.4.1" # update __init__.py python_version = ">=3.10" # Long description from README.md From 0884ae77c27fb896aa03e793a556b26c7aba8bad Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 00:06:13 -0800 Subject: [PATCH 2/8] fix: Attempting to fix import error. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1dbddd5..3ea9b26 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,9 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Utilities' ], - package_dir = {'': 'nexport'}, + package_dir = { + 'nexport': 'nexport' + }, install_requires = [ "jutils", "numpy<1.24", From b751e3f523d6a71ca8352e5ab28fe6c40b6b4971 Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 00:12:07 -0800 Subject: [PATCH 3/8] build: Changed dependency name. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3ea9b26..6e130fa 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ 'nexport': 'nexport' }, install_requires = [ - "jutils", + "jutl", "numpy<1.24", ], extras_require = { From bb160102953136fee0a1ab6afd129359f7a5d678 Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 00:20:13 -0800 Subject: [PATCH 4/8] fix: Still attempting to fix import issue. --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6e130fa..bad6ea8 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ with open("README.md", "r") as fh: long_description = fh.read() +# Define list of +py_modules = ["calculators", "colors", "generic", "models", "utils", "pytorch", "tensorflow"] + # Run stup function setup( name = 'nexport', @@ -19,7 +22,6 @@ author = 'Jordan Welsman', author_email = 'welsman@lbl.gov', url = 'https://github.com/JordanWelsman/nexport/', - py_modules = ["__init__", "calculators", "colors", "models", "utils"], classifiers = [ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', @@ -35,8 +37,8 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Utilities' ], - package_dir = { - 'nexport': 'nexport' + package_data = { + 'nexport': py_modules }, install_requires = [ "jutl", From c13e0eda0d38ada5bdc6b5eaf1bafecd652facbe Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 11:43:47 -0800 Subject: [PATCH 5/8] docs: Completed comment. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bad6ea8..fa30aa9 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ with open("README.md", "r") as fh: long_description = fh.read() -# Define list of +# Define list of submodules py_modules = ["calculators", "colors", "generic", "models", "utils", "pytorch", "tensorflow"] # Run stup function From c45da604dfd124d9ebe14ff5dadcc4961407ab9b Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 11:44:21 -0800 Subject: [PATCH 6/8] build(.gitignore): Removed specific filenames with ignored types. --- .gitignore | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index a1603f7..d5149c9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,17 +8,11 @@ dist/ test/__pycache__/ nexport/__pycache__/ nexport/nexport.egg-info/ +nexport.egg-info/ *.csv *.ipynb *.json *.pyc *.txt *.xml -model.csv -model.json -model.txt -test.py -weights_and_biases.csv -weights_and_biases.json -weights_and_biases.txt -weights_and_biases.xml +test.py \ No newline at end of file From 13e8a4646e6d4d9412201f5eb4b7d5a3688f6d40 Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Mon, 6 Feb 2023 11:46:37 -0800 Subject: [PATCH 7/8] build: Added egg info directories to unbuild script. --- unbuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unbuild b/unbuild index 145e6a9..93fce44 100755 --- a/unbuild +++ b/unbuild @@ -14,9 +14,9 @@ if [[ $REPLY =~ ^[Yy]$ ]] then rm -rf build # remove build directory if exists rm -rf dist # remove distribution directory if exists - rm -rf nexport/nexport.egg-info # remove egg info directory if exists find . -name __pycache__ -type d -print0|xargs -0 rm -r -- # remove all pycache directories find . -name .pytest_cache -type d -print0|xargs -0 rm -r -- # remove all pytest cache directories + find . -name nexport.egg-info -type d -print0|xargs -0 rm -r -- # remove all egg-info directories echo "Project successfully unbuilt." else echo "Operation aborted." From 7bf2a3710795fee514f4177a7f48f8b419577a72 Mon Sep 17 00:00:00 2001 From: Jordan Welsman Date: Thu, 9 Feb 2023 15:53:46 -0800 Subject: [PATCH 8/8] fix: Removed os.getlogin() parameter default & added metadata object for json_exp. --- nexport/pytorch/exporting.py | 38 +++++++++++++++++++++++------------- nexport/utils.py | 7 +++---- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/nexport/pytorch/exporting.py b/nexport/pytorch/exporting.py index 20939b0..6b2af65 100644 --- a/nexport/pytorch/exporting.py +++ b/nexport/pytorch/exporting.py @@ -1,6 +1,7 @@ # Module imports import nexport import json +# import os # disbaling os.getlogin() as it causes issues on supercomputers import datetime as dt import time as t @@ -94,17 +95,18 @@ def create_layer_object(weights: list, biases: list, verbose: int = None) -> lis return neuron_list # return constructed layer -def create_model_metadata(model_name: str, model_author: str = None) -> dict: +def create_model_metadata(model_name: str, model_author: str = None, using_skip_connections: bool = None) -> dict: model_metadata = { "modelName": model_name, "modelAuthor": model_author, - "compilationDate": str(dt.datetime.now()) + "compilationDate": str(dt.datetime.now()), + "usingSkipConnections": using_skip_connections } return model_metadata # return model metadata object -def create_model_object(model: object, verbose: int = None, include_metadata: bool = None, model_name: str = None, model_author: str = None) -> object: +def create_model_object(model: object, verbose: int = None, include_metadata: bool = None, model_name: str = None, model_author: str = None, using_skip_connections: bool = None) -> object: """ Function which creates a model object from a collection of layers instantiated with layer @@ -116,7 +118,7 @@ def create_model_object(model: object, verbose: int = None, include_metadata: bo weights, biases = create_paramater_arrays(model=model, verbose=verbose) if include_metadata: # insert model metadata into model object - model_object["model"] = create_model_metadata(model_name=model_name, model_author=model_author) + model_object["metadata"] = create_model_metadata(model_name=model_name, model_author=model_author, using_skip_connections=using_skip_connections) if verbose >= 3: # if verbose set to at least 3 print(f"{c.YELLOW}Creating layers...{c.DEFAULT}") @@ -137,14 +139,7 @@ def create_model_object(model: object, verbose: int = None, include_metadata: bo return model_object # return constructed network -def new_func(): - """ - Template function - """ - pass - - -def export_to_json(model: object, filename: str = None, indent: int = None, verbose: int = None, include_metadata: bool = None, model_name: str = None, model_author: str = None) -> None: +def export_to_json(model: object, filename: str = None, indent: int = None, verbose: int = None, include_metadata: bool = None, model_name: str = None, model_author: str = None, using_skip_connections: bool = None) -> None: """ Function which exports a passed model object to a JSON file. @@ -152,7 +147,7 @@ def export_to_json(model: object, filename: str = None, indent: int = None, verb t1 = t.time() model_object = {} if include_metadata: - model_object = create_model_object(model=model, verbose=verbose, include_metadata=include_metadata, model_name=model_name, model_author=model_author) + model_object = create_model_object(model=model, verbose=verbose, include_metadata=include_metadata, model_name=model_name, model_author=model_author, using_skip_connections=using_skip_connections) else: model_object = create_model_object(model=model, verbose=verbose) json_object = json.dumps(obj=model_object, indent=indent) @@ -169,7 +164,7 @@ def export_to_json(model: object, filename: str = None, indent: int = None, verb print(f"{c.MAGENTA} Time taken: {c.LIGHTMAGENTA}{round(time, 2)}{c.MAGENTA}s{c.DEFAULT}") -def export_to_json_experimental(model: object, filename: str = None, indent: int = None, verbose: int = None) -> None: +def export_to_json_experimental(model: object, filename: str = None, indent: int = None, verbose: int = None, include_metadata: bool = None, model_name: str = None, model_author: str = None, using_skip_connections: bool = None) -> None: """ Function which exports a passed model object to a JSON file, but @@ -177,10 +172,25 @@ def export_to_json_experimental(model: object, filename: str = None, indent: int """ t1 = t.time() model_object = create_model_object(model=model, verbose=verbose) + model_metadata = create_model_metadata(model_name=model_name, model_author=model_author, using_skip_connections=using_skip_connections) indent = " " with open(nexport.append_extension(filename=filename, extension="json"), "w") as outfile: outfile.write("{\n") + if include_metadata: + outfile.write(f"{indent}\"metadata\": " + "{\n") + for d, data in enumerate(model_metadata.keys()): + if type(model_metadata[data]) is str: + outfile.write(f"{indent}{indent}\"{data}\": \"{model_metadata[data]}\"") + elif type(model_metadata[data]) is bool: + outfile.write(f"{indent}{indent}\"{data}\": {str(model_metadata[data]).lower()}") + else: + outfile.write(f"{indent}{indent}\"{data}\": null") + if d < len(model_metadata.keys()) - 1: + outfile.write(",\n") + else: + outfile.write("\n") + outfile.write(f"{indent}" + "},\n") for layer_type in model_object.keys(): if layer_type == "hidden_layers": outfile.write(f"{indent}\"{layer_type}\": [\n") diff --git a/nexport/utils.py b/nexport/utils.py index d80dbc9..7de5880 100644 --- a/nexport/utils.py +++ b/nexport/utils.py @@ -1,7 +1,6 @@ # Module imports import nexport import sys -import os # File imports from nexport.pytorch import exporting as npte @@ -37,16 +36,16 @@ def detect_framework(imported: object = sys.modules.keys()) -> str: return "none" -def export(model: object, filetype: str, filename: str = "model", indent: int = 4, verbose: int = 1, include_metadata: bool = False, model_name: str = "My Model", model_author: str = os.getlogin()): +def export(model: object, filetype: str, filename: str = "model", indent: int = 4, verbose: int = 1, include_metadata: bool = False, model_name: str = "My Model", model_author: str = None, using_skip_connections: bool = None) -> None: match nexport.__framework__: case "pytorch": match filetype: case "txt": npte.export_to_file(model=model, filename=filename) case "json": - npte.export_to_json(model=model, filename=filename, indent=indent, verbose=verbose, include_metadata=include_metadata, model_name=model_name, model_author=model_author) + npte.export_to_json(model=model, filename=filename, indent=indent, verbose=verbose, include_metadata=include_metadata, model_name=model_name, model_author=model_author, using_skip_connections=using_skip_connections) case "json_exp": - npte.export_to_json_experimental(model=model, filename=filename, indent=indent, verbose=verbose) + npte.export_to_json_experimental(model=model, filename=filename, indent=indent, verbose=verbose, include_metadata=include_metadata, model_name=model_name, model_author=model_author, using_skip_connections=using_skip_connections) case "csv" | "xml": raise NotImplementedError(f"This feature (exporting {nexport.__framework__} in {filetype}) has not yet been implemented.") case other: