Merge pull request #1286 from pyiron/executable_container

Implement ExecutableJobContainer

pyiron_base/__init__.py

@@ -18,9 +18,10 @@
 from pyiron_base.jobs.job.extension.executable import Executable
 from pyiron_base.project.external import Notebook, load, dump
 from pyiron_base.jobs.dynamic import warn_dynamic_job_classes
+from pyiron_base.jobs.flex.factory import create_job_factory
+from pyiron_base.jobs.job.extension.server.queuestatus import validate_que_request
 from pyiron_base.jobs.job.generic import GenericJob
 from pyiron_base.jobs.job.interactive import InteractiveBase
-from pyiron_base.jobs.master.interactivewrapper import InteractiveWrapper
 from pyiron_base.jobs.job.extension.jobstatus import (
     JobStatus,
     job_status_successful_lst,
@@ -30,14 +31,14 @@
 from pyiron_base.jobs.job.jobtype import JOB_CLASS_DICT, JobType, JobTypeChoice
 from pyiron_base.jobs.job.template import TemplateJob, PythonTemplateJob
 from pyiron_base.jobs.job.factory import JobFactoryCore
+from pyiron_base.jobs.master.flexible import FlexibleMaster
 from pyiron_base.jobs.master.generic import GenericMaster, get_function_from_string
+from pyiron_base.jobs.master.interactivewrapper import InteractiveWrapper
 from pyiron_base.jobs.master.list import ListMaster
 from pyiron_base.jobs.master.parallel import ParallelMaster, JobGenerator
 from pyiron_base.jobs.master.serial import SerialMasterBase
-from pyiron_base.jobs.master.flexible import FlexibleMaster
 from pyiron_base.project.generic import Project, Creator
 from pyiron_base.utils.parser import Logstatus, extract_data_from_file
-from pyiron_base.jobs.job.extension.server.queuestatus import validate_que_request
 from pyiron_base.state.settings import Settings
 from pyiron_base.state.install import install_dialog
 from pyiron_base.jobs.datamining import PyironTable, TableJob

pyiron_base/jobs/flex/executablecontainer.py

@@ -0,0 +1,99 @@
+import cloudpickle
+import numpy as np
+from pyiron_base.jobs.job.template import TemplateJob
+
+
+class ExecutableContainerJob(TemplateJob):
+    """
+    The ExecutableContainerJob is designed to wrap any kind of external executable into a pyiron job object by providing
+    a write_input(input_dict, working_directory) and a collect_output(working_directory) function.
+
+    Example:
+
+    >>> def write_input(input_dict, working_directory="."):
+    >>>     with open(os.path.join(working_directory, "input_file"), "w") as f:
+    >>>         f.write(str(input_dict["energy"]))
+    >>>
+    >>>
+    >>> def collect_output(working_directory="."):
+    >>>     with open(os.path.join(working_directory, "output_file"), "r") as f:
+    >>>         return {"energy": float(f.readline())}
+    >>>
+    >>>
+    >>> from pyiron_base import Project
+    >>> pr = Project("test")
+    >>> pr.create_job_class(
+    >>>     class_name="CatJob",
+    >>>     write_input_funct=write_input,
+    >>>     collect_output_funct=collect_output,
+    >>>     default_input_dict={"energy": 1.0},
+    >>>     executable_str="cat input_file > output_file",
+    >>> )
+    >>> job = self.project.create.job.CatJob(job_name="job_test")
+    >>> job.input["energy"] = 2.0
+    >>> job.run()
+    >>> job.output
+
+    """
+
+    def __init__(self, project, job_name):
+        super().__init__(project, job_name)
+        self._write_input_funct = None
+        self._collect_output_funct = None
+
+    def set_job_type(
+        self,
+        write_input_funct,
+        executable_str,
+        collect_output_funct,
+        default_input_dict=None,
+    ):
+        """
+        Set the pre-defined write_input() and collect_output() function plus a dictionary of default inputs and an
+        executable string.
+
+        Args:
+            write_input_funct (callable): The write input function write_input(input_dict, working_directory)
+            executable_str (str): Call to an external executable
+            collect_output_funct (callable): The collect output function collect_output(working_directory)
+            default_input_dict (dict/None): Default input for the newly created job class
+
+        Returns:
+            callable: Function which requires a project and a job_name as input and returns a job object
+        """
+        self._write_input_funct = write_input_funct
+        self.executable = executable_str
+        self._collect_output_funct = collect_output_funct
+        if default_input_dict is not None:
+            self.input.update(default_input_dict)
+
+    def write_input(self):
+        self._write_input_funct(
+            input_dict=self.input.to_builtin(), working_directory=self.working_directory
+        )
+
+    def collect_output(self):
+        self.output.update(
+            self._collect_output_funct(working_directory=self.working_directory)
+        )
+        self.to_hdf()
+
+    def to_hdf(self, hdf=None, group_name=None):
+        super().to_hdf(hdf=hdf, group_name=group_name)
+        if self._write_input_funct is not None:
+            self.project_hdf5["write_input_function"] = np.void(
+                cloudpickle.dumps(self._write_input_funct)
+            )
+        if self._collect_output_funct is not None:
+            self.project_hdf5["collect_output_function"] = np.void(
+                cloudpickle.dumps(self._collect_output_funct)
+            )
+
+    def from_hdf(self, hdf=None, group_name=None):
+        super().from_hdf(hdf=hdf, group_name=group_name)
+        self._write_input_funct = cloudpickle.loads(
+            self.project_hdf5["write_input_function"]
+        )
+        self._collect_output_funct = cloudpickle.loads(
+            self.project_hdf5["collect_output_function"]
+        )

pyiron_base/jobs/flex/factory.py

@@ -0,0 +1,73 @@
+from pyiron_base.utils.instance import static_isinstance
+
+
+def create_job_factory(
+    write_input_funct,
+    executable_str,
+    collect_output_funct,
+    default_input_dict=None,
+):
+    """
+    Create a new job class based on pre-defined write_input() and collect_output() function plus a dictionary of
+    default inputs and an executable string.
+
+    Args:
+        write_input_funct (callable): The write input function write_input(input_dict, working_directory)
+        executable_str (str): Call to an external executable
+        collect_output_funct (callable): The collect output function collect_output(working_directory)
+        default_input_dict (dict/None): Default input for the newly created job class
+
+    Example:
+
+    >>> def write_input(input_dict, working_directory="."):
+    >>>     with open(os.path.join(working_directory, "input_file"), "w") as f:
+    >>>         f.write(str(input_dict["energy"]))
+    >>>
+    >>>
+    >>> def collect_output(working_directory="."):
+    >>>     with open(os.path.join(working_directory, "output_file"), "r") as f:
+    >>>         return {"energy": float(f.readline())}
+    >>>
+    >>>
+    >>> from pyiron_base import Project, create_job_factory
+    >>> pr = Project("test")
+    >>> create_catjob = create_job_factory(
+    >>>     write_input_funct=write_input,
+    >>>     collect_output_funct=collect_output,
+    >>>     default_input_dict={"energy": 1.0},
+    >>>     executable_str="cat input_file > output_file",
+    >>> )
+    >>> job = create_catjob(project=pr, job_name="job_test")
+    >>> job.input["energy"] = 2.0
+    >>> job.run()
+    >>> job.output
+    """
+
+    def job_factory(project, job_name):
+        """
+        Create a job based on the previously defined write_input(), collect_output() and the executable string.
+
+        Args:
+            project (ProjectHDFio/ Project): ProjectHDFio instance which points to the HDF5 file the job is stored in
+            job_name (str): name of the job, which has to be unique within the project
+
+        Returns:
+            pyiron_base.jobs.flex.executablecontainer.ExecutableContainerJob: pyiron job object
+        """
+        if static_isinstance(project, "pyiron_base.project.generic.Project"):
+            job = project.create.job.ExecutableContainerJob(job_name=job_name)
+        elif static_isinstance(project, "pyiron_base.storage.hdfio.ProjectHDFio"):
+            job = project.project.create.job.ExecutableContainerJob(job_name=job_name)
+        else:
+            raise TypeError(
+                "Expected ProjectHDFio/ Project but recieved", type(project)
+            )
+        job.set_job_type(
+            write_input_funct=write_input_funct,
+            collect_output_funct=collect_output_funct,
+            default_input_dict=default_input_dict,
+            executable_str=executable_str,
+        )
+        return job
+
+    return job_factory

pyiron_base/jobs/job/jobtype.py

@@ -33,6 +33,7 @@
     "SerialMasterBase": "pyiron_base.jobs.master.serial",
     "TableJob": "pyiron_base.jobs.datamining",
     "WorkerJob": "pyiron_base.jobs.worker",
+    "ExecutableContainerJob": "pyiron_base.jobs.flex.executablecontainer",
     "PythonFunctionContainerJob": "pyiron_base.jobs.flex.pythonfunctioncontainer",
 }
 

pyiron_base/project/generic.py

@@ -34,9 +34,15 @@
 from pyiron_base.storage.hdfio import ProjectHDFio
 from pyiron_base.storage.filedata import load_file
 from pyiron_base.utils.deprecate import deprecate
-from pyiron_base.jobs.job.util import _special_symbol_replacements, _get_safe_job_name
 from pyiron_base.interfaces.has_groups import HasGroups
-from pyiron_base.jobs.job.jobtype import JobType, JobTypeChoice, JobFactory
+from pyiron_base.jobs.flex.factory import create_job_factory
+from pyiron_base.jobs.job.util import _special_symbol_replacements, _get_safe_job_name
+from pyiron_base.jobs.job.jobtype import (
+    JobType,
+    JobTypeChoice,
+    JobFactory,
+    JOB_CLASS_DICT,
+)
 from pyiron_base.jobs.job.extension.server.queuestatus import (
     queue_delete_job,
     queue_is_empty,
@@ -326,6 +332,58 @@ def create_group(self, group):
         new = self.copy()
         return new.open(group, history=False)
 
+    def create_job_class(
+        self,
+        class_name,
+        write_input_funct,
+        collect_output_funct,
+        default_input_dict,
+        executable_str,
+    ):
+        """
+        Create a new job class based on pre-defined write_input() and collect_output() function plus a dictionary of
+        default inputs and an executable string.
+
+        Args:
+            class_name (str): A name for the newly created job class, so it is accessible via pr.create.job.<class_name>
+            write_input_funct (callable): The write input function write_input(input_dict, working_directory)
+            collect_output_funct (callable): The collect output function collect_output(working_directory)
+            default_input_dict (dict): Default input for the newly created job class
+            executable_str (str): Call to an external executable
+
+        Example:
+
+        >>> def write_input(input_dict, working_directory="."):
+        >>>     with open(os.path.join(working_directory, "input_file"), "w") as f:
+        >>>         f.write(str(input_dict["energy"]))
+        >>>
+        >>>
+        >>> def collect_output(working_directory="."):
+        >>>     with open(os.path.join(working_directory, "output_file"), "r") as f:
+        >>>         return {"energy": float(f.readline())}
+        >>>
+        >>>
+        >>> from pyiron_base import Project
+        >>> pr = Project("test")
+        >>> pr.create_job_class(
+        >>>     class_name="CatJob",
+        >>>     write_input_funct=write_input,
+        >>>     collect_output_funct=collect_output,
+        >>>     default_input_dict={"energy": 1.0},
+        >>>     executable_str="cat input_file > output_file",
+        >>> )
+        >>> job = pr.create.job.CatJob(job_name="job_test")
+        >>> job.input["energy"] = 2.0
+        >>> job.run()
+        >>> job.output
+        """
+        JOB_CLASS_DICT[class_name] = create_job_factory(
+            write_input_funct=write_input_funct,
+            collect_output_funct=collect_output_funct,
+            default_input_dict=default_input_dict,
+            executable_str=executable_str,
+        )
+
     def create_job(self, job_type, job_name, delete_existing_job=False):
         """
         Create one of the following jobs:

tests/flex/test_executablecontainer.py

@@ -0,0 +1,82 @@
+import os
+from pyiron_base._tests import TestWithProject
+from pyiron_base.jobs.job.jobtype import JOB_CLASS_DICT
+from pyiron_base import create_job_factory
+from pyiron_base.storage.hdfio import ProjectHDFio
+
+
+def write_input(input_dict, working_directory="."):
+    with open(os.path.join(working_directory, "input_file"), "w") as f:
+        f.write(str(input_dict["energy"]))
+
+
+def collect_output(working_directory="."):
+    with open(os.path.join(working_directory, "output_file"), "r") as f:
+        return {"energy": float(f.readline())}
+
+
+class TestExecutableContainer(TestWithProject):
+    def test_create_job_class(self):
+        energy_value = 2.0
+        self.project.create_job_class(
+            class_name="CatJob",
+            write_input_funct=write_input,
+            collect_output_funct=collect_output,
+            default_input_dict={"energy": 1.0},
+            executable_str="cat input_file > output_file",
+        )
+        job = self.project.create.job.CatJob(job_name="job_test")
+        job.input["energy"] = energy_value
+        job.run()
+        self.assertEqual(job.output["energy"], energy_value)
+        job_reload = self.project.load(job.job_name)
+        self.assertEqual(job_reload.input["energy"], energy_value)
+        self.assertEqual(job_reload.output["energy"], energy_value)
+        del JOB_CLASS_DICT["CatJob"]
+
+    def test_create_job_factory_with_project(self):
+        energy_value = 2.0
+        create_catjob = create_job_factory(
+            write_input_funct=write_input,
+            collect_output_funct=collect_output,
+            default_input_dict={"energy": 1.0},
+            executable_str="cat input_file > output_file",
+        )
+        job = create_catjob(project=self.project, job_name="job_test")
+        job.input["energy"] = energy_value
+        job.run()
+        self.assertEqual(job.output["energy"], energy_value)
+        job_reload = self.project.load(job.job_name)
+        self.assertEqual(job_reload.input["energy"], energy_value)
+        self.assertEqual(job_reload.output["energy"], energy_value)
+
+    def test_create_job_factory_with_projecthdfio(self):
+        energy_value = 2.0
+        create_catjob = create_job_factory(
+            write_input_funct=write_input,
+            collect_output_funct=collect_output,
+            default_input_dict={"energy": 1.0},
+            executable_str="cat input_file > output_file",
+        )
+        job = create_catjob(
+            project=ProjectHDFio(project=self.project, file_name="any.h5", h5_path=None, mode=None),
+            job_name="job_test"
+        )
+        job.input["energy"] = energy_value
+        job.run()
+        self.assertEqual(job.output["energy"], energy_value)
+        job_reload = self.project.load(job.job_name)
+        self.assertEqual(job_reload.input["energy"], energy_value)
+        self.assertEqual(job_reload.output["energy"], energy_value)
+
+    def test_create_job_factory_typeerror(self):
+        create_catjob = create_job_factory(
+            write_input_funct=write_input,
+            collect_output_funct=collect_output,
+            executable_str="cat input_file > output_file",
+        )
+        with self.assertRaises(TypeError):
+            create_catjob(
+                project="project",
+                job_name="job_test"
+            )