Merge pull request #520 from ikondov/mongomock

Optionally use mongomock instead of pymongo/mongodb

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2
 
 jobs:
-  pytest:
+  pytest_mongodb:
     working_directory: ~/fireworks
     docker:
       - image: continuumio/miniconda3:4.6.14
@@ -26,7 +26,37 @@ jobs:
             pip install .[workflow-checks,graph-plotting,flask-plotting]
             pytest fireworks
 
+  pytest_mongomock:
+    working_directory: ~/fireworks
+    docker:
+      - image: continuumio/miniconda3:4.6.14
+    steps:
+      - checkout
+      - run:
+          command: |
+            export PATH=$HOME/miniconda3/bin:$PATH
+            conda config --set always_yes yes --set changeps1 no
+            conda update -q conda
+            conda info -a
+            conda create -q -n test-environment python=3.8
+            source activate test-environment
+            conda update --quiet --all
+            pip install --quiet --ignore-installed -r requirements.txt -r requirements-ci.txt
+      - run:
+          name: Run fireworks tests
+          command: |
+            export PATH=$HOME/miniconda3/bin:$PATH
+            source activate test-environment
+            pip install .[workflow-checks,graph-plotting,flask-plotting,mongomock]
+            server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json
+            echo "{}" > $server_store_file
+            export MONGOMOCK_SERVERSTORE_FILE=$server_store_file
+            pytest -m "not mongodb" fireworks
+            rm $server_store_file
+
 workflows:
   version: 2
   build_and_test:
-    jobs: [pytest]
+    jobs:
+    - pytest_mongodb
+    - pytest_mongomock

.github/workflows/test.yml

@@ -32,3 +32,28 @@ jobs:
 
       - name: Run fireworks tests
         run: pytest fireworks
+
+  pytest_mongomock:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt -r requirements-ci.txt
+          pip install '.[workflow-checks,graph-plotting,flask-plotting,mongomock]'
+
+      - name: Setup mongomock server store and run pytest
+        run: |
+          server_store_file=$PWD/server_store_${RANDOM}-${RANDOM}-${RANDOM}.json
+          echo "{}" > $server_store_file
+          export MONGOMOCK_SERVERSTORE_FILE=$server_store_file
+          pytest -m "not mongodb" fireworks
+          rm -f $server_store_file

docs_rst/config_tutorial.rst

@@ -76,6 +76,7 @@ A few basic parameters that can be tweaked are:
 * ``WEBSERVER_HOST: 127.0.0.1`` - the default host on which to run the web server
 * ``WEBSERVER_PORT: 5000`` - the default port on which to run the web server
 * ``QUEUE_JOBNAME_MAXLEN: 20`` - the max length of the job name to send to the queuing system (some queuing systems limit the size of job names)
+* ``MONGOMOCK_SERVERSTORE_FILE`` - path to a non-empty JSON file, if set then mongomock will be used instead of MongoDB; this file should be initialized with '{}'
 * ``ROCKET_STREAM_LOGLEVEL: INFO`` - the streaming log level of the rocket launcher logger (valid values: DEBUG, INFO, WARNING, ERROR, CRITICAL)
 
 Parameters that you probably shouldn't change

docs_rst/index.rst

@@ -115,6 +115,14 @@ To get a first glimpse of FireWorks, we suggest that you follow our installation
    installation
    quickstart
 
+Quickstart (tutorial mode)
+==========================
+
+.. toctree::
+   :maxdepth: 1
+
+   quickstart_tutorial
+
 Basic usage
 ===========
 

docs_rst/quickstart_tutorial.rst

@@ -0,0 +1,75 @@
+=============================================
+Two-minute installation, setup and quickstart
+=============================================
+
+Install and setup
+=================
+
+Supposed you have a :doc:`virtual environment </virtualenv_tutorial>` with the `pip` package installed. Then simply type::
+
+    pip install fireworks[mongomock]
+    mkdir -p ~/.fireworks
+    echo MONGOMOCK_SERVERSTORE_FILE: $HOME/.fireworks/mongomock.json > ~/.fireworks/FW_config.yaml
+    echo '{}' > ~/.fireworks/mongomock.json
+    lpad reset --password="$(date +%Y-%m-%d)"
+
+See that the database contains no workflows::
+
+    lpad get_wflows
+
+*Output*::
+
+    []
+
+Add and display a workflow
+==========================
+
+Add a script that prints the date as a single firework in a workflow::
+
+    lpad add_scripts 'date' -n date_printer_firework -w date_printer_workflow
+
+Let us display the workflow just added::
+
+    lpad get_wflows -d more
+
+*Output*::
+
+    {
+        "state": "READY",
+        "name": "date_printer_workflow--1",
+        "created_on": "2024-06-07T15:05:02.096000",
+        "updated_on": "2024-06-07T15:05:02.096000",
+        "states": {
+            "date_printer_firework--1": "READY"
+        },
+        "launch_dirs": {
+            "date_printer_firework--1": []
+        }
+    }
+
+We have only one workflow with only one firework on the database.
+
+Run a workflow
+==============
+
+Now we can run the firework in our workflow locally with this simple command::
+
+    rlaunch singleshot
+
+*Output*::
+
+    2024-06-07 17:15:08,515 INFO Hostname/IP lookup (this will take a few seconds)
+    2024-06-07 17:15:08,517 INFO Launching Rocket
+    2024-06-07 17:15:08,608 INFO RUNNING fw_id: 1 in directory: /home/ubuntu
+    2024-06-07 17:15:08,610 INFO Task started: ScriptTask.
+    Fri Jun  7 17:15:08 CEST 2024
+    2024-06-07 17:15:08,612 INFO Task completed: ScriptTask
+    2024-06-07 17:15:08,616 INFO Rocket finished
+
+Further steps
+=============
+
+This setup uses a JSON file on the local computer as a database instead of MongoDB. You can continue with the other tutorials
+and do local testing by using this setting. If you want to complete the more advanced tutorials, such as the
+:doc:`queue tutorial </queue_tutorial>`, or use FireWorks productively on a computing cluster, then you should consider
+:doc:`installing and setting up FireWorks </installation>` with a MongoDB server.

fireworks/core/launchpad.py

@@ -15,11 +15,12 @@
 from bson import ObjectId
 from monty.os.path import zpath
 from monty.serialization import loadfn
-from pymongo import ASCENDING, DESCENDING, MongoClient
+from pymongo import ASCENDING, DESCENDING
 from pymongo.errors import DocumentTooLarge
 from tqdm import tqdm
 
 from fireworks.core.firework import Firework, FWAction, Launch, Tracker, Workflow
+from fireworks.fw_config import MongoClient
 from fireworks.fw_config import (
     GRIDFS_FALLBACK_COLLECTION,
     LAUNCHPAD_LOC,
@@ -413,7 +414,7 @@ def bulk_add_wfs(self, wfs) -> None:
 
         """
         # Make all fireworks workflows
-        wfs = [Workflow.from_firework(wf) if isinstance(wf, Firework) else wf for wf in wfs]
+        wfs = [Workflow.from_Firework(wf) if isinstance(wf, Firework) else wf for wf in wfs]
 
         # Initialize new firework counter, starting from the next fw id
         total_num_fws = sum(len(wf) for wf in wfs)

fireworks/core/tests/test_launchpad.py

@@ -15,7 +15,6 @@
 
 import pytest
 from monty.os import cd
-from pymongo import MongoClient
 from pymongo import __version__ as PYMONGO_VERSION
 from pymongo.errors import OperationFailure
 
@@ -43,7 +42,7 @@ class AuthenticationTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         try:
-            client = MongoClient()
+            client = fireworks.fw_config.MongoClient()
             client.not_the_admin_db.command("createUser", "myuser", pwd="mypassword", roles=["dbOwner"])
         except Exception:
             raise unittest.SkipTest("MongoDB is not running in localhost:27017! Skipping tests.")
@@ -1310,7 +1309,7 @@ def test_recover_errors(self) -> None:
 
         assert fw.state == "FIZZLED"
 
-
+@pytest.mark.mongodb
 class GridfsStoredDataTest(unittest.TestCase):
     """
     Tests concerning the storage of data in Gridfs when the size of the
@@ -1344,6 +1343,7 @@ def tearDown(self) -> None:
         for ldir in glob.glob(os.path.join(MODULE_DIR, "launcher_*")):
             shutil.rmtree(ldir)
 
+
     def test_many_detours(self) -> None:
         task = DetoursTask(n_detours=2000, data_per_detour=["a" * 100] * 100)
         fw = Firework([task])

fireworks/fw_config.py

@@ -3,10 +3,12 @@
 from __future__ import annotations
 
 import os
+import importlib
 from typing import Any
 
 from monty.design_patterns import singleton
 from monty.serialization import dumpfn, loadfn
+import pymongo
 
 __author__ = "Anubhav Jain"
 __copyright__ = "Copyright 2012, The Materials Project"
@@ -104,6 +106,12 @@
 # a dynamically generated document exceeds the 16MB limit. Functionality disabled if None.
 GRIDFS_FALLBACK_COLLECTION = "fw_gridfs"
 
+# path to a database file to use with mongomock, do not use mongomock if None
+MONGOMOCK_SERVERSTORE_FILE = None
+
+# default mongoclient class
+MongoClient = pymongo.MongoClient
+
 
 def override_user_settings() -> None:
     module_dir = os.path.dirname(os.path.abspath(__file__))
@@ -155,6 +163,22 @@ def override_user_settings() -> None:
             if len(m_paths) > 0:
                 globals()[k] = m_paths[0]
 
+    if 'MONGOMOCK_SERVERSTORE_FILE' in os.environ:
+        globals()['MONGOMOCK_SERVERSTORE_FILE'] = os.environ['MONGOMOCK_SERVERSTORE_FILE']
+    if globals()['MONGOMOCK_SERVERSTORE_FILE']:
+        try:
+            mongomock_persistence = importlib.import_module('mongomock_persistence')
+            mongomock_gridfs = importlib.import_module('mongomock.gridfs')
+        except (ModuleNotFoundError, ImportError) as err:
+            msg = ('\nTo use mongomock instead of mongodb the extra mongomock must'
+                   ' be installed, for example like this:\npip install fireworks[mongomock]')
+            raise RuntimeError(msg) from err
+        if not os.environ.get('MONGOMOCK_SERVERSTORE_FILE'):
+            os.environ['MONGOMOCK_SERVERSTORE_FILE'] = globals()['MONGOMOCK_SERVERSTORE_FILE']
+        globals()['MongoClient'] = getattr(mongomock_persistence, 'MongoClient')
+        if globals()['GRIDFS_FALLBACK_COLLECTION']:
+            mongomock_gridfs.enable_gridfs_integration()
+
 
 override_user_settings()
 

fireworks/scripts/tests/test_lpad_run.py

@@ -19,6 +19,7 @@ def lp(capsys):
         lp.reset(password=None, require_password=False)
 
 
+@pytest.mark.mongodb
 @pytest.mark.parametrize(("detail", "expected_1", "expected_2"), [("count", "0\n", "1\n"), ("ids", "[]\n", "1\n")])
 def test_lpad_get_fws(capsys, lp, detail, expected_1, expected_2) -> None:
     """Test lpad CLI get_fws command."""

fireworks/user_objects/firetasks/filepad_tasks.py

@@ -1,7 +1,11 @@
 import os
-
+import json
+from glob import glob
+from pymongo import DESCENDING
+from ruamel.yaml import YAML
 from fireworks.core.firework import FiretaskBase
 from fireworks.utilities.filepad import FilePad
+from fireworks.utilities.dict_mods import arrow_to_dot
 
 __author__ = "Kiran Mathew, Johannes Hoermann"
 __email__ = "kmathew@lbl.gov, johannes.hoermann@imtek.uni-freiburg.de"
@@ -28,7 +32,6 @@ class AddFilesTask(FiretaskBase):
     optional_params = ["identifiers", "directory", "filepad_file", "compress", "metadata"]
 
     def run_task(self, fw_spec) -> None:
-        from glob import glob
 
         directory = os.path.abspath(self.get("directory", "."))
 
@@ -143,19 +146,12 @@ class GetFilesByQueryTask(FiretaskBase):
     ]
 
     def run_task(self, fw_spec) -> None:
-        import json
-
-        import pymongo
-        from ruamel.yaml import YAML
-
-        from fireworks.utilities.dict_mods import arrow_to_dot
-
         fpad = get_fpad(self.get("filepad_file", None))
         dest_dir = self.get("dest_dir", os.path.abspath("."))
         new_file_names = self.get("new_file_names", [])
         query = self.get("query", {})
         sort_key = self.get("sort_key", None)
-        sort_direction = self.get("sort_direction", pymongo.DESCENDING)
+        sort_direction = self.get("sort_direction", DESCENDING)
         limit = self.get("limit", None)
         fizzle_empty_result = self.get("fizzle_empty_result", True)
         fizzle_degenerate_file_name = self.get("fizzle_degenerate_file_name", True)