Dev/make kafka optional

.github/workflows/testing.yml

@@ -36,8 +36,7 @@ jobs:
         # installation problems if out of date.
         python -m pip install --upgrade pip setuptools numpy
 
-        pip install .
-        pip install -r requirements-dev.txt
+        pip install .[all]
         pip install "pydantic${{ matrix.pydantic-version }}"
         pip install bluesky==1.11.0
 

bluesky_queueserver/manager/start_manager.py

@@ -1,5 +1,6 @@
 import argparse
 import atexit
+import importlib
 import logging
 import os
 import re
@@ -627,6 +628,13 @@ def formatter(prog):
             "Use --zmq-info-addr instead."
         )
 
+    if settings.kafka_topic is not None or settings.kafka_server is not None:
+        if importlib.util.find_spec("bluesky-kafka") is None:
+            logger.error(
+                "Must have bluesky-kafka installed to publish documents to kafka",
+            )
+        sys.exit(1)
+
     msg_queue = Queue()
     setup_console_output_redirection(msg_queue)
 

bluesky_queueserver/manager/worker.py

@@ -1394,7 +1394,6 @@ def _worker_startup_code(self):
         from bluesky import RunEngine
         from bluesky.callbacks.best_effort import BestEffortCallback
         from bluesky.utils import PersistentDict
-        from bluesky_kafka import Publisher as kafkaPublisher
 
         from .profile_tools import global_user_namespace
 
@@ -1511,6 +1510,8 @@ def factory(name, doc):
                             self._RE.subscribe(self._db.insert)
 
                 if "kafka" in self._config_dict:
+                    from bluesky_kafka import Publisher as kafkaPublisher
+
                     logger.info(
                         "Subscribing to Kafka: topic '%s', servers '%s'",
                         self._config_dict["kafka"]["topic"],

docs/source/installation.rst

@@ -42,6 +42,10 @@ Installation Steps
 
     $ pip install bluesky-queueserver
 
+  Install from PyPI with kafka dependencies::
+
+    $ pip install bluesky-queueserver[kafka]
+
   Install from conda-forge::
 
     $ conda install bluesky-queueserver -c conda-forge

requirements-dev.txt

@@ -25,5 +25,4 @@ fastapi[all]
 # moved from requirements.txt
 pandas
 pyarrow
-matplotlib # Needed for BEC, should be factored out
 scikit-image # Needed for BEC/other stuff to run, factor out

requirements.txt

@@ -1,5 +1,4 @@
 bluesky>=1.7.0
-bluesky-kafka
 databroker
 event-model
 ipykernel

setup.py

@@ -34,6 +34,16 @@
     # Parse requirements.txt, ignoring any commented-out lines.
     requirements = [line for line in requirements_file.read().splitlines() if not line.startswith("#")]
 
+with open(path.join(here, "requirements-dev.txt")) as requirements_file:
+    # Parse requirements.txt, ignoring any commented-out lines.
+    requirements_dev = [line for line in requirements_file.read().splitlines() if not line.startswith("#")]
+
+requirements_kafka = ["bluesky-kafka"]
+
+extras_require = {"dev": sorted(set(requirements_dev)), "kafka": requirements_kafka}
+extras_require["all"] = sorted(set(sum(extras_require.values(), [])))
+
+
 setup(
     name="bluesky-queueserver",
     version=versioneer.get_version(),
@@ -58,6 +68,7 @@
             "qserver-console-monitor = bluesky_queueserver.manager.output_streaming:qserver_console_monitor_cli",
         ],
     },
+    extras_require=extras_require,
     include_package_data=True,
     package_data={
         "bluesky_queueserver": [