Merge remote-tracking branch 'refs/remotes/origin/2224-review-email-c…

…onfiguration' into 2224-review-email-configuration

CHANGELOG.md

@@ -1,5 +1,18 @@
 **Note**: Numbers like (\#1234) point to closed Pull Requests on the fractal-server repository.
 
+# 2.12.1
+
+* API:
+    * Deprecate `use_dataset_filters` query parameter for `/project/{project_id}/dataset/{dataset_id}/images/query/` (\#2231).
+* App:
+    * Add fractal-server version to logs (\#2228).
+* Database:
+    * Remove `run_migrations_offline` from `env.py` and make `run_migrations_online` sync (\#2239).
+* Runner
+    * Sudo/SLURM executor checks the fractal-server version using `FRACTAL_SLURM_WORKER_PYTHON` config variable, if set (\#2240).
+* Dependencies:
+    * Bump `uvicorn` version (\#2242).
+
 # 2.12.0
 
 > WARNING: The database schema update introduced via this version is non-reversible.
@@ -14,6 +27,7 @@
     * Drop V1 tests (\#2230).
     *  Update V2 tests to keep coverage stable (\#2230).
 
+
 # 2.11.1
 
 * Database

fractal_server/__main__.py

@@ -99,6 +99,11 @@ def set_db(skip_init_data: bool = False):
     from pathlib import Path
     import fractal_server
 
+    # Check settings
+    settings = Inject(get_settings)
+    settings.check_db()
+
+    # Perform migrations
     alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
     alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
     print(f"START: Run alembic.config, with argv={alembic_args}")
@@ -108,12 +113,10 @@ def set_db(skip_init_data: bool = False):
     if skip_init_data:
         return
 
-    # Insert default group
+    # Create default group and user
     print()
     _create_first_group()
     print()
-    # NOTE: It will be fixed with #1739
-    settings = Inject(get_settings)
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,

fractal_server/app/routes/api/v2/images.py

@@ -118,7 +118,6 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    use_dataset_filters: bool = False,  # query param
     page: int = 1,  # query param
     page_size: Optional[int] = None,  # query param
     query: Optional[ImageQuery] = None,  # body
@@ -138,17 +137,6 @@ async def query_dataset_images(
     dataset = output["dataset"]
     images = dataset.images
 
-    if use_dataset_filters is True:
-        images = [
-            image
-            for image in images
-            if match_filter(
-                image=image,
-                type_filters=dataset.type_filters,
-                attribute_filters=dataset.attribute_filters,
-            )
-        ]
-
     attributes = {}
     for image in images:
         for k, v in image["attributes"].items():

fractal_server/app/runner/executors/slurm/sudo/executor.py

@@ -10,6 +10,7 @@
 #
 # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
 # University of Zurich
+import json
 import math
 import shlex
 import subprocess  # nosec
@@ -258,6 +259,14 @@ def __init__(
         self.slurm_account = slurm_account
 
         self.common_script_lines = common_script_lines or []
+        settings = Inject(get_settings)
+
+        if settings.FRACTAL_SLURM_WORKER_PYTHON is not None:
+            try:
+                self.check_remote_python_interpreter()
+            except Exception as e:
+                self._stop_and_join_wait_thread()
+                raise RuntimeError(f"Original error {str(e)}")
 
         # Check that SLURM account is not set here
         try:
@@ -289,7 +298,6 @@ def __init__(
         # Set the attribute slurm_poll_interval for self.wait_thread (see
         # cfut.SlurmWaitThread)
         if not slurm_poll_interval:
-            settings = Inject(get_settings)
             slurm_poll_interval = settings.FRACTAL_SLURM_POLL_INTERVAL
         self.wait_thread.slurm_poll_interval = slurm_poll_interval
         self.wait_thread.slurm_user = self.slurm_user
@@ -1243,3 +1251,29 @@ def __exit__(self, *args, **kwargs):
         )
         self._stop_and_join_wait_thread()
         logger.debug("[FractalSlurmExecutor.__exit__] End")
+
+    def check_remote_python_interpreter(self):
+        """
+        Check fractal-server version on the _remote_ Python interpreter.
+        """
+        settings = Inject(get_settings)
+        output = _subprocess_run_or_raise(
+            (
+                f"{settings.FRACTAL_SLURM_WORKER_PYTHON} "
+                "-m fractal_server.app.runner.versions"
+            )
+        )
+        runner_version = json.loads(output.stdout.strip("\n"))[
+            "fractal_server"
+        ]
+
+        if runner_version != __VERSION__:
+            error_msg = (
+                "Fractal-server version mismatch.\n"
+                "Local interpreter: "
+                f"({sys.executable}): {__VERSION__}.\n"
+                "Remote interpreter: "
+                f"({settings.FRACTAL_SLURM_WORKER_PYTHON}): {runner_version}."
+            )
+            logger.error(error_msg)
+            raise ValueError(error_msg)

fractal_server/main.py

@@ -28,6 +28,7 @@
 from .logger import reset_logger_handlers
 from .logger import set_logger
 from .syringe import Inject
+from fractal_server import __VERSION__
 
 
 def collect_routers(app: FastAPI) -> None:
@@ -77,7 +78,7 @@ def check_settings() -> None:
 async def lifespan(app: FastAPI):
     app.state.jobsV2 = []
     logger = set_logger("fractal_server.lifespan")
-    logger.info("Start application startup")
+    logger.info(f"[startup] START (fractal-server {__VERSION__})")
     check_settings()
     settings = Inject(get_settings)
 
@@ -88,31 +89,31 @@ async def lifespan(app: FastAPI):
         app.state.fractal_ssh_list = FractalSSHList()
 
         logger.info(
-            "Added empty FractalSSHList to app.state "
+            "[startup] Added empty FractalSSHList to app.state "
             f"(id={id(app.state.fractal_ssh_list)})."
         )
     else:
         app.state.fractal_ssh_list = None
 
     config_uvicorn_loggers()
-    logger.info("End application startup")
+    logger.info("[startup] END")
     reset_logger_handlers(logger)
 
     yield
 
     logger = get_logger("fractal_server.lifespan")
-    logger.info("Start application shutdown")
+    logger.info("[teardown] START")
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         logger.info(
-            "Close FractalSSH connections "
+            "[teardown] Close FractalSSH connections "
             f"(current size: {app.state.fractal_ssh_list.size})."
         )
 
         app.state.fractal_ssh_list.close_all()
 
     logger.info(
-        f"Current worker with pid {os.getpid()} is shutting down. "
+        f"[teardown] Current worker with pid {os.getpid()} is shutting down. "
         f"Current jobs: {app.state.jobsV2=}"
     )
     if _backend_supports_shutdown(settings.FRACTAL_RUNNER_BACKEND):
@@ -128,9 +129,11 @@ async def lifespan(app: FastAPI):
                 f"Original error: {e}"
             )
     else:
-        logger.info("Shutdown not available for this backend runner.")
+        logger.info(
+            "[teardown] Shutdown not available for this backend runner."
+        )
 
-    logger.info("End application shutdown")
+    logger.info("[teardown] END")
     reset_logger_handlers(logger)
 
 

fractal_server/migrations/env.py

@@ -1,16 +1,11 @@
-import asyncio
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy.engine import Connection
 from sqlmodel import SQLModel
 
-from fractal_server.config import get_settings
 from fractal_server.migrations.naming_convention import NAMING_CONVENTION
-from fractal_server.syringe import Inject
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
+# Alembic Config object (provides access to the values within the .ini file)
 config = context.config
 
 
@@ -20,77 +15,35 @@
     fileConfig(config.config_file_name)
 
 
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
 target_metadata = SQLModel.metadata
 target_metadata.naming_convention = NAMING_CONVENTION
-# Importing `fractal_server.app.models` after defining
+# Importing `fractal_server.app.models` *after* defining
 # `SQLModel.metadata.naming_convention` in order to apply the naming convention
 # when autogenerating migrations (see issue #1819).
 from fractal_server.app import models  # noqa
 
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
 
+def run_migrations_online() -> None:
     """
-    settings = Inject(get_settings)
-    settings.check_db()
-    context.configure(
-        url=settings.DATABASE_ASYNC_URL,
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
-        render_as_batch=True,
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def do_run_migrations(connection: Connection) -> None:
-    context.configure(
-        connection=connection,
-        target_metadata=target_metadata,
-        render_as_batch=True,
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-async def run_migrations_online() -> None:
-    """Run migrations in 'online' mode.
+    Run migrations in 'online' mode.
 
     In this scenario we need to create an Engine
     and associate a connection with the context.
-
     """
     from fractal_server.app.db import DB
 
-    engine = DB.engine_async()
-    async with engine.connect() as connection:
-        await connection.run_sync(do_run_migrations)
+    engine = DB.engine_sync()
+    with engine.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
 
-    await engine.dispose()
+    engine.dispose()
 
 
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    asyncio.run(run_migrations_online())
+run_migrations_online()

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "sqlalchemy[asyncio] >=2.0.23,<2.1",
     "fastapi-users[oauth] >=14,<15",
     "alembic >=1.13.1, <2.0.0",
-    "uvicorn == 0.29.0",
+    "uvicorn >= 0.29.0, <0.35.0",
     "pydantic >=1.10.8,<2",
     "packaging >= 23.2.0, <24.0.0",
     "clusterfutures == 0.5",