Merge pull request #1562 from fractal-analytics-platform/996-update-f…

…rom-pyscopg2+asyncpg-to-psycopg3

Update to `psycopg3`

.github/workflows/benchmarks.yaml

@@ -43,11 +43,11 @@ jobs:
         cache: "poetry"
 
      - name: Install dependencies
-       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres
+       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres-psycopg-binary
 
      - name: Run Fractal with Gunicorn
        run: |
-        export DB_ENGINE=postgres
+        export DB_ENGINE=postgres-psycopg
         export POSTGRES_DB=fractal_test
         export POSTGRES_PASSWORD=postgres
         export POSTGRES_USER=postgres
@@ -127,7 +127,7 @@ jobs:
         cache: "poetry"
 
      - name: Install dependencies
-       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres
+       run: poetry install --with dev --without docs --no-interaction -E gunicorn -E postgres-psycopg-binary
 
      - name: Benchmark
        run: |

.github/workflows/ci.yml

@@ -16,13 +16,13 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9", "3.10"]
-        db: ["sqlite", "postgres"]
+        db: ["sqlite", "postgres", "postgres-psycopg"]
 
     services:
       postgres:
         # Disable postgres service when using sqlite, through the workaround
         # described in https://github.com/actions/runner/issues/822
-        image: ${{ (matrix.db == 'postgres') && 'postgres' || '' }}
+        image: ${{ (matrix.db != 'sqlite') && 'postgres' || '' }}
         env:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: fractal_test
@@ -57,6 +57,8 @@ jobs:
         run: |
           if [[ ${{ matrix.db }} == "postgres" ]]; then
             DB="-E postgres"
+          elif [[ ${{ matrix.db }} == "postgres-psycopg" ]]; then
+            DB="-E postgres-psycopg-binary"
           fi
           poetry install --with dev --without docs --no-interaction -E gunicorn $DB
 
@@ -78,13 +80,13 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9", "3.10"]
-        db: ["sqlite", "postgres"]
+        db: ["sqlite", "postgres", "postgres-psycopg"]
 
     services:
       postgres:
         # Disable postgres service when using sqlite, through the workaround
         # described in https://github.com/actions/runner/issues/822
-        image: ${{ (matrix.db == 'postgres') && 'postgres' || '' }}
+        image: ${{ (matrix.db != 'sqlite') && 'postgres' || '' }}
         env:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: fractal_test
@@ -119,6 +121,8 @@ jobs:
         run: |
           if [[ ${{ matrix.db }} == "postgres" ]]; then
             DB="-E postgres"
+          elif [[ ${{ matrix.db }} == "postgres-psycopg" ]]; then
+            DB="-E postgres-psycopg-binary"
           fi
           poetry install --with dev --without docs --no-interaction -E gunicorn $DB
 

CHANGELOG.md

@@ -25,8 +25,11 @@
     * Add custom gunicorn/uvicorn worker to handle SIGABRT signal (\#1526).
     * Move `FractalGunicornLogger` and `FractalWorker` in `fractal_server/gunicorn_fractal.py` (\#1535).
     * Store list of submitted jobs in app state (\#1538).
+* Database:
+    * Add new Postgres adapter `psycopg` (\#1562).
 * Dependencies
     * Add `fabric` to `dev` dependencies (\#1518).
+    * Add new `postgres-psycopg-binary` extra (\#1562).
 * Testing:
     * Extract `pytest-docker` fixtures into a dedicated module (\#1516).
     * Rename SLURM containers in CI (\#1516).

fractal_server/app/db/__init__.py

@@ -63,7 +63,7 @@ def set_async_db(cls):
             }
 
         cls._engine_async = create_async_engine(
-            settings.DATABASE_URL,
+            settings.DATABASE_ASYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
             **engine_kwargs_async,

fractal_server/config.py

@@ -166,7 +166,7 @@ def collect_oauth_clients(cls, values):
     ###########################################################################
     # DATABASE
     ###########################################################################
-    DB_ENGINE: Literal["sqlite", "postgres"] = "sqlite"
+    DB_ENGINE: Literal["sqlite", "postgres", "postgres-psycopg"] = "sqlite"
     """
     Select which database engine to use (supported: `sqlite` and `postgres`).
     """
@@ -201,39 +201,51 @@ def collect_oauth_clients(cls, values):
     """
 
     @property
-    def DATABASE_URL(self) -> URL:
-        if self.DB_ENGINE == "sqlite":
-            if not self.SQLITE_PATH:
-                raise FractalConfigurationError(
-                    "SQLITE_PATH path cannot be None"
-                )
-            sqlite_path = abspath(self.SQLITE_PATH)
+    def DATABASE_ASYNC_URL(self) -> URL:
+        if self.DB_ENGINE == "postgres":
             url = URL.create(
-                drivername="sqlite+aiosqlite",
-                database=sqlite_path,
+                drivername="postgresql+asyncpg",
+                username=self.POSTGRES_USER,
+                password=self.POSTGRES_PASSWORD,
+                host=self.POSTGRES_HOST,
+                port=self.POSTGRES_PORT,
+                database=self.POSTGRES_DB,
             )
-            return url
-        elif "postgres":
+        elif self.DB_ENGINE == "postgres-psycopg":
             url = URL.create(
-                drivername="postgresql+asyncpg",
+                drivername="postgresql+psycopg",
                 username=self.POSTGRES_USER,
                 password=self.POSTGRES_PASSWORD,
                 host=self.POSTGRES_HOST,
                 port=self.POSTGRES_PORT,
                 database=self.POSTGRES_DB,
             )
-            return url
+        else:
+            if not self.SQLITE_PATH:
+                raise FractalConfigurationError(
+                    "SQLITE_PATH path cannot be None"
+                )
+            sqlite_path = abspath(self.SQLITE_PATH)
+            url = URL.create(
+                drivername="sqlite+aiosqlite",
+                database=sqlite_path,
+            )
+        return url
 
     @property
     def DATABASE_SYNC_URL(self):
-        if self.DB_ENGINE == "sqlite":
+        if self.DB_ENGINE == "postgres":
+            return self.DATABASE_ASYNC_URL.set(
+                drivername="postgresql+psycopg2"
+            )
+        elif self.DB_ENGINE == "postgres-psycopg":
+            return self.DATABASE_ASYNC_URL.set(drivername="postgresql+psycopg")
+        else:
             if not self.SQLITE_PATH:
                 raise FractalConfigurationError(
                     "SQLITE_PATH path cannot be None"
                 )
-            return self.DATABASE_URL.set(drivername="sqlite")
-        elif self.DB_ENGINE == "postgres":
-            return self.DATABASE_URL.set(drivername="postgresql+psycopg2")
+            return self.DATABASE_ASYNC_URL.set(drivername="sqlite")
 
     ###########################################################################
     # FRACTAL SPECIFIC
@@ -420,6 +432,14 @@ def check_db(self) -> None:
                     "DB engine is `postgres` but `psycopg2` or `asyncpg` "
                     "are not available"
                 )
+        elif self.DB_ENGINE == "postgres-psycopg":
+            try:
+                import psycopg  # noqa: F401
+            except ModuleNotFoundError:
+                raise FractalConfigurationError(
+                    "DB engine is `postgres-psycopg` but `psycopg` is not "
+                    "available"
+                )
         else:
             if not self.SQLITE_PATH:
                 raise FractalConfigurationError(

fractal_server/migrations/env.py

@@ -54,7 +54,7 @@ def run_migrations_offline() -> None:
     settings = Inject(get_settings)
     settings.check_db()
     context.configure(
-        url=settings.DATABASE_URL,
+        url=settings.DATABASE_ASYNC_URL,
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},