root: expose CONN_MAX_AGE, CONN_HEALTH_CHECKS and DISABLE_SERVER_SIDE…

…_CURSORS for PostgreSQL config (cherry-pick #10159) (#12419)

root: expose CONN_MAX_AGE, CONN_HEALTH_CHECKS and DISABLE_SERVER_SIDE_CURSORS for PostgreSQL config (#10159)

Co-authored-by: Tomás Farías Santana <tomas@tomasfarias.dev>
Co-authored-by: Marc 'risson' Schmitt <marc.schmitt@risson.space>
Co-authored-by: Tana M Berry <tana@goauthentik.com>

authentik/lib/config.py

@@ -280,9 +280,24 @@ def get_int(self, path: str, default=0) -> int:
             self.log("warning", "Failed to parse config as int", path=path, exc=str(exc))
             return default
 
+    def get_optional_int(self, path: str, default=None) -> int | None:
+        """Wrapper for get that converts value into int or None if set"""
+        value = self.get(path, default)
+
+        try:
+            return int(value)
+        except (ValueError, TypeError) as exc:
+            if value is None or (isinstance(value, str) and value.lower() == "null"):
+                return None
+            self.log("warning", "Failed to parse config as int", path=path, exc=str(exc))
+            return default
+
     def get_bool(self, path: str, default=False) -> bool:
         """Wrapper for get that converts value into boolean"""
-        return str(self.get(path, default)).lower() == "true"
+        value = self.get(path, UNSET)
+        if value is UNSET:
+            return default
+        return str(self.get(path)).lower() == "true"
 
     def get_keys(self, path: str, sep=".") -> list[str]:
         """List attribute keys by using yaml path"""
@@ -354,20 +369,33 @@ def django_db_config(config: ConfigLoader | None = None) -> dict:
                 "sslcert": config.get("postgresql.sslcert"),
                 "sslkey": config.get("postgresql.sslkey"),
             },
+            "CONN_MAX_AGE": CONFIG.get_optional_int("postgresql.conn_max_age", 0),
+            "CONN_HEALTH_CHECKS": CONFIG.get_bool("postgresql.conn_health_checks", False),
+            "DISABLE_SERVER_SIDE_CURSORS": CONFIG.get_bool(
+                "postgresql.disable_server_side_cursors", False
+            ),
             "TEST": {
                 "NAME": config.get("postgresql.test.name"),
             },
         }
     }
 
+    conn_max_age = CONFIG.get_optional_int("postgresql.conn_max_age", UNSET)
+    disable_server_side_cursors = CONFIG.get_bool("postgresql.disable_server_side_cursors", UNSET)
     if config.get_bool("postgresql.use_pgpool", False):
         db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True
+        if disable_server_side_cursors is not UNSET:
+            db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = disable_server_side_cursors
 
     if config.get_bool("postgresql.use_pgbouncer", False):
         # https://docs.djangoproject.com/en/4.0/ref/databases/#transaction-pooling-server-side-cursors
         db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True
         # https://docs.djangoproject.com/en/4.0/ref/databases/#persistent-connections
         db["default"]["CONN_MAX_AGE"] = None  # persistent
+        if disable_server_side_cursors is not UNSET:
+            db["default"]["DISABLE_SERVER_SIDE_CURSORS"] = disable_server_side_cursors
+        if conn_max_age is not UNSET:
+            db["default"]["CONN_MAX_AGE"] = conn_max_age
 
     for replica in config.get_keys("postgresql.read_replicas"):
         _database = deepcopy(db["default"])

authentik/lib/default.yml

@@ -6,8 +6,6 @@ postgresql:
   user: authentik
   port: 5432
   password: "env://POSTGRES_PASSWORD"
-  use_pgbouncer: false
-  use_pgpool: false
   test:
     name: test_authentik
   read_replicas: {}

authentik/lib/tests/test_config.py

@@ -214,6 +214,9 @@ def test_db_default(self):
                     "PORT": "foo",
                     "TEST": {"NAME": "foo"},
                     "USER": "foo",
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
+                    "DISABLE_SERVER_SIDE_CURSORS": False,
                 }
             },
         )
@@ -251,6 +254,9 @@ def test_db_read_replicas(self):
                     "PORT": "foo",
                     "TEST": {"NAME": "foo"},
                     "USER": "foo",
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
+                    "DISABLE_SERVER_SIDE_CURSORS": False,
                 },
                 "replica_0": {
                     "ENGINE": "authentik.root.db",
@@ -266,6 +272,72 @@ def test_db_read_replicas(self):
                     "PORT": "foo",
                     "TEST": {"NAME": "foo"},
                     "USER": "foo",
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
+                    "DISABLE_SERVER_SIDE_CURSORS": False,
+                },
+            },
+        )
+
+    def test_db_read_replicas_pgbouncer(self):
+        """Test read replicas"""
+        config = ConfigLoader()
+        config.set("postgresql.host", "foo")
+        config.set("postgresql.name", "foo")
+        config.set("postgresql.user", "foo")
+        config.set("postgresql.password", "foo")
+        config.set("postgresql.port", "foo")
+        config.set("postgresql.sslmode", "foo")
+        config.set("postgresql.sslrootcert", "foo")
+        config.set("postgresql.sslcert", "foo")
+        config.set("postgresql.sslkey", "foo")
+        config.set("postgresql.test.name", "foo")
+        config.set("postgresql.use_pgbouncer", True)
+        # Read replica
+        config.set("postgresql.read_replicas.0.host", "bar")
+        # Override conn_max_age
+        config.set("postgresql.read_replicas.0.conn_max_age", 10)
+        # This isn't supported
+        config.set("postgresql.read_replicas.0.use_pgbouncer", False)
+        conf = django_db_config(config)
+        self.assertEqual(
+            conf,
+            {
+                "default": {
+                    "DISABLE_SERVER_SIDE_CURSORS": True,
+                    "CONN_MAX_AGE": None,
+                    "CONN_HEALTH_CHECKS": False,
+                    "ENGINE": "authentik.root.db",
+                    "HOST": "foo",
+                    "NAME": "foo",
+                    "OPTIONS": {
+                        "sslcert": "foo",
+                        "sslkey": "foo",
+                        "sslmode": "foo",
+                        "sslrootcert": "foo",
+                    },
+                    "PASSWORD": "foo",
+                    "PORT": "foo",
+                    "TEST": {"NAME": "foo"},
+                    "USER": "foo",
+                },
+                "replica_0": {
+                    "DISABLE_SERVER_SIDE_CURSORS": True,
+                    "CONN_MAX_AGE": 10,
+                    "CONN_HEALTH_CHECKS": False,
+                    "ENGINE": "authentik.root.db",
+                    "HOST": "bar",
+                    "NAME": "foo",
+                    "OPTIONS": {
+                        "sslcert": "foo",
+                        "sslkey": "foo",
+                        "sslmode": "foo",
+                        "sslrootcert": "foo",
+                    },
+                    "PASSWORD": "foo",
+                    "PORT": "foo",
+                    "TEST": {"NAME": "foo"},
+                    "USER": "foo",
                 },
             },
         )
@@ -294,6 +366,8 @@ def test_db_read_replicas_pgpool(self):
             {
                 "default": {
                     "DISABLE_SERVER_SIDE_CURSORS": True,
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
                     "ENGINE": "authentik.root.db",
                     "HOST": "foo",
                     "NAME": "foo",
@@ -310,6 +384,8 @@ def test_db_read_replicas_pgpool(self):
                 },
                 "replica_0": {
                     "DISABLE_SERVER_SIDE_CURSORS": True,
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
                     "ENGINE": "authentik.root.db",
                     "HOST": "bar",
                     "NAME": "foo",
@@ -362,6 +438,9 @@ def test_db_read_replicas_diff_ssl(self):
                     "PORT": "foo",
                     "TEST": {"NAME": "foo"},
                     "USER": "foo",
+                    "DISABLE_SERVER_SIDE_CURSORS": False,
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
                 },
                 "replica_0": {
                     "ENGINE": "authentik.root.db",
@@ -377,6 +456,9 @@ def test_db_read_replicas_diff_ssl(self):
                     "PORT": "foo",
                     "TEST": {"NAME": "foo"},
                     "USER": "foo",
+                    "DISABLE_SERVER_SIDE_CURSORS": False,
+                    "CONN_MAX_AGE": 0,
+                    "CONN_HEALTH_CHECKS": False,
                 },
             },
         )

website/docs/install-config/configuration/configuration.mdx

@@ -70,14 +70,17 @@ To check if your config has been applied correctly, you can run the following co
 - `AUTHENTIK_POSTGRESQL__USER`: Database user
 - `AUTHENTIK_POSTGRESQL__PORT`: Database port, defaults to 5432
 - `AUTHENTIK_POSTGRESQL__PASSWORD`: Database password, defaults to the environment variable `POSTGRES_PASSWORD`
-- `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER`: Adjust configuration to support connection to PgBouncer
-- `AUTHENTIK_POSTGRESQL__USE_PGPOOL`: Adjust configuration to support connection to Pgpool
+- `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER`: Adjust configuration to support connection to PgBouncer. Deprecated, see below
+- `AUTHENTIK_POSTGRESQL__USE_PGPOOL`: Adjust configuration to support connection to Pgpool. Deprecated, see below
 - `AUTHENTIK_POSTGRESQL__SSLMODE`: Strictness of ssl verification. Defaults to `"verify-ca"`
 - `AUTHENTIK_POSTGRESQL__SSLROOTCERT`: CA root for server ssl verification
 - `AUTHENTIK_POSTGRESQL__SSLCERT`: Path to x509 client certificate to authenticate to server
 - `AUTHENTIK_POSTGRESQL__SSLKEY`: Path to private key of `SSLCERT` certificate
+- `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE`: Database connection lifetime. Defaults to `0` (no persistent connections). Can be set to `null` for unlimited persistent connections. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#conn-max-age) for more details.
+- `AUTHENTIK_POSTGRESQL__CONN_HEALTH_CHECK`: Existing persistent database connections will be health checked before they are reused if set to `true`. Defaults to `false`. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#conn-health-checks) for more details.
+- `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS`: Disable server side cursors when set to `true`. Defaults to `false`. See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/settings/#disable-server-side-cursors) for more details.
 
-All PostgreSQL settings, apart from `USE_PGBOUNCER` and `USE_PGPOOL`, support hot-reloading. Adding and removing read replicas doesn't support hot-reloading.
+The PostgreSQL settings `HOST`, `PORT`, `USER`, and `PASSWORD` support hot-reloading. Adding and removing read replicas doesn't support hot-reloading.
 
 ### Read replicas
 
@@ -96,8 +99,25 @@ The same PostgreSQL settings as described above are used for each read replica.
 - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLROOTCERT`
 - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLCERT`
 - `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__SSLKEY`
+- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__CONN_MAX_AGE`
+- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__CONN_HEALTH_CHECK`
+- `AUTHENTIK_POSTGRESQL__READ_REPLICAS__0__DISABLE_SERVER_SIDE_CURSORS`
 
-Note that `USE_PGBOUNCER` and `USE_PGPOOL` are inherited from the main database configuration and are _not_ overridable on read replicas.
+### Using a PostgreSQL connection pooler (PgBouncer or PgPool)
+
+When your PostgreSQL database(s) are running behind a connection pooler, like PgBouncer or PgPool, two settings need to be overridden:
+
+- `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE`
+
+    A connection pooler running in session pool mode (PgBouncer default) can be incompatible with unlimited persistent connections enabled by setting this to `null`: If the connection from the connection pooler to the database server is dropped, the connection pooler will wait for the client to disconnect before releasing the connection; however this will **never** happen as authentik is configured to keep the connection to the connection pooler forever.
+
+    To address this incompatibility, either configure the connection pooler to run in transaction pool mode, or update this setting to a value lower than any timeouts that may cause the connection to the database to be dropped (up to `0`).
+
+- `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS`
+
+    Using a connection pooler in transaction pool mode (e.g. PgPool, or PgBouncer in transaction or statement pool mode) requires disabling server-side cursors, so this setting must be set to `false`.
+
+Additionally, you can set `AUTHENTIK_POSTGRESQL__CONN_HEALTH_CHECK` to perform health checks on persistent database connections before they are re-used.
 
 ## Redis Settings
 

website/docs/releases/2024/v2024.12.md

@@ -24,6 +24,16 @@ To try out the release candidate, replace your Docker image tag with the latest
 
     You can disable this behavior in the **Admin interface** under **System** > **Settings**.
 
+- **Deprecated PostgreSQL `USE_PGBOUNCER` and `USE_PGPOOL` settings**
+
+    With this release, the `AUTHENTIK_POSTGRESQL__USE_PGBOUNCER` and `AUTHENTIK_POSTGRESQL__USE_PGPOOL` settings have been deprecated in favor of exposing the underlying database settings: `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE` and `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS`.
+
+    If you are using PgBouncer or PgPool as connection poolers and wish to maintain the same behavior as previous versions, `AUTHENTIK_POSTGRESQL__DISABLE_SERVER_SIDE_CURSORS` must be set to `true`. Moreover, if you are using PgBouncer `AUTHENTIK_POSTGRESQL__CONN_MAX_AGE` must be set to `null`.
+
+    The newly exposed settings allow supporting a wider set of connection pooler configurations. For details on how these settings interact with different configurations of connection poolers, please refer to the [PostgreSQL documentation](../../install-config/configuration/configuration.mdx#postgresql-settings).
+
+    These settings will be removed in a future version.
+
 ## New features
 
 - **Redirect stage**