♻️ Maintenance: refactors application settings repo-wide (ITISFounda…

…tion#7112)

packages/common-library/src/common_library/basic_types.py

@@ -3,11 +3,12 @@
 
 from pydantic_core import PydanticUndefined
 
-# SEE https://github.com/fastapi/fastapi/blob/master/fastapi/_compat.py#L75-L78
 Undefined = PydanticUndefined
 DEFAULT_FACTORY: Any = Undefined
-# Use `UNSET` as default when default_factory
+# Use `DEFAULT_FACTORY` as field default when using Field(default_factory=...)
 # SEE https://github.com/ITISFoundation/osparc-simcore/pull/6882
+# SEE https://github.com/ITISFoundation/osparc-simcore/pull/7112#discussion_r1933432238
+# SEE https://github.com/fastapi/fastapi/blob/master/fastapi/_compat.py#L75-L78
 
 
 class LogLevel(StrEnum):

packages/settings-library/setup.cfg

@@ -14,6 +14,7 @@ universal = 1
 # Define setup.py command aliases here
 test = pytest
 
-# NOTE: uncomment when pytest-asyncio is added in requirements
-# [tool:pytest]
-# asyncio_mode = auto
+[tool:pytest]
+# SEE https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+filterwarnings =
+    error

packages/settings-library/src/settings_library/application.py

@@ -1,3 +1,5 @@
+from typing import Annotated
+
 from pydantic import Field, PositiveInt
 
 from .base import BaseCustomSettings
@@ -18,11 +20,13 @@ class BaseApplicationSettings(BaseCustomSettings):
     # @Dockerfile
     SC_BOOT_MODE: BootModeEnum | None = None
     SC_BOOT_TARGET: BuildTargetEnum | None = None
-    SC_HEALTHCHECK_TIMEOUT: PositiveInt | None = Field(
-        default=None,
-        description="If a single run of the check takes longer than timeout seconds "
-        "then the check is considered to have failed."
-        "It takes retries consecutive failures of the health check for the container to be considered unhealthy.",
-    )
+    SC_HEALTHCHECK_TIMEOUT: Annotated[
+        PositiveInt | None,
+        Field(
+            description="If a single run of the check takes longer than timeout seconds "
+            "then the check is considered to have failed."
+            "It takes retries consecutive failures of the health check for the container to be considered unhealthy.",
+        ),
+    ] = None
     SC_USER_ID: int | None = None
     SC_USER_NAME: str | None = None

packages/settings-library/src/settings_library/director_v0.py

@@ -1,4 +1,5 @@
 from functools import cached_property
+from typing import Annotated
 
 from pydantic import AnyHttpUrl, Field, TypeAdapter
 from settings_library.base import BaseCustomSettings
@@ -10,9 +11,9 @@ class DirectorV0Settings(BaseCustomSettings):
 
     DIRECTOR_HOST: str = "director"
     DIRECTOR_PORT: PortInt = TypeAdapter(PortInt).validate_python(8000)
-    DIRECTOR_VTAG: VersionTag = Field(
-        default="v0", description="Director-v0 service API's version tag"
-    )
+    DIRECTOR_VTAG: Annotated[
+        VersionTag, Field(description="Director-v0 service API's version tag")
+    ] = "v0"
 
     @cached_property
     def endpoint(self) -> str:

packages/settings-library/src/settings_library/docker_registry.py

@@ -1,5 +1,5 @@
 from functools import cached_property
-from typing import Any, Self
+from typing import Annotated, Any, Self
 
 from pydantic import (
     AnyHttpUrl,
@@ -15,37 +15,45 @@
 
 
 class RegistrySettings(BaseCustomSettings):
-    REGISTRY_AUTH: bool = Field(..., description="do registry authentication")
-    REGISTRY_PATH: str | None = Field(
-        default=None,
-        # This is useful in case of a local registry, where the registry url (path) is relative to the host docker engine"
-        description="development mode only, in case a local registry is used - "
-        "this is the hostname to the docker registry as seen from the host running the containers (e.g. 127.0.0.1:5000)",
-    )
-    # NOTE: name is missleading, http or https protocol are not included
-    REGISTRY_URL: str = Field(
-        ...,
-        description="hostname of docker registry (without protocol but with port if available)",
-        min_length=1,
-    )
+    REGISTRY_AUTH: Annotated[bool, Field(description="do registry authentication")]
+    REGISTRY_PATH: Annotated[
+        str | None,
+        Field(
+            # This is useful in case of a local registry, where the registry url (path) is relative to the host docker engine"
+            description="development mode only, in case a local registry is used - "
+            "this is the hostname to the docker registry as seen from the host running the containers (e.g. 127.0.0.1:5000)",
+        ),
+    ] = None
 
-    REGISTRY_USER: str = Field(
-        ..., description="username to access the docker registry"
-    )
-    REGISTRY_PW: SecretStr = Field(
-        ..., description="password to access the docker registry"
-    )
-    REGISTRY_SSL: bool = Field(
-        ..., description="True if docker registry is using HTTPS protocol"
-    )
+    REGISTRY_URL: Annotated[
+        str,
+        Field(
+            # NOTE: name is missleading, http or https protocol are not included
+            description="hostname of docker registry (without protocol but with port if available)",
+            min_length=1,
+        ),
+    ]
+
+    REGISTRY_USER: Annotated[
+        str,
+        Field(description="username to access the docker registry"),
+    ]
+    REGISTRY_PW: Annotated[
+        SecretStr,
+        Field(description="password to access the docker registry"),
+    ]
+    REGISTRY_SSL: Annotated[
+        bool,
+        Field(description="True if docker registry is using HTTPS protocol"),
+    ]
 
     @field_validator("REGISTRY_PATH", mode="before")
     @classmethod
     def _escape_none_string(cls, v) -> Any | None:
         return None if v == "None" else v
 
     @model_validator(mode="after")
-    def check_registry_authentication(self: Self) -> Self:
+    def _check_registry_authentication(self: Self) -> Self:
         if self.REGISTRY_AUTH and any(
             not v for v in (self.REGISTRY_USER, self.REGISTRY_PW)
         ):

packages/settings-library/src/settings_library/ec2.py

@@ -1,18 +1,19 @@
 from typing import Annotated
 
-from pydantic import AnyHttpUrl, BeforeValidator, Field, TypeAdapter
+from pydantic import BeforeValidator, Field
 from pydantic_settings import SettingsConfigDict
 
 from .base import BaseCustomSettings
-
-ANY_HTTP_URL_ADAPTER: TypeAdapter = TypeAdapter(AnyHttpUrl)
+from .utils_validators import validate_nullable_url
 
 
 class EC2Settings(BaseCustomSettings):
     EC2_ACCESS_KEY_ID: str
     EC2_ENDPOINT: Annotated[
-        str, BeforeValidator(lambda x: str(ANY_HTTP_URL_ADAPTER.validate_python(x)))
-    ] | None = Field(default=None, description="do not define if using standard AWS")
+        str | None,
+        BeforeValidator(validate_nullable_url),
+        Field(description="do not define if using standard AWS"),
+    ] = None
     EC2_REGION_NAME: str = "us-east-1"
     EC2_SECRET_ACCESS_KEY: str
 

packages/settings-library/src/settings_library/efs.py

@@ -1,19 +1,26 @@
 from pathlib import Path
+from typing import Annotated
 
 from pydantic import Field
 
 from .base import BaseCustomSettings
 
 
 class AwsEfsSettings(BaseCustomSettings):
-    EFS_DNS_NAME: str = Field(
-        description="AWS Elastic File System DNS name",
-        examples=["fs-xxx.efs.us-east-1.amazonaws.com"],
-    )
+    EFS_DNS_NAME: Annotated[
+        str,
+        Field(
+            description="AWS Elastic File System DNS name",
+            examples=["fs-xxx.efs.us-east-1.amazonaws.com"],
+        ),
+    ]
     EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: str
-    EFS_MOUNTED_PATH: Path = Field(
-        description="This is the path where EFS is mounted to the EC2 machine",
-    )
+    EFS_MOUNTED_PATH: Annotated[
+        Path,
+        Field(
+            description="This is the path where EFS is mounted to the EC2 machine",
+        ),
+    ]
 
 
 NFS_PROTOCOL = "4.1"

packages/settings-library/src/settings_library/email.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Self
+from typing import Annotated, Self
 
 from pydantic import model_validator
 from pydantic.fields import Field
@@ -25,12 +25,15 @@ class SMTPSettings(BaseCustomSettings):
 
     SMTP_HOST: str
     SMTP_PORT: PortInt
-    SMTP_PROTOCOL: EmailProtocol = Field(
-        EmailProtocol.UNENCRYPTED,
-        description="Select between TLS, STARTTLS Secure Mode or unencrypted communication",
-    )
-    SMTP_USERNAME: str | None = Field(None, min_length=1)
-    SMTP_PASSWORD: SecretStr | None = Field(None, min_length=1)
+    SMTP_PROTOCOL: Annotated[
+        EmailProtocol,
+        Field(
+            description="Select between TLS, STARTTLS Secure Mode or unencrypted communication",
+        ),
+    ] = EmailProtocol.UNENCRYPTED
+
+    SMTP_USERNAME: Annotated[str | None, Field(min_length=1)] = None
+    SMTP_PASSWORD: Annotated[SecretStr | None, Field(min_length=1)] = None
 
     @model_validator(mode="after")
     def _both_credentials_must_be_set(self) -> Self:

packages/settings-library/src/settings_library/http_client_request.py

@@ -1,3 +1,5 @@
+from typing import Annotated
+
 from pydantic import Field
 
 from .base import BaseCustomSettings
@@ -6,26 +8,32 @@
 class ClientRequestSettings(BaseCustomSettings):
     # NOTE: These entries are used in some old services as well. These need to be updated if these
     # variable names or defaults are changed.
-    HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT: int | None = Field(
-        default=20,
-        description="timeout in seconds used for outgoing http requests",
-    )
+    HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT: Annotated[
+        int | None,
+        Field(
+            description="timeout in seconds used for outgoing http requests",
+        ),
+    ] = 20
 
-    HTTP_CLIENT_REQUEST_AIOHTTP_CONNECT_TIMEOUT: int | None = Field(
-        default=None,
-        description=(
-            "Maximal number of seconds for acquiring a connection"
-            " from pool. The time consists connection establishment"
-            " for a new connection or waiting for a free connection"
-            " from a pool if pool connection limits are exceeded. "
-            "For pure socket connection establishment time use sock_connect."
+    HTTP_CLIENT_REQUEST_AIOHTTP_CONNECT_TIMEOUT: Annotated[
+        int | None,
+        Field(
+            description=(
+                "Maximal number of seconds for acquiring a connection"
+                " from pool. The time consists connection establishment"
+                " for a new connection or waiting for a free connection"
+                " from a pool if pool connection limits are exceeded. "
+                "For pure socket connection establishment time use sock_connect."
+            ),
         ),
-    )
+    ] = None
 
-    HTTP_CLIENT_REQUEST_AIOHTTP_SOCK_CONNECT_TIMEOUT: int | None = Field(
-        default=5,
-        description=(
-            "aiohttp specific field used in ClientTimeout, timeout for connecting to a "
-            "peer for a new connection not given a pool"
+    HTTP_CLIENT_REQUEST_AIOHTTP_SOCK_CONNECT_TIMEOUT: Annotated[
+        int | None,
+        Field(
+            description=(
+                "aiohttp specific field used in ClientTimeout, timeout for connecting to a "
+                "peer for a new connection not given a pool"
+            ),
         ),
-    )
+    ] = 5

packages/settings-library/src/settings_library/postgres.py

@@ -1,4 +1,5 @@
 from functools import cached_property
+from typing import Annotated
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
 from pydantic import (
@@ -25,26 +26,28 @@ class PostgresSettings(BaseCustomSettings):
     POSTGRES_PASSWORD: SecretStr
 
     # database
-    POSTGRES_DB: str = Field(..., description="Database name")
+    POSTGRES_DB: Annotated[str, Field(description="Database name")]
 
     # pool connection limits
-    POSTGRES_MINSIZE: int = Field(
-        default=1, description="Minimum number of connections in the pool", ge=1
-    )
-    POSTGRES_MAXSIZE: int = Field(
-        default=50, description="Maximum number of connections in the pool", ge=1
-    )
+    POSTGRES_MINSIZE: Annotated[
+        int, Field(description="Minimum number of connections in the pool", ge=1)
+    ] = 1
+    POSTGRES_MAXSIZE: Annotated[
+        int, Field(description="Maximum number of connections in the pool", ge=1)
+    ] = 50
 
-    POSTGRES_CLIENT_NAME: str | None = Field(
-        default=None,
-        description="Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)",
-        validation_alias=AliasChoices(
-            "POSTGRES_CLIENT_NAME",
-            # This is useful when running inside a docker container, then the hostname is set each client gets a different name
-            "HOST",
-            "HOSTNAME",
+    POSTGRES_CLIENT_NAME: Annotated[
+        str | None,
+        Field(
+            description="Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)",
+            validation_alias=AliasChoices(
+                "POSTGRES_CLIENT_NAME",
+                # This is useful when running inside a docker container, then the hostname is set each client gets a different name
+                "HOST",
+                "HOSTNAME",
+            ),
         ),
-    )
+    ] = None
 
     @field_validator("POSTGRES_MAXSIZE")
     @classmethod

packages/settings-library/src/settings_library/r_clone.py

@@ -1,4 +1,5 @@
 from enum import StrEnum
+from typing import Annotated
 
 from pydantic import Field, NonNegativeInt
 
@@ -13,19 +14,25 @@ class S3Provider(StrEnum):
 
 
 class RCloneSettings(BaseCustomSettings):
-    R_CLONE_S3: S3Settings = Field(json_schema_extra={"auto_default_from_env": True})
+    R_CLONE_S3: Annotated[
+        S3Settings, Field(json_schema_extra={"auto_default_from_env": True})
+    ]
     R_CLONE_PROVIDER: S3Provider
 
-    # SEE https://rclone.org/docs/#transfers-n
-    R_CLONE_OPTION_TRANSFERS: NonNegativeInt = Field(
-        default=5, description="`--transfers X`: sets the amount of parallel transfers"
-    )
-    # SEE https://rclone.org/docs/#retries-int
-    R_CLONE_OPTION_RETRIES: NonNegativeInt = Field(
-        default=3, description="`--retries X`: times to retry each individual transfer"
-    )
-    # SEE https://rclone.org/docs/#buffer-size-size
-    R_CLONE_OPTION_BUFFER_SIZE: str = Field(
-        default="16M",
-        description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
-    )
+    R_CLONE_OPTION_TRANSFERS: Annotated[
+        # SEE https://rclone.org/docs/#transfers-n
+        NonNegativeInt,
+        Field(description="`--transfers X`: sets the amount of parallel transfers"),
+    ] = 5
+    R_CLONE_OPTION_RETRIES: Annotated[
+        # SEE https://rclone.org/docs/#retries-int
+        NonNegativeInt,
+        Field(description="`--retries X`: times to retry each individual transfer"),
+    ] = 3
+    R_CLONE_OPTION_BUFFER_SIZE: Annotated[
+        # SEE https://rclone.org/docs/#buffer-size-size
+        str,
+        Field(
+            description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
+        ),
+    ] = "16M"