diff --git a/packages/models-library/src/models_library/user_preferences.py b/packages/models-library/src/models_library/user_preferences.py index 0d912c1af19..435fd972cb5 100644 --- a/packages/models-library/src/models_library/user_preferences.py +++ b/packages/models-library/src/models_library/user_preferences.py @@ -98,6 +98,7 @@ def to_db(self) -> dict: @classmethod def update_preference_default_value(cls, new_default: Any) -> None: + # pylint: disable=unsubscriptable-object expected_type = get_type(cls.model_fields["value"]) detected_type = type(new_default) if expected_type != detected_type: diff --git a/packages/models-library/src/models_library/utils/labels_annotations.py b/packages/models-library/src/models_library/utils/labels_annotations.py index 26c7d7d73f5..4bb93347c63 100644 --- a/packages/models-library/src/models_library/utils/labels_annotations.py +++ b/packages/models-library/src/models_library/utils/labels_annotations.py @@ -10,7 +10,7 @@ from common_library.json_serialization import json_dumps -LabelsAnnotationsDict: TypeAlias = dict[str, str] +LabelsAnnotationsDict: TypeAlias = dict[str, str | float | bool | None] # SEE https://docs.docker.com/config/labels-custom-metadata/#label-keys-and-values # "Authors of third-party tools should prefix each label key with the reverse DNS notation of a @@ -28,7 +28,7 @@ def to_labels( """converts config into labels annotations""" # FIXME: null is loaded as 'null' string value? is that correct? json -> None upon deserialization? - labels = {} + labels: LabelsAnnotationsDict = {} for key, value in config.items(): if trim_key_head: if isinstance(value, str): @@ -57,7 +57,7 @@ def from_labels( for key, label in labels.items(): if key.startswith(f"{prefix_key}."): try: - value = json.loads(label) + value = json.loads(label) # type: ignore except JSONDecodeError: value = label diff --git a/packages/models-library/tests/test_rest_ordering.py b/packages/models-library/tests/test_rest_ordering.py index b2e6e0765b5..f827ca18582 100644 --- a/packages/models-library/tests/test_rest_ordering.py +++ b/packages/models-library/tests/test_rest_ordering.py @@ -197,7 +197,9 @@ def test_ordering_query_model_class__defaults(): model = OrderQueryParamsModel.model_validate({"order_by": {"field": "name"}}) assert model.order_by assert model.order_by.field == "name" - assert model.order_by.direction == OrderBy.model_fields["direction"].default + assert ( # pylint: disable=unsubscriptable-object + model.order_by.direction == OrderBy.model_fields["direction"].default + ) # direction alone is invalid with pytest.raises(ValidationError) as err_info: diff --git a/packages/service-integration/requirements/prod.txt b/packages/service-integration/requirements/prod.txt index deb20913395..8cebc4d6898 100644 --- a/packages/service-integration/requirements/prod.txt +++ b/packages/service-integration/requirements/prod.txt @@ -9,6 +9,7 @@ # installs base + tests requirements --requirement _base.txt +simcore-common-library @ ../common-library/ simcore-models-library @ ../models-library # current module diff --git a/packages/service-integration/src/service_integration/cli/_compose_spec.py b/packages/service-integration/src/service_integration/cli/_compose_spec.py index afccc0e268e..806b9013fa4 100644 --- a/packages/service-integration/src/service_integration/cli/_compose_spec.py +++ b/packages/service-integration/src/service_integration/cli/_compose_spec.py @@ -6,7 +6,7 @@ import rich import typer import yaml -from models_library.utils.labels_annotations import to_labels +from models_library.utils.labels_annotations import LabelsAnnotationsDict, to_labels from rich.console import Console from yarl import URL @@ -93,7 +93,7 @@ def create_docker_compose_image_spec( rich.print("No runtime config found (optional), using default.") # OCI annotations (optional) - extra_labels = {} + extra_labels: LabelsAnnotationsDict = {} try: oci_spec = yaml.safe_load( (config_basedir / f"{OCI_LABEL_PREFIX}.yml").read_text() diff --git a/packages/service-integration/src/service_integration/cli/_config.py b/packages/service-integration/src/service_integration/cli/_config.py index 4437907efa0..7ae9c65f689 100644 --- a/packages/service-integration/src/service_integration/cli/_config.py +++ b/packages/service-integration/src/service_integration/cli/_config.py @@ -5,6 +5,7 @@ import rich import typer import yaml +from models_library.utils.labels_annotations import LabelsAnnotationsDict from pydantic import BaseModel from ..compose_spec_model import ComposeSpecification @@ -20,7 +21,7 @@ ) -def _get_labels_or_raise(build_labels) -> dict[str, str]: +def _get_labels_or_raise(build_labels) -> LabelsAnnotationsDict: if isinstance(build_labels, list): return dict(item.strip().split("=") for item in build_labels) if isinstance(build_labels, dict): @@ -56,7 +57,9 @@ def _save(service_name: str, filename: Path, model: BaseModel): rich.print(f"Creating {output_path} ...", end="") with output_path.open("wt") as fh: - data = json.loads(model.model_dump_json(by_alias=True, exclude_none=True)) + data = json.loads( + model.model_dump_json(by_alias=True, exclude_none=True) + ) yaml.safe_dump(data, fh, sort_keys=False) rich.print("DONE") @@ -68,7 +71,7 @@ def _save(service_name: str, filename: Path, model: BaseModel): service_name ].build.labels: # AttributeError if build is str - labels: dict[str, str] = _get_labels_or_raise(build_labels) + labels = _get_labels_or_raise(build_labels) meta_cfg = MetadataConfig.from_labels_annotations(labels) _save(service_name, metadata_path, meta_cfg) @@ -86,11 +89,7 @@ def _save(service_name: str, filename: Path, model: BaseModel): runtime_cfg = RuntimeConfig.from_labels_annotations(labels) _save(service_name, service_specs_path, runtime_cfg) - except ( # noqa: PERF203 - AttributeError, - TypeError, - ValueError, - ) as err: + except (AttributeError, TypeError, ValueError) as err: rich.print( f"WARNING: failure producing specs for {service_name}: {err}" ) diff --git a/packages/service-integration/src/service_integration/compose_spec_model.py b/packages/service-integration/src/service_integration/compose_spec_model.py index 3b45e73ddad..5ee4ce6f29a 100644 --- a/packages/service-integration/src/service_integration/compose_spec_model.py +++ b/packages/service-integration/src/service_integration/compose_spec_model.py @@ -10,6 +10,7 @@ from ._compose_spec_model_autogenerated import ( # type:ignore BuildItem, ComposeSpecification, + ListOrDict, Service, Volume1, ) @@ -23,6 +24,7 @@ __all__: tuple[str, ...] = ( "BuildItem", "ComposeSpecification", + "ListOrDict", "SCHEMA_VERSION", "Service", "ServiceVolume", diff --git a/packages/service-integration/src/service_integration/oci_image_spec.py b/packages/service-integration/src/service_integration/oci_image_spec.py index 3b3ea7ffe8a..3c61a09ec13 100644 --- a/packages/service-integration/src/service_integration/oci_image_spec.py +++ b/packages/service-integration/src/service_integration/oci_image_spec.py @@ -10,7 +10,11 @@ from typing import Annotated, Any from models_library.basic_types import SHA1Str, VersionStr -from models_library.utils.labels_annotations import from_labels, to_labels +from models_library.utils.labels_annotations import ( + LabelsAnnotationsDict, + from_labels, + to_labels, +) from pydantic import BaseModel, ConfigDict, Field from pydantic.networks import AnyUrl @@ -132,17 +136,16 @@ class OciImageSpecAnnotations(BaseModel): @classmethod def from_labels_annotations( - cls, labels: dict[str, str] + cls, labels: LabelsAnnotationsDict ) -> "OciImageSpecAnnotations": data = from_labels(labels, prefix_key=OCI_LABEL_PREFIX, trim_key_head=False) return cls.model_validate(data) - def to_labels_annotations(self) -> dict[str, str]: - labels: dict[str, str] = to_labels( + def to_labels_annotations(self) -> LabelsAnnotationsDict: + return to_labels( self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OCI_LABEL_PREFIX, ) - return labels class LabelSchemaAnnotations(BaseModel): @@ -164,7 +167,7 @@ class LabelSchemaAnnotations(BaseModel): @classmethod def create_from_env(cls) -> "LabelSchemaAnnotations": data = {} - for field_name in cls.model_fields: + for field_name in cls.model_fields: # pylint: disable=not-an-iterable if value := os.environ.get(field_name.upper()): data[field_name] = value return cls.model_validate(data) diff --git a/packages/service-integration/src/service_integration/osparc_config.py b/packages/service-integration/src/service_integration/osparc_config.py index dbc97b1c8e5..9a3f2e0c116 100644 --- a/packages/service-integration/src/service_integration/osparc_config.py +++ b/packages/service-integration/src/service_integration/osparc_config.py @@ -34,12 +34,14 @@ from models_library.services_types import ServiceKey from models_library.utils.labels_annotations import ( OSPARC_LABEL_PREFIXES, + LabelsAnnotationsDict, from_labels, to_labels, ) from pydantic import ( ConfigDict, NonNegativeInt, + TypeAdapter, ValidationError, ValidationInfo, field_validator, @@ -121,24 +123,21 @@ def _check_contact_in_authors(cls, v, info: ValidationInfo): def from_yaml(cls, path: Path) -> "MetadataConfig": with path.open() as fh: data = yaml_safe_load(fh) - model: "MetadataConfig" = cls.model_validate(data) - return model + return cls.model_validate(data) @classmethod - def from_labels_annotations(cls, labels: dict[str, str]) -> "MetadataConfig": + def from_labels_annotations(cls, labels: LabelsAnnotationsDict) -> "MetadataConfig": data = from_labels( labels, prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False ) - model: "MetadataConfig" = cls.model_validate(data) - return model + return cls.model_validate(data) - def to_labels_annotations(self) -> dict[str, str]: - labels: dict[str, str] = to_labels( + def to_labels_annotations(self) -> LabelsAnnotationsDict: + return to_labels( self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OSPARC_LABEL_PREFIXES[0], trim_key_head=False, ) - return labels def service_name(self) -> str: """name used as key in the compose-spec services map""" @@ -151,7 +150,9 @@ def image_name(self, settings: AppSettings, registry="local") -> str: if registry in "dockerhub": # dockerhub allows only one-level names -> dot it # TODO: check thisname is compatible with REGEX - service_path = ServiceKey(service_path.replace("/", ".")) + service_path = TypeAdapter(ServiceKey).validate_python( + service_path.replace("/", ".") + ) service_version = self.version return f"{registry_prefix}{service_path}:{service_version}" @@ -264,13 +265,12 @@ def from_yaml(cls, path: Path) -> "RuntimeConfig": return cls.model_validate(data) @classmethod - def from_labels_annotations(cls, labels: dict[str, str]) -> "RuntimeConfig": + def from_labels_annotations(cls, labels: LabelsAnnotationsDict) -> "RuntimeConfig": data = from_labels(labels, prefix_key=OSPARC_LABEL_PREFIXES[1]) return cls.model_validate(data) - def to_labels_annotations(self) -> dict[str, str]: - labels: dict[str, str] = to_labels( + def to_labels_annotations(self) -> LabelsAnnotationsDict: + return to_labels( self.model_dump(exclude_unset=True, by_alias=True, exclude_none=True), prefix_key=OSPARC_LABEL_PREFIXES[1], ) - return labels diff --git a/packages/service-integration/src/service_integration/osparc_image_specs.py b/packages/service-integration/src/service_integration/osparc_image_specs.py index 7f6dec6ca15..a94d6dfc1fc 100644 --- a/packages/service-integration/src/service_integration/osparc_image_specs.py +++ b/packages/service-integration/src/service_integration/osparc_image_specs.py @@ -2,10 +2,13 @@ """ +from typing import Any +from models_library.utils.labels_annotations import LabelsAnnotationsDict from service_integration.compose_spec_model import ( BuildItem, ComposeSpecification, + ListOrDict, Service, ) @@ -19,14 +22,14 @@ def create_image_spec( docker_compose_overwrite_cfg: DockerComposeOverwriteConfig, runtime_cfg: RuntimeConfig | None = None, *, - extra_labels: dict[str, str] | None = None, + extra_labels: LabelsAnnotationsDict | None = None, **_context ) -> ComposeSpecification: """Creates the image-spec provided the osparc-config and a given context (e.g. development) - the image-spec simplifies building an image to ``docker compose build`` """ - labels = {**meta_cfg.to_labels_annotations()} + labels = meta_cfg.to_labels_annotations() if extra_labels: labels.update(extra_labels) if runtime_cfg: @@ -36,19 +39,26 @@ def create_image_spec( assert docker_compose_overwrite_cfg.services # nosec - if not docker_compose_overwrite_cfg.services[service_name].build.context: - docker_compose_overwrite_cfg.services[service_name].build.context = "./" + build = docker_compose_overwrite_cfg.services[service_name].build + assert isinstance(build, BuildItem) # nosec + if not build.context: + build.context = "./" - docker_compose_overwrite_cfg.services[service_name].build.labels = labels + build.labels = ListOrDict(root=labels) - overwrite_options = docker_compose_overwrite_cfg.services[ - service_name - ].build.model_dump(exclude_none=True, serialize_as_any=True) + overwrite_options = build.model_dump(exclude_none=True, serialize_as_any=True) build_spec = BuildItem(**overwrite_options) + service_kwargs: dict[str, Any] = { + "image": meta_cfg.image_name(settings), + "build": build_spec, + } + if docker_compose_overwrite_cfg.services[service_name].depends_on: + service_kwargs["depends_on"] = docker_compose_overwrite_cfg.services[ + service_name + ].depends_on + return ComposeSpecification( version=settings.COMPOSE_VERSION, - services={ - service_name: Service(image=meta_cfg.image_name(settings), build=build_spec) - }, + services={service_name: Service(**service_kwargs)}, ) diff --git a/packages/service-integration/tests/data/docker-compose.overwrite.yml b/packages/service-integration/tests/data/docker-compose.overwrite.yml index 2444dbef082..b55b5120790 100644 --- a/packages/service-integration/tests/data/docker-compose.overwrite.yml +++ b/packages/service-integration/tests/data/docker-compose.overwrite.yml @@ -1,4 +1,6 @@ services: osparc-python-runner: + depends_on: + - another-service build: dockerfile: Dockerfile diff --git a/packages/service-integration/tests/test_command_compose.py b/packages/service-integration/tests/test_command_compose.py index 50f8b5b67b4..b3aa3cd78d7 100644 --- a/packages/service-integration/tests/test_command_compose.py +++ b/packages/service-integration/tests/test_command_compose.py @@ -3,14 +3,22 @@ # pylint: disable=unused-variable import os +import traceback from collections.abc import Callable from pathlib import Path import yaml +from click.testing import Result from service_integration.compose_spec_model import ComposeSpecification from service_integration.osparc_config import MetadataConfig +def _format_cli_error(result: Result) -> str: + assert result.exception + tb_message = "\n".join(traceback.format_tb(result.exception.__traceback__)) + return f"Below exception was raised by the cli:\n{tb_message}" + + def test_make_docker_compose_meta( run_program_with_args: Callable, docker_compose_overwrite_path: Path, @@ -33,7 +41,7 @@ def test_make_docker_compose_meta( "--to-spec-file", target_compose_specs, ) - assert result.exit_code == os.EX_OK, result.output + assert result.exit_code == os.EX_OK, _format_cli_error(result) # produces a compose spec assert target_compose_specs.exists() @@ -50,6 +58,4 @@ def test_make_docker_compose_meta( assert compose_labels assert isinstance(compose_labels.root, dict) - assert ( - MetadataConfig.from_labels_annotations(compose_labels.root) == metadata_cfg - ) + assert MetadataConfig.from_labels_annotations(compose_labels.root) == metadata_cfg