From 771aede8ae6a8f129caffbac1abd420f55574ba8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 31 Jan 2024 17:43:39 +0530 Subject: [PATCH 1/5] add a test image build method to DockerWorkerConfig --- .../syft/src/syft/custom_worker/config.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 319273cc1a4..4493261e37f 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -4,6 +4,7 @@ from typing import Any from typing import Dict from typing import List +from typing import Literal from typing import Optional from typing import Union @@ -15,7 +16,9 @@ # relative from ..serde.serializable import serializable +from ..service.response import SyftError from ..types.base import SyftBaseModel +from .builder_types import ImageBuildResult PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -136,3 +139,29 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text + + def test_image_build( + self, orchestration_type: Literal["docker", "k8s"], tag: str, **build_args + ) -> Union[ImageBuildResult, SyftError]: + # relative + from .builder_docker import DockerBuilder + from .builder_k8s import KubernetesBuilder + + builder = ( + KubernetesBuilder() if orchestration_type == "k8s" else DockerBuilder() + ) + + # TODO: Remove this check once we know how test with k8s + if orchestration_type == "k8s": + return SyftError( + message="We currently support test builds using `docker` only." + ) + + try: + return builder.build_image( + tag=tag, + dockerfile=self.dockerfile, + buildargs=build_args, + ) + except Exception as e: + return SyftError(message=f"Failed to build: {e}") From cd7a9b692544aebe98aaa07aa61e055ddf5c72fc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 5 Feb 2024 18:55:14 +0530 Subject: [PATCH 2/5] update test build config to work with only docker --- .../syft/src/syft/custom_worker/config.py | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index ccb516e5c59..fa49d5b1c8b 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -1,14 +1,18 @@ # stdlib +import contextlib from hashlib import sha256 +import io +import json from pathlib import Path from typing import Any from typing import Dict +from typing import Iterable from typing import List -from typing import Literal from typing import Optional from typing import Union # third party +import docker from packaging import version from pydantic import validator from typing_extensions import Self @@ -17,8 +21,8 @@ # relative from ..serde.serializable import serializable from ..service.response import SyftError +from ..service.response import SyftSuccess from ..types.base import SyftBaseModel -from .builder_types import ImageBuildResult PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -163,28 +167,33 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text - def test_image_build( - self, orchestration_type: Literal["docker", "k8s"], tag: str, **build_args - ) -> Union[ImageBuildResult, SyftError]: - # relative - from .builder_docker import DockerBuilder - from .builder_k8s import KubernetesBuilder - - builder = ( - KubernetesBuilder() if orchestration_type == "k8s" else DockerBuilder() - ) - - # TODO: Remove this check once we know how test with k8s - if orchestration_type == "k8s": - return SyftError( - message="We currently support test builds using `docker` only." - ) - + def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: try: - return builder.build_image( - tag=tag, - dockerfile=self.dockerfile, - buildargs=build_args, - ) + with contextlib.closing(docker.from_env()) as client: + if not client.ping(): + return SyftError( + "Cannot reach docker server. Please check if docker is running." + ) + + kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) + _, logs = client.images.build( + tag=tag, + timeout=self.BUILD_MAX_WAIT, + **kwargs, + ) + return SyftSuccess(message=self._parse_output(logs)) except Exception as e: return SyftError(message=f"Failed to build: {e}") + + @staticmethod + def _parse_output(log_iterator: Iterable) -> str: + log = "" + for line in log_iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log From cb79f7cfedbd0f01528c5276fa2c553a4179b759 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 5 Feb 2024 19:08:19 +0530 Subject: [PATCH 3/5] [custom_worker] make a reusable function to convert iteratable to string [custom_worker] reuse the function to parse docker logs --- .../src/syft/custom_worker/builder_docker.py | 13 ++----------- packages/syft/src/syft/custom_worker/config.py | 18 ++---------------- packages/syft/src/syft/custom_worker/utils.py | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 27 deletions(-) create mode 100644 packages/syft/src/syft/custom_worker/utils.py diff --git a/packages/syft/src/syft/custom_worker/builder_docker.py b/packages/syft/src/syft/custom_worker/builder_docker.py index e1f24520c25..9446a5530a4 100644 --- a/packages/syft/src/syft/custom_worker/builder_docker.py +++ b/packages/syft/src/syft/custom_worker/builder_docker.py @@ -1,7 +1,6 @@ # stdlib import contextlib import io -import json from pathlib import Path from typing import Iterable from typing import Optional @@ -13,6 +12,7 @@ from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult +from .utils import iterator_to_string __all__ = ["DockerBuilder"] @@ -70,13 +70,4 @@ def push_image( return ImagePushResult(logs=result, exit_code=0) def _parse_output(self, log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log + return iterator_to_string(iterator=log_iterator) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index fa49d5b1c8b..1b55e34e021 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -2,11 +2,9 @@ import contextlib from hashlib import sha256 import io -import json from pathlib import Path from typing import Any from typing import Dict -from typing import Iterable from typing import List from typing import Optional from typing import Union @@ -23,6 +21,7 @@ from ..service.response import SyftError from ..service.response import SyftSuccess from ..types.base import SyftBaseModel +from .utils import iterator_to_string PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -181,19 +180,6 @@ def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: timeout=self.BUILD_MAX_WAIT, **kwargs, ) - return SyftSuccess(message=self._parse_output(logs)) + return SyftSuccess(message=iterator_to_string(iterator=logs)) except Exception as e: return SyftError(message=f"Failed to build: {e}") - - @staticmethod - def _parse_output(log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py new file mode 100644 index 00000000000..be6d3cb5915 --- /dev/null +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -0,0 +1,16 @@ +# stdlib +import json +from typing import Iterable + + +def iterator_to_string(iterator: Iterable) -> str: + log = "" + for line in iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log From d38797182cc358ad24bc442b0eb97313681d6ee6 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 7 Feb 2024 16:44:50 +0530 Subject: [PATCH 4/5] add retry to tests in request_multiple_nodes_test.py --- packages/syft/tests/syft/request/request_multiple_nodes_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/syft/tests/syft/request/request_multiple_nodes_test.py b/packages/syft/tests/syft/request/request_multiple_nodes_test.py index 4beb780cc31..9ec214ea7fe 100644 --- a/packages/syft/tests/syft/request/request_multiple_nodes_test.py +++ b/packages/syft/tests/syft/request/request_multiple_nodes_test.py @@ -110,6 +110,7 @@ def dataset_2(client_do_2): return client_do_2.datasets[0].assets[0] +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_blocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): @@ -147,6 +148,7 @@ def compute_sum(data) -> float: assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_nonblocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): From 1a7714ee59862289c422a4033cdde7a3c2815df4 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 7 Feb 2024 17:15:02 +0530 Subject: [PATCH 5/5] add image test build method in notebooks --- notebooks/api/0.8/10-container-images.ipynb | 24 ++++++++++++++++++- .../syft/src/syft/custom_worker/config.py | 1 - 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb index 81ba6a05d13..35f26e791ff 100644 --- a/notebooks/api/0.8/10-container-images.ipynb +++ b/notebooks/api/0.8/10-container-images.ipynb @@ -133,6 +133,28 @@ "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "62762ceb-38da-46f1-acac-cdf5bbf29513", + "metadata": {}, + "outputs": [], + "source": [ + "# test image build locally\n", + "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n", + "test_build_res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0235e567-c65c-48fe-825d-79ea3e219166", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1406,7 +1428,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 1b55e34e021..c54d4f77c40 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -177,7 +177,6 @@ def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) _, logs = client.images.build( tag=tag, - timeout=self.BUILD_MAX_WAIT, **kwargs, ) return SyftSuccess(message=iterator_to_string(iterator=logs))