diff --git a/.env-devel b/.env-devel index 6a32129a920..17f909698c2 100644 --- a/.env-devel +++ b/.env-devel @@ -211,7 +211,7 @@ RESOURCE_USAGE_TRACKER_S3=null RESOURCE_USAGE_TRACKER_TRACING={} # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface. -R_CLONE_OPTION_BUFFER_SIZE=0M +R_CLONE_OPTION_BUFFER_SIZE=16M R_CLONE_OPTION_RETRIES=3 R_CLONE_OPTION_TRANSFERS=5 R_CLONE_PROVIDER=MINIO diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py index c4288466928..062022be5aa 100644 --- a/packages/settings-library/src/settings_library/r_clone.py +++ b/packages/settings-library/src/settings_library/r_clone.py @@ -26,6 +26,6 @@ class RCloneSettings(BaseCustomSettings): ) # SEE https://rclone.org/docs/#buffer-size-size R_CLONE_OPTION_BUFFER_SIZE: str = Field( - default="0M", + default="16M", description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer", ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py index bbfe14e7f39..0283a11fe26 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py @@ -8,10 +8,9 @@ from pathlib import Path from typing import Final -from common_library.errors_classes import OsparcErrorMixin - from aiocache import cached # type: ignore[import-untyped] from aiofiles import tempfile +from common_library.errors_classes import OsparcErrorMixin from models_library.basic_types import IDStr from pydantic import AnyUrl, BaseModel, ByteSize from servicelib.progress_bar import ProgressBarData @@ -207,7 +206,6 @@ async def _sync_sources( f"{r_clone_settings.R_CLONE_OPTION_TRANSFERS}", # below two options reduce to a minimum the memory footprint # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4 - "--use-mmap", # docs https://rclone.org/docs/#use-mmap "--buffer-size", # docs https://rclone.org/docs/#buffer-size-size r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE, "--use-json-log", diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py index c32e9f75888..c4ef5e83e2a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py @@ -52,14 +52,14 @@ class SyncProgressLogParser(BaseLogParser): This command: - rclone --use-mmap --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress + rclone --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress generates this but the rclone modifies the terminal printed lines which python does not like so much Transferred: 4.666 GiB / 4.666 GiB, 100%, 530.870 MiB/s, ETA 0s Transferred: 4 / 4, 100% Elapsed time: 9.6s This other command: - rclone --use-mmap --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 + rclone --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 prints stuff such as: {"level":"info","msg":"Copied (new)","object":"README.ipynb","objectType":"*s3.Object","size":5123,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408277+00:00"} {"level":"info","msg":"Copied (new)","object":".hidden_do_not_remove","objectType":"*s3.Object","size":219,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408246+00:00"} diff --git a/services/agent/src/simcore_service_agent/services/backup.py b/services/agent/src/simcore_service_agent/services/backup.py index fb6de148eef..a7e125af0c4 100644 --- a/services/agent/src/simcore_service_agent/services/backup.py +++ b/services/agent/src/simcore_service_agent/services/backup.py @@ -161,7 +161,6 @@ async def _store_in_s3( f"{settings.AGENT_VOLUMES_CLEANUP_PARALLELISM}", # below two options reduce to a minimum the memory footprint # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4 - "--use-mmap", # docs https://rclone.org/docs/#use-mmap "--buffer-size", # docs https://rclone.org/docs/#buffer-size-size "0M", "--stats", diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index dfe79b57cad..02493e4c86a 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -63,7 +63,7 @@ S3_BUCKET_NAME=simcore R_CLONE_PROVIDER=MINIO R_CLONE_OPTION_TRANSFERS=5 R_CLONE_OPTION_RETRIES=3 -R_CLONE_OPTION_BUFFER_SIZE=0M +R_CLONE_OPTION_BUFFER_SIZE=16M TRACING_OBSERVABILITY_BACKEND_ENDPOINT=http://jaeger:9411 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 6f39b8a3e24..52e0c6f4be0 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -286,7 +286,7 @@ def expected_dynamic_sidecar_spec( "RABBIT_PORT": "5672", "RABBIT_USER": "admin", "RABBIT_SECURE": "False", - "R_CLONE_OPTION_BUFFER_SIZE": "0M", + "R_CLONE_OPTION_BUFFER_SIZE": "16M", "R_CLONE_OPTION_RETRIES": "3", "R_CLONE_OPTION_TRANSFERS": "5", "R_CLONE_PROVIDER": "MINIO",