Skip to content

Commit

Permalink
removed type IDStr form description
Browse files Browse the repository at this point in the history
  • Loading branch information
Andrei Neagu committed Jan 30, 2025
1 parent 60132fe commit d27914b
Showing 22 changed files with 73 additions and 109 deletions.
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@
from typing import Final

import tqdm
from models_library.basic_types import IDStr
from pydantic import NonNegativeInt
from servicelib.logging_utils import log_catch
from tqdm.contrib.logging import tqdm_logging_redirect
@@ -199,7 +198,7 @@ async def archive_dir(
) -> None:
if progress_bar is None:
progress_bar = ProgressBarData(
num_steps=1, description=IDStr(f"compressing {dir_to_compress.name}")
num_steps=1, description=f"compressing {dir_to_compress.name}"
)

options = " ".join(
@@ -223,7 +222,7 @@ async def archive_dir(

async with AsyncExitStack() as exit_stack:
sub_progress = await exit_stack.enter_async_context(
progress_bar.sub_progress(folder_size_bytes, description=IDStr("..."))
progress_bar.sub_progress(folder_size_bytes, description="...")
)

tqdm_progress = exit_stack.enter_context(
@@ -290,7 +289,7 @@ async def unarchive_dir(
) -> set[Path]:
if progress_bar is None:
progress_bar = ProgressBarData(
num_steps=1, description=IDStr(f"extracting {archive_to_extract.name}")
num_steps=1, description=f"extracting {archive_to_extract.name}"
)

# get archive information
@@ -304,7 +303,7 @@ async def unarchive_dir(

async with AsyncExitStack() as exit_stack:
sub_prog = await exit_stack.enter_async_context(
progress_bar.sub_progress(steps=total_bytes, description=IDStr("..."))
progress_bar.sub_progress(steps=total_bytes, description="...")
)

tqdm_progress = exit_stack.enter_context(
Original file line number Diff line number Diff line change
@@ -3,7 +3,6 @@
from typing import Final

import httpx
from models_library.basic_types import IDStr
from models_library.docker import DockerGenericTag
from pydantic import ByteSize, TypeAdapter, ValidationError
from settings_library.docker_registry import RegistrySettings
@@ -129,9 +128,7 @@ async def pull_images(
num_steps=images_total_size,
progress_report_cb=progress_cb,
progress_unit="Byte",
description=IDStr(
f"pulling {len(images)} images",
),
description=f"pulling {len(images)} images",
) as pbar:

await asyncio.gather(
5 changes: 2 additions & 3 deletions packages/service-library/src/servicelib/progress_bar.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@
from inspect import isawaitable
from typing import Final, Optional, Protocol, runtime_checkable

from models_library.basic_types import IDStr
from models_library.progress_bar import (
ProgressReport,
ProgressStructuredMessage,
@@ -84,7 +83,7 @@ async def main_fct():
"description": "Optionally defines the step relative weight (defaults to steps of equal weights)"
},
)
description: IDStr = field(metadata={"description": "define the progress name"})
description: str = field(metadata={"description": "define the progress name"})
progress_unit: ProgressUnit | None = None
progress_report_cb: AsyncReportCB | ReportCB | None = None
_current_steps: float = _INITIAL_VALUE
@@ -210,7 +209,7 @@ async def finish(self) -> None:
def sub_progress(
self,
steps: int,
description: IDStr,
description: str,
step_weights: list[float] | None = None,
progress_unit: ProgressUnit | None = None,
) -> "ProgressBarData":
51 changes: 24 additions & 27 deletions packages/service-library/tests/test_progress_bar.py
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@

import pytest
from faker import Faker
from models_library.basic_types import IDStr
from models_library.progress_bar import ProgressReport, ProgressStructuredMessage
from pydantic import ValidationError
from pytest_mock import MockerFixture
@@ -55,7 +54,7 @@ async def test_progress_bar_progress_report_cb(
num_steps=outer_num_steps,
progress_report_cb=mocked_cb,
progress_unit="Byte",
description=IDStr(faker.pystr()),
description=faker.pystr(),
) as root:
assert root.num_steps == outer_num_steps
assert root.step_weights is None # i.e. all steps have equal weight
@@ -97,7 +96,7 @@ async def test_progress_bar_progress_report_cb(
# 2nd step is a sub progress bar of 10 steps
inner_num_steps_step2 = 100
async with root.sub_progress(
steps=inner_num_steps_step2, description=IDStr(faker.pystr())
steps=inner_num_steps_step2, description=faker.pystr()
) as sub:
assert sub._current_steps == pytest.approx(0) # noqa: SLF001
assert root._current_steps == pytest.approx(1) # noqa: SLF001
@@ -126,7 +125,7 @@ async def test_progress_bar_progress_report_cb(
# 3rd step is another subprogress of 50 steps
inner_num_steps_step3 = 50
async with root.sub_progress(
steps=inner_num_steps_step3, description=IDStr(faker.pystr())
steps=inner_num_steps_step3, description=faker.pystr()
) as sub:
assert sub._current_steps == pytest.approx(0) # noqa: SLF001
assert root._current_steps == pytest.approx(2) # noqa: SLF001
@@ -148,7 +147,7 @@ async def test_progress_bar_progress_report_cb(
def test_creating_progress_bar_with_invalid_unit_fails(faker: Faker):
with pytest.raises(ValidationError):
ProgressBarData(
num_steps=321, progress_unit="invalid", description=IDStr(faker.pystr())
num_steps=321, progress_unit="invalid", description=faker.pystr()
)


@@ -159,7 +158,7 @@ async def test_progress_bar_always_reports_0_on_creation_and_1_on_finish(
progress_bar = ProgressBarData(
num_steps=num_steps,
progress_report_cb=mocked_progress_bar_cb,
description=IDStr(faker.pystr()),
description=faker.pystr(),
)
assert progress_bar._current_steps == _INITIAL_VALUE # noqa: SLF001
async with progress_bar as root:
@@ -207,7 +206,7 @@ async def test_progress_bar_always_reports_1_on_finish(
progress_bar = ProgressBarData(
num_steps=num_steps,
progress_report_cb=mocked_progress_bar_cb,
description=IDStr(faker.pystr()),
description=faker.pystr(),
)
assert progress_bar._current_steps == _INITIAL_VALUE # noqa: SLF001
async with progress_bar as root:
@@ -249,7 +248,7 @@ async def test_progress_bar_always_reports_1_on_finish(


async def test_set_progress(caplog: pytest.LogCaptureFixture, faker: Faker):
async with ProgressBarData(num_steps=50, description=IDStr(faker.pystr())) as root:
async with ProgressBarData(num_steps=50, description=faker.pystr()) as root:
assert root._current_steps == pytest.approx(0) # noqa: SLF001
assert root.num_steps == 50
assert root.step_weights is None
@@ -264,7 +263,7 @@ async def test_set_progress(caplog: pytest.LogCaptureFixture, faker: Faker):


async def test_reset_progress(caplog: pytest.LogCaptureFixture, faker: Faker):
async with ProgressBarData(num_steps=50, description=IDStr(faker.pystr())) as root:
async with ProgressBarData(num_steps=50, description=faker.pystr()) as root:
assert root._current_steps == pytest.approx(0) # noqa: SLF001
assert root.num_steps == 50
assert root.step_weights is None
@@ -292,43 +291,41 @@ async def test_reset_progress(caplog: pytest.LogCaptureFixture, faker: Faker):

async def test_concurrent_progress_bar(faker: Faker):
async def do_something(root: ProgressBarData):
async with root.sub_progress(steps=50, description=IDStr(faker.pystr())) as sub:
async with root.sub_progress(steps=50, description=faker.pystr()) as sub:
assert sub.num_steps == 50
assert sub.step_weights is None
assert sub._current_steps == 0 # noqa: SLF001
for n in range(50):
await sub.update()
assert sub._current_steps == (n + 1) # noqa: SLF001

async with ProgressBarData(num_steps=12, description=IDStr(faker.pystr())) as root:
async with ProgressBarData(num_steps=12, description=faker.pystr()) as root:
assert root._current_steps == pytest.approx(0) # noqa: SLF001
assert root.step_weights is None
await asyncio.gather(*[do_something(root) for n in range(12)])
assert root._current_steps == pytest.approx(12) # noqa: SLF001


async def test_too_many_sub_progress_bars_raises(faker: Faker):
async with ProgressBarData(num_steps=2, description=IDStr(faker.pystr())) as root:
async with ProgressBarData(num_steps=2, description=faker.pystr()) as root:
assert root.num_steps == 2
assert root.step_weights is None
async with root.sub_progress(steps=50, description=IDStr(faker.pystr())) as sub:
async with root.sub_progress(steps=50, description=faker.pystr()) as sub:
for _ in range(50):
await sub.update()
async with root.sub_progress(steps=50, description=IDStr(faker.pystr())) as sub:
async with root.sub_progress(steps=50, description=faker.pystr()) as sub:
for _ in range(50):
await sub.update()

with pytest.raises(RuntimeError):
async with root.sub_progress(
steps=50, description=IDStr(faker.pystr())
) as sub:
async with root.sub_progress(steps=50, description=faker.pystr()) as sub:
...


async def test_too_many_updates_does_not_raise_but_show_warning_with_stack(
caplog: pytest.LogCaptureFixture, faker: Faker
):
async with ProgressBarData(num_steps=2, description=IDStr(faker.pystr())) as root:
async with ProgressBarData(num_steps=2, description=faker.pystr()) as root:
assert root.num_steps == 2
assert root.step_weights is None
await root.update()
@@ -344,7 +341,7 @@ async def test_weighted_progress_bar(mocked_progress_bar_cb: mock.Mock, faker: F
num_steps=outer_num_steps,
step_weights=[1, 3, 1],
progress_report_cb=mocked_progress_bar_cb,
description=IDStr(faker.pystr()),
description=faker.pystr(),
) as root:
mocked_progress_bar_cb.assert_called_once_with(
ProgressReport(
@@ -399,7 +396,7 @@ async def test_weighted_progress_bar_with_weighted_sub_progress(
num_steps=outer_num_steps,
step_weights=[1, 3, 1],
progress_report_cb=mocked_progress_bar_cb,
description=IDStr(faker.pystr()),
description=faker.pystr(),
) as root:
mocked_progress_bar_cb.assert_called_once_with(
ProgressReport(
@@ -426,7 +423,7 @@ async def test_weighted_progress_bar_with_weighted_sub_progress(

# 2nd step is a sub progress bar of 5 steps
async with root.sub_progress(
steps=5, step_weights=[2, 5, 1, 2, 3], description=IDStr(faker.pystr())
steps=5, step_weights=[2, 5, 1, 2, 3], description=faker.pystr()
) as sub:
assert sub.step_weights == [2 / 13, 5 / 13, 1 / 13, 2 / 13, 3 / 13, 0]
assert sub._current_steps == pytest.approx(0) # noqa: SLF001
@@ -487,7 +484,7 @@ async def test_weighted_progress_bar_with_weighted_sub_progress(
async def test_weighted_progress_bar_wrong_num_weights_raises(faker: Faker):
with pytest.raises(RuntimeError):
async with ProgressBarData(
num_steps=3, step_weights=[3, 1], description=IDStr(faker.pystr())
num_steps=3, step_weights=[3, 1], description=faker.pystr()
):
...

@@ -496,7 +493,7 @@ async def test_weighted_progress_bar_with_0_weights_is_equivalent_to_standard_pr
faker: Faker,
):
async with ProgressBarData(
num_steps=3, step_weights=[0, 0, 0], description=IDStr(faker.pystr())
num_steps=3, step_weights=[0, 0, 0], description=faker.pystr()
) as root:
assert root.step_weights == [1, 1, 1, 0]

@@ -509,13 +506,13 @@ async def test_concurrent_sub_progress_update_correct_sub_progress(
num_steps=3,
step_weights=[3, 1, 2],
progress_report_cb=mocked_progress_bar_cb,
description=IDStr(faker.pystr()),
description=faker.pystr(),
) as root:
sub_progress1 = root.sub_progress(23, description=IDStr(faker.pystr()))
sub_progress1 = root.sub_progress(23, description=faker.pystr())
assert sub_progress1._current_steps == _INITIAL_VALUE # noqa: SLF001
sub_progress2 = root.sub_progress(45, description=IDStr(faker.pystr()))
sub_progress2 = root.sub_progress(45, description=faker.pystr())
assert sub_progress2._current_steps == _INITIAL_VALUE # noqa: SLF001
sub_progress3 = root.sub_progress(12, description=IDStr(faker.pystr()))
sub_progress3 = root.sub_progress(12, description=faker.pystr())
assert sub_progress3._current_steps == _INITIAL_VALUE # noqa: SLF001

# NOTE: in a gather call there is no control on which step finishes first
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@
from pathlib import Path
from tempfile import TemporaryDirectory

from models_library.basic_types import IDStr
from models_library.projects import ProjectID
from models_library.projects_nodes_io import NodeID, StorageFileID
from models_library.users import UserID
@@ -105,7 +104,7 @@ async def _pull_legacy_archive(
) -> None:
# NOTE: the legacy way of storing states was as zip archives
async with progress_bar.sub_progress(
steps=2, description=IDStr(f"pulling {destination_path.name}")
steps=2, description=f"pulling {destination_path.name}"
) as sub_prog:
with TemporaryDirectory() as tmp_dir_name:
archive_file = Path(tmp_dir_name) / __get_s3_name(
Original file line number Diff line number Diff line change
@@ -6,10 +6,8 @@
from asyncio.streams import StreamReader
from pathlib import Path

from common_library.errors_classes import OsparcErrorMixin

from aiocache import cached # type: ignore[import-untyped]
from models_library.basic_types import IDStr
from common_library.errors_classes import OsparcErrorMixin
from pydantic import AnyUrl, ByteSize
from servicelib.progress_bar import ProgressBarData
from servicelib.utils import logged_gather
@@ -242,7 +240,7 @@ async def _sync_sources(
async with progress_bar.sub_progress(
steps=folder_size,
progress_unit="Byte",
description=IDStr(f"transferring {local_dir.name}"),
description=f"transferring {local_dir.name}",
) as sub_progress:
aws_s3_cli_log_parsers: list[BaseLogParser] = (
[DebugLogParser()] if debug_logs else []
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
RequestInfo,
)
from models_library.api_schemas_storage import ETag, FileUploadSchema, UploadedPart
from models_library.basic_types import IDStr, SHA256Str
from models_library.basic_types import SHA256Str
from multidict import MultiMapping
from pydantic import AnyUrl, NonNegativeInt
from servicelib.aiohttp import status
@@ -216,7 +216,7 @@ async def download_link_to_file(
sub_progress = await stack.enter_async_context(
progress_bar.sub_progress(
steps=file_size or 1,
description=IDStr(f"downloading {file_path.name}"),
description=f"downloading {file_path.name}",
)
)

@@ -400,7 +400,7 @@ async def upload_file_to_presigned_links(
)
sub_progress = await stack.enter_async_context(
progress_bar.sub_progress(
steps=file_size, description=IDStr(f"uploading {file_name}")
steps=file_size, description=f"uploading {file_name}"
)
)

Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@
LinkType,
UploadedPart,
)
from models_library.basic_types import IDStr, SHA256Str
from models_library.basic_types import SHA256Str
from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
from models_library.users import UserID
from pydantic import AnyUrl, ByteSize, TypeAdapter
@@ -364,7 +364,7 @@ async def _upload_path( # pylint: disable=too-many-arguments
)

if not progress_bar:
progress_bar = ProgressBarData(num_steps=1, description=IDStr("uploading"))
progress_bar = ProgressBarData(num_steps=1, description="uploading")

is_directory: bool = isinstance(path_to_upload, Path) and path_to_upload.is_dir()
if (
Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@
from aiocache import cached # type: ignore[import-untyped]
from aiofiles import tempfile
from common_library.errors_classes import OsparcErrorMixin
from models_library.basic_types import IDStr
from pydantic import AnyUrl, BaseModel, ByteSize
from servicelib.progress_bar import ProgressBarData
from servicelib.utils import logged_gather
@@ -224,7 +223,7 @@ async def _sync_sources(
async with progress_bar.sub_progress(
steps=folder_size,
progress_unit="Byte",
description=IDStr(f"transferring {local_dir.name}"),
description=f"transferring {local_dir.name}",
) as sub_progress:
r_clone_log_parsers: list[BaseLogParser] = (
[DebugLogParser()] if debug_logs else []
Original file line number Diff line number Diff line change
@@ -7,7 +7,6 @@
from typing import Any

from models_library.api_schemas_storage import LinkType
from models_library.basic_types import IDStr
from models_library.projects import ProjectIDStr
from models_library.projects_nodes_io import NodeIDStr
from models_library.services_types import ServicePortKey
@@ -229,7 +228,7 @@ async def _set_with_notifications(

tasks = []
async with progress_bar.sub_progress(
steps=len(port_values.items()), description=IDStr("set multiple")
steps=len(port_values.items()), description="set multiple"
) as sub_progress:
for port_key, (value, set_kwargs) in port_values.items():
tasks.append(
Loading
Oops, something went wrong.

0 comments on commit d27914b

Please sign in to comment.