From b07f1703ffcfc332a4f1f88019bb7718b1b40504 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 5 Oct 2024 03:39:54 +0600 Subject: [PATCH 01/99] Sync common dynamic config --- miner/app/src/compute_horde_miner/miner/tasks.py | 5 +++++ validator/app/src/compute_horde_validator/validator/tasks.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index e5d3bc465..1da29f514 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -165,7 +165,12 @@ def get_receipts_from_old_miner(): @app.task def fetch_dynamic_config() -> None: + # if same key exists in both places, common config wins sync_dynamic_config( config_url=f"https://raw.githubusercontent.com/backend-developers-ltd/compute-horde-dynamic-config/master/miner-config-{settings.DYNAMIC_CONFIG_ENV}.json", namespace=config, ) + sync_dynamic_config( + config_url=f"https://raw.githubusercontent.com/backend-developers-ltd/compute-horde-dynamic-config/master/common-config-{settings.DYNAMIC_CONFIG_ENV}.json", + namespace=config, + ) diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 12bc6a679..e370d74eb 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1137,10 +1137,15 @@ def send_events_to_facilitator(): @app.task def fetch_dynamic_config() -> None: + # if same key exists in both places, common config wins sync_dynamic_config( config_url=f"https://raw.githubusercontent.com/backend-developers-ltd/compute-horde-dynamic-config/master/validator-config-{settings.DYNAMIC_CONFIG_ENV}.json", namespace=config, ) + sync_dynamic_config( + config_url=f"https://raw.githubusercontent.com/backend-developers-ltd/compute-horde-dynamic-config/master/common-config-{settings.DYNAMIC_CONFIG_ENV}.json", + namespace=config, + ) @app.task( From 514e6ceb7878289841bd87d4e988d3485e5224d4 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 10 Oct 2024 01:52:01 +0600 Subject: [PATCH 02/99] Properly handle miner responses in organic client --- .../compute_horde/miner_client/organic.py | 65 ++++++++++++------- .../tests/test_organic_miner_client.py | 55 +++++++++++----- compute_horde/tests/test_run_organic_job.py | 7 +- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index bd2b6d5bb..e6292dd3f 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -90,15 +90,22 @@ def __init__( self.miner_manifest = loop.create_future() self.online_executor_count = 0 - # for waiting on miner responses (replaces JobState) - self.miner_ready_or_declining_future: asyncio.Future[ - V0DeclineJobRequest | V0ExecutorFailedRequest | V0ExecutorReadyRequest + # for waiting on miner responses + self.miner_accepting_or_declining_future: asyncio.Future[ + V0AcceptJobRequest | V0DeclineJobRequest ] = loop.create_future() - self.miner_ready_or_declining_timestamp: int = 0 + self.miner_accepting_or_declining_timestamp: int = 0 + + self.executor_ready_or_failed_future: asyncio.Future[ + V0ExecutorReadyRequest | V0ExecutorFailedRequest + ] = loop.create_future() + self.executor_ready_or_failed_timestamp: int = 0 + self.miner_finished_or_failed_future: asyncio.Future[ V0JobFailedRequest | V0JobFinishedRequest ] = loop.create_future() self.miner_finished_or_failed_timestamp: int = 0 + self.miner_machine_specs: MachineSpecs | None = None name = f"{miner_hotkey}({miner_address}:{miner_port})" @@ -168,14 +175,16 @@ async def handle_message(self, msg: BaseRequest) -> None: ) return - if isinstance(msg, V0AcceptJobRequest): - logger.info(f"Miner {self.miner_name} accepted job") - elif isinstance( - msg, V0DeclineJobRequest | V0ExecutorFailedRequest | V0ExecutorReadyRequest - ): + if isinstance(msg, V0AcceptJobRequest | V0DeclineJobRequest): try: - self.miner_ready_or_declining_future.set_result(msg) - self.miner_ready_or_declining_timestamp = int(time.time()) + self.miner_accepting_or_declining_future.set_result(msg) + self.miner_accepting_or_declining_timestamp = int(time.time()) + except asyncio.InvalidStateError: + logger.warning(f"Received {msg} from {self.miner_name} but future was already set") + elif isinstance(msg, V0ExecutorReadyRequest | V0ExecutorFailedRequest): + try: + self.executor_ready_or_failed_future.set_result(msg) + self.executor_ready_or_failed_timestamp = int(time.time()) except asyncio.InvalidStateError: logger.warning(f"Received {msg} from {self.miner_name} but future was already set") elif isinstance(msg, V0JobFailedRequest | V0JobFinishedRequest): @@ -291,6 +300,7 @@ async def connect(self) -> None: class FailureReason(enum.Enum): MINER_CONNECTION_FAILED = enum.auto() INITIAL_RESPONSE_TIMED_OUT = enum.auto() + EXECUTOR_READINESS_RESPONSE_TIMED_OUT = enum.auto() FINAL_RESPONSE_TIMED_OUT = enum.auto() JOB_DECLINED = enum.auto() EXECUTOR_FAILED = enum.auto() @@ -364,15 +374,22 @@ async def run_organic_job( try: initial_response = await asyncio.wait_for( - client.miner_ready_or_declining_future, + client.miner_accepting_or_declining_future, timeout=min(job_timer.time_left(), wait_timeout), ) except TimeoutError as exc: raise OrganicJobError(FailureReason.INITIAL_RESPONSE_TIMED_OUT) from exc - if isinstance(initial_response, V0DeclineJobRequest): raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) - elif isinstance(initial_response, V0ExecutorFailedRequest): + + try: + executor_readiness_response = await asyncio.wait_for( + client.executor_ready_or_failed_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError as exc: + raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc + if isinstance(executor_readiness_response, V0ExecutorFailedRequest): raise OrganicJobError(FailureReason.EXECUTOR_FAILED, initial_response) await client.send_job_started_receipt_message( @@ -399,16 +416,14 @@ async def run_organic_job( client.miner_finished_or_failed_future, timeout=job_timer.time_left(), ) + if isinstance(final_response, V0JobFailedRequest): + raise OrganicJobError(FailureReason.JOB_FAILED, final_response) + return final_response.docker_process_stdout, final_response.docker_process_stderr except TimeoutError as exc: raise OrganicJobError(FailureReason.FINAL_RESPONSE_TIMED_OUT) from exc - - if isinstance(final_response, V0JobFailedRequest): - raise OrganicJobError(FailureReason.JOB_FAILED, final_response) - - await client.send_job_finished_receipt_message( - started_timestamp=job_timer.start_time.timestamp(), - time_took_seconds=job_timer.passed_time(), - score=0, # no score for organic jobs (at least right now) - ) - - return final_response.docker_process_stdout, final_response.docker_process_stderr + finally: + await client.send_job_finished_receipt_message( + started_timestamp=job_timer.start_time.timestamp(), + time_took_seconds=job_timer.passed_time(), + score=0, # no score for organic jobs (at least right now) + ) diff --git a/compute_horde/tests/test_organic_miner_client.py b/compute_horde/tests/test_organic_miner_client.py index 679fd99d1..1773362ad 100644 --- a/compute_horde/tests/test_organic_miner_client.py +++ b/compute_horde/tests/test_organic_miner_client.py @@ -6,6 +6,7 @@ from compute_horde.base_requests import BaseRequest from compute_horde.miner_client.organic import OrganicMinerClient from compute_horde.mv_protocol.miner_requests import ( + V0AcceptJobRequest, V0DeclineJobRequest, V0ExecutorFailedRequest, V0ExecutorReadyRequest, @@ -35,11 +36,12 @@ def get_miner_client( @pytest.mark.asyncio @pytest.mark.parametrize( - "initial_msg,final_msg", + "initial_msg,executor_msg,final_msg", [ - (V0DeclineJobRequest(job_uuid=JOB_UUID), None), - (V0ExecutorFailedRequest(job_uuid=JOB_UUID), None), + (V0DeclineJobRequest(job_uuid=JOB_UUID), None, None), + (V0AcceptJobRequest(job_uuid=JOB_UUID), V0ExecutorFailedRequest(job_uuid=JOB_UUID), None), ( + V0AcceptJobRequest(job_uuid=JOB_UUID), V0ExecutorReadyRequest(job_uuid=JOB_UUID), V0JobFailedRequest( job_uuid=JOB_UUID, @@ -49,6 +51,7 @@ def get_miner_client( ), ), ( + V0AcceptJobRequest(job_uuid=JOB_UUID), V0ExecutorReadyRequest(job_uuid=JOB_UUID), V0JobFinishedRequest( job_uuid=JOB_UUID, @@ -58,22 +61,41 @@ def get_miner_client( ), ], ) -async def test_organic_miner_client__futures__properly_set(initial_msg, final_msg, keypair): +async def test_organic_miner_client__futures__properly_set( + initial_msg, executor_msg, final_msg, keypair +): miner_client = get_miner_client(keypair) - assert not miner_client.miner_ready_or_declining_future.done() - assert miner_client.miner_ready_or_declining_timestamp == 0 + assert not miner_client.miner_accepting_or_declining_future.done() + assert miner_client.miner_accepting_or_declining_timestamp == 0 await miner_client.handle_message(initial_msg) - assert miner_client.miner_ready_or_declining_future.done() - assert miner_client.miner_ready_or_declining_timestamp != 0 - assert await miner_client.miner_ready_or_declining_future == initial_msg + assert miner_client.miner_accepting_or_declining_future.done() + assert miner_client.miner_accepting_or_declining_timestamp != 0 + assert await miner_client.miner_accepting_or_declining_future == initial_msg # should set only once - set_time = miner_client.miner_ready_or_declining_timestamp + set_time = miner_client.miner_accepting_or_declining_timestamp await miner_client.handle_message(initial_msg) - assert miner_client.miner_ready_or_declining_timestamp == set_time + assert miner_client.miner_accepting_or_declining_timestamp == set_time + + if executor_msg is None: + return + + assert not miner_client.executor_ready_or_failed_future.done() + assert miner_client.executor_ready_or_failed_timestamp == 0 + + await miner_client.handle_message(executor_msg) + + assert miner_client.executor_ready_or_failed_future.done() + assert miner_client.executor_ready_or_failed_future != 0 + assert await miner_client.executor_ready_or_failed_future == executor_msg + + # should set only once + set_time = miner_client.executor_ready_or_failed_timestamp + await miner_client.handle_message(executor_msg) + assert miner_client.executor_ready_or_failed_timestamp == set_time if final_msg is None: return @@ -97,21 +119,20 @@ async def test_organic_miner_client__futures__properly_set(initial_msg, final_ms @pytest.mark.parametrize( "initial_msg", [ + V0AcceptJobRequest(job_uuid=str(uuid.uuid4())), V0DeclineJobRequest(job_uuid=str(uuid.uuid4())), - V0ExecutorFailedRequest(job_uuid=str(uuid.uuid4())), - V0ExecutorReadyRequest(job_uuid=str(uuid.uuid4())), ], ) async def test_organic_miner_client__skip_different_job__initial_future(initial_msg, keypair): miner_client = get_miner_client(keypair) - assert not miner_client.miner_ready_or_declining_future.done() - assert miner_client.miner_ready_or_declining_timestamp == 0 + assert not miner_client.miner_accepting_or_declining_future.done() + assert miner_client.miner_accepting_or_declining_timestamp == 0 await miner_client.handle_message(initial_msg) - assert not miner_client.miner_ready_or_declining_future.done() - assert miner_client.miner_ready_or_declining_timestamp == 0 + assert not miner_client.miner_accepting_or_declining_future.done() + assert miner_client.miner_accepting_or_declining_timestamp == 0 @pytest.mark.asyncio diff --git a/compute_horde/tests/test_run_organic_job.py b/compute_horde/tests/test_run_organic_job.py index 89d9a5073..ad43f8555 100644 --- a/compute_horde/tests/test_run_organic_job.py +++ b/compute_horde/tests/test_run_organic_job.py @@ -5,7 +5,11 @@ OrganicMinerClient, run_organic_job, ) -from compute_horde.mv_protocol.miner_requests import V0ExecutorReadyRequest, V0JobFinishedRequest +from compute_horde.mv_protocol.miner_requests import ( + V0AcceptJobRequest, + V0ExecutorReadyRequest, + V0JobFinishedRequest, +) from compute_horde.mv_protocol.validator_requests import ( BaseValidatorRequest, V0AuthenticateRequest, @@ -34,6 +38,7 @@ async def test_run_organic_job__success(keypair): mock_transport = MinerStubTransport( "mock", [ + V0AcceptJobRequest(job_uuid=JOB_UUID).model_dump_json(), V0ExecutorReadyRequest(job_uuid=JOB_UUID).model_dump_json(), V0JobFinishedRequest( job_uuid=JOB_UUID, From 3ec6f3ae5a79089ca2588af2696eb6c93421afca Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 10 Oct 2024 03:16:01 +0600 Subject: [PATCH 03/99] Update organic miner client usage --- .../validator/organic_jobs/miner_driver.py | 46 ++++++++++++++++++- .../validator/tests/helpers.py | 6 ++- .../validator/tests/test_miner_driver.py | 30 ++++++++---- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 5a0cdfb77..ab7030fb9 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -9,6 +9,7 @@ from compute_horde.base.volume import InlineVolume, Volume, ZipUrlVolume from compute_horde.executor_class import ExecutorClass from compute_horde.mv_protocol.miner_requests import ( + V0AcceptJobRequest, V0DeclineJobRequest, V0ExecutorFailedRequest, V0ExecutorReadyRequest, @@ -147,7 +148,7 @@ async def handle_send_error_event(msg: str): try: msg = await asyncio.wait_for( - miner_client.miner_ready_or_declining_future, + miner_client.miner_accepting_or_declining_future, timeout=min(job_timer.time_left(), wait_timeout), ) except TimeoutError: @@ -165,7 +166,7 @@ async def handle_send_error_event(msg: str): await notify_callback(JobStatusUpdate.from_job(job, "failed")) return - if isinstance(msg, V0DeclineJobRequest | V0ExecutorFailedRequest): + if isinstance(msg, V0DeclineJobRequest): comment = f"Miner {miner_client.miner_name} won't do job: {msg.model_dump_json()}" job.status = OrganicJob.Status.FAILED job.comment = comment @@ -179,6 +180,47 @@ async def handle_send_error_event(msg: str): if notify_callback: await notify_callback(JobStatusUpdate.from_job(job, "rejected")) return + elif isinstance(msg, V0AcceptJobRequest): + logger.debug(f"Miner {miner_client.miner_name} accepted job: {msg}") + else: + raise ValueError(f"Unexpected msg from miner {miner_client.miner_name}: {msg}") + + try: + msg = await asyncio.wait_for( + miner_client.executor_ready_or_failed_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError: + comment = f"Miner {miner_client.miner_name} timed out while preparing executor for job {job.job_uuid} after {wait_timeout} seconds" + job.status = OrganicJob.Status.FAILED + job.comment = comment + await job.asave() + + logger.warning(comment) + await save_event( + subtype=SystemEvent.EventSubType.JOB_NOT_STARTED, + long_description=comment, + ) + if notify_callback: + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + return + + if isinstance(msg, V0ExecutorFailedRequest): + comment = ( + f"Miner {miner_client.miner_name} failed to start executor: {msg.model_dump_json()}" + ) + job.status = OrganicJob.Status.FAILED + job.comment = comment + await job.asave() + + logger.info(comment) + await save_event( + subtype=SystemEvent.EventSubType.JOB_REJECTED, + long_description=comment, + ) + if notify_callback: + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + return elif isinstance(msg, V0ExecutorReadyRequest): logger.debug(f"Miner {miner_client.miner_name} ready for job: {msg}") if notify_callback: diff --git a/validator/app/src/compute_horde_validator/validator/tests/helpers.py b/validator/app/src/compute_horde_validator/validator/tests/helpers.py index b222779e2..6ec366dcd 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/helpers.py +++ b/validator/app/src/compute_horde_validator/validator/tests/helpers.py @@ -15,6 +15,7 @@ from bittensor import Balance from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.mv_protocol.miner_requests import ( + V0AcceptJobRequest, V0ExecutorReadyRequest, V0JobFinishedRequest, ) @@ -119,7 +120,10 @@ def _query_sent_models(self, condition=None, model_class=None): class MockJobStateMinerClient(MockMinerClient): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.miner_ready_or_declining_future.set_result( + self.miner_accepting_or_declining_future.set_result( + V0AcceptJobRequest(job_uuid=self.job_uuid) + ) + self.executor_ready_or_failed_future.set_result( V0ExecutorReadyRequest(job_uuid=self.job_uuid) ) self.miner_finished_or_failed_future.set_result( diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index 803cbf6d6..bad2cebae 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -3,7 +3,9 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.mv_protocol.miner_requests import ( + V0AcceptJobRequest, V0DeclineJobRequest, + V0ExecutorFailedRequest, V0ExecutorReadyRequest, V0JobFailedRequest, V0JobFinishedRequest, @@ -40,7 +42,7 @@ ), [ ( - (None, None), + (None, None, None), ["failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, @@ -48,7 +50,7 @@ False, ), ( - (V0DeclineJobRequest, None), + (V0DeclineJobRequest, None, None), ["rejected"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, @@ -56,7 +58,15 @@ False, ), ( - (V0ExecutorReadyRequest, None), + (V0AcceptJobRequest, V0ExecutorFailedRequest, None), + ["failed"], + OrganicJob.Status.FAILED, + get_dummy_job_request_v0, + False, + False, + ), + ( + (V0AcceptJobRequest, V0ExecutorReadyRequest, None), ["accepted", "failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, @@ -64,7 +74,7 @@ False, ), ( - (V0ExecutorReadyRequest, V0JobFailedRequest), + (V0AcceptJobRequest, V0ExecutorReadyRequest, V0JobFailedRequest), ["accepted", "failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, @@ -72,7 +82,7 @@ False, ), ( - (V0ExecutorReadyRequest, V0JobFinishedRequest), + (V0AcceptJobRequest, V0ExecutorReadyRequest, V0JobFinishedRequest), ["accepted", "completed"], OrganicJob.Status.COMPLETED, get_dummy_job_request_v0, @@ -80,7 +90,7 @@ True, ), ( - (V0ExecutorReadyRequest, V0JobFinishedRequest), + (V0AcceptJobRequest, V0ExecutorReadyRequest, V0JobFinishedRequest), ["accepted", "completed"], OrganicJob.Status.COMPLETED, get_dummy_job_request_v1, @@ -111,12 +121,14 @@ async def test_miner_driver( job_description="User job from facilitator", ) miner_client = get_miner_client(MockMinerClient, job_uuid) - f0, f1 = futures_result + f0, f1, f2 = futures_result if f0: - miner_client.miner_ready_or_declining_future.set_result(f0(job_uuid=job_uuid)) + miner_client.miner_accepting_or_declining_future.set_result(f0(job_uuid=job_uuid)) if f1: + miner_client.executor_ready_or_failed_future.set_result(f1(job_uuid=job_uuid)) + if f2: miner_client.miner_finished_or_failed_future.set_result( - f1( + f2( job_uuid=job_uuid, docker_process_stdout="mocked stdout", docker_process_stderr="mocked stderr", From b5df4c361671080f37c4b61ecc125600ab6403e5 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 13:15:35 +0200 Subject: [PATCH 04/99] add receipt models to library --- compute_horde/README.md | 19 ++++- .../compute_horde/mv_protocol/__init__.py | 1 + .../compute_horde/mv_protocol/apps.py | 11 +++ .../compute_horde/mv_protocol/models.py | 76 +++++++++++++++++++ compute_horde/pdm.lock | 21 ++--- compute_horde/pyproject.toml | 1 + compute_horde/settings.py | 3 + 7 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 compute_horde/compute_horde/mv_protocol/apps.py create mode 100644 compute_horde/compute_horde/mv_protocol/models.py create mode 100644 compute_horde/settings.py diff --git a/compute_horde/README.md b/compute_horde/README.md index ba6ebf519..7d0b070c1 100644 --- a/compute_horde/README.md +++ b/compute_horde/README.md @@ -1 +1,18 @@ -# compute-horde \ No newline at end of file +# compute-horde + +## Common django models +This library contains common Django models for the Validator <-> Miner communication. +To use them, update your `INSTALLED_APPS`: +```python +INSTALLED_APPS = [ + ..., + 'compute_horde.mv_protocol', + ..., +] +``` + +## Migrations +To make new migrations after doing some changes in the model files, run: +```shell +DJANGO_SETTINGS_MODULE=settings pdm run django-admin makemigrations +``` diff --git a/compute_horde/compute_horde/mv_protocol/__init__.py b/compute_horde/compute_horde/mv_protocol/__init__.py index e69de29bb..823734570 100644 --- a/compute_horde/compute_horde/mv_protocol/__init__.py +++ b/compute_horde/compute_horde/mv_protocol/__init__.py @@ -0,0 +1 @@ +default_app_config = "compute_horde.mv_protocol.apps.ComputeHordeMVProtocolConfig" diff --git a/compute_horde/compute_horde/mv_protocol/apps.py b/compute_horde/compute_horde/mv_protocol/apps.py new file mode 100644 index 000000000..16cf6561b --- /dev/null +++ b/compute_horde/compute_horde/mv_protocol/apps.py @@ -0,0 +1,11 @@ +import logging + +from django.apps import AppConfig + +logger = logging.getLogger(__name__) + + +class ComputeHordeMVProtocolConfig(AppConfig): + name = "compute_horde.mv_protocol" + verbose_name = "Miner-Validator protocol dependencies" + default_auto_field = "django.db.models.BigAutoField" \ No newline at end of file diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/mv_protocol/models.py new file mode 100644 index 000000000..29dc38021 --- /dev/null +++ b/compute_horde/compute_horde/mv_protocol/models.py @@ -0,0 +1,76 @@ +from datetime import timedelta + +from django.db import models + +from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass +from compute_horde.mv_protocol.validator_requests import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.receipts import Receipt + + +class AbstractReceipt(models.Model): + job_uuid = models.UUIDField() + validator_hotkey = models.CharField(max_length=256) + miner_hotkey = models.CharField(max_length=256) + validator_signature = models.CharField(max_length=256) + miner_signature = models.CharField(max_length=256, null=True, blank=True) + + class Meta: + abstract = True + constraints = [ + models.UniqueConstraint(fields=["job_uuid"], name="unique_%(class)s_job_uuid"), + ] + + def __str__(self): + return f"job_uuid: {self.job_uuid}" + + +class JobStartedReceipt(AbstractReceipt): + executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) + time_accepted = models.DateTimeField() + max_timeout = models.IntegerField() + + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobStartedReceipt"] + + def to_receipt(self): + return Receipt( + payload=JobStartedReceiptPayload( + job_uuid=str(self.job_uuid), + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.validator_hotkey, + executor_class=ExecutorClass(self.executor_class), + time_accepted=self.time_accepted, + max_timeout=self.max_timeout, + ), + validator_signature=self.validator_signature, + miner_signature=self.miner_signature, + ) + + +class JobFinishedReceipt(AbstractReceipt): + time_started = models.DateTimeField() + time_took_us = models.BigIntegerField() + score_str = models.CharField(max_length=256) + + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobFinishedReceipt"] + + def time_took(self): + return timedelta(microseconds=self.time_took_us) + + def score(self): + return float(self.score_str) + + def to_receipt(self): + return Receipt( + payload=JobFinishedReceiptPayload( + job_uuid=str(self.job_uuid), + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.validator_hotkey, + time_started=self.time_started, + time_took_us=self.time_took_us, + score_str=self.score_str, + ), + validator_signature=self.validator_signature, + miner_signature=self.miner_signature, + ) diff --git a/compute_horde/pdm.lock b/compute_horde/pdm.lock index cf775d1d8..006f67895 100644 --- a/compute_horde/pdm.lock +++ b/compute_horde/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "format", "lint", "release", "test", "type_check"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f657e43907f918877250290da1a797a3d5e198422ae2e835bd326c4cb5caf3ae" +content_hash = "sha256:6c0e95ad053091a2938575210bf10a2efa03ab8148a955d0227d0a8f2ebcdea4" [[metadata.targets]] requires_python = "==3.11.*" @@ -151,7 +151,7 @@ name = "asgiref" version = "3.8.1" requires_python = ">=3.8" summary = "ASGI specs, helper code, and adapters" -groups = ["type_check"] +groups = ["default", "type_check"] dependencies = [ "typing-extensions>=4; python_version < \"3.11\"", ] @@ -449,18 +449,19 @@ files = [ [[package]] name = "django" -version = "5.1.1" -requires_python = ">=3.10" +version = "4.2.16" +requires_python = ">=3.8" summary = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." -groups = ["type_check"] +groups = ["default", "type_check"] dependencies = [ - "asgiref<4,>=3.8.1", + "asgiref<4,>=3.6.0", + "backports-zoneinfo; python_version < \"3.9\"", "sqlparse>=0.3.1", "tzdata; sys_platform == \"win32\"", ] files = [ - {file = "Django-5.1.1-py3-none-any.whl", hash = "sha256:71603f27dac22a6533fb38d83072eea9ddb4017fead6f67f2562a40402d61c3f"}, - {file = "Django-5.1.1.tar.gz", hash = "sha256:021ffb7fdab3d2d388bc8c7c2434eb9c1f6f4d09e6119010bbb1694dda286bc2"}, + {file = "Django-4.2.16-py3-none-any.whl", hash = "sha256:1ddc333a16fc139fd253035a1606bb24261951bbc3a6ca256717fa06cc41a898"}, + {file = "Django-4.2.16.tar.gz", hash = "sha256:6f1616c2786c408ce86ab7e10f792b8f15742f7b7b7460243929cb371e7f1dad"}, ] [[package]] @@ -1470,7 +1471,7 @@ name = "sqlparse" version = "0.5.1" requires_python = ">=3.8" summary = "A non-validating SQL parser." -groups = ["type_check"] +groups = ["default", "type_check"] files = [ {file = "sqlparse-0.5.1-py3-none-any.whl", hash = "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4"}, {file = "sqlparse-0.5.1.tar.gz", hash = "sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e"}, @@ -1636,7 +1637,7 @@ name = "tzdata" version = "2024.2" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["type_check"] +groups = ["default", "type_check"] marker = "sys_platform == \"win32\"" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, diff --git a/compute_horde/pyproject.toml b/compute_horde/pyproject.toml index 23adc8cab..b4f5c431e 100644 --- a/compute_horde/pyproject.toml +++ b/compute_horde/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "websockets>=11.0", "more-itertools>=10.2.0", "requests>=2.32.2", + "Django~=4.2.4", ] [build-system] diff --git a/compute_horde/settings.py b/compute_horde/settings.py new file mode 100644 index 000000000..c29570020 --- /dev/null +++ b/compute_horde/settings.py @@ -0,0 +1,3 @@ +INSTALLED_APPS = [ + "compute_horde.mv_protocol" +] \ No newline at end of file From 63c2feb2438641a8ea6dd4bfa33b7ddbc1739e06 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 13:21:43 +0200 Subject: [PATCH 05/99] disallow creating unsigned Receipt objects --- compute_horde/compute_horde/mv_protocol/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/mv_protocol/models.py index 29dc38021..21e8bcd0d 100644 --- a/compute_horde/compute_horde/mv_protocol/models.py +++ b/compute_horde/compute_horde/mv_protocol/models.py @@ -32,7 +32,10 @@ class JobStartedReceipt(AbstractReceipt): # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 objects: models.Manager["JobStartedReceipt"] - def to_receipt(self): + def to_receipt(self) -> Receipt: + if self.miner_signature is None: + raise ValueError("Miner signature is required") + return Receipt( payload=JobStartedReceiptPayload( job_uuid=str(self.job_uuid), @@ -62,6 +65,9 @@ def score(self): return float(self.score_str) def to_receipt(self): + if self.miner_signature is None: + raise ValueError("Miner signature is required") + return Receipt( payload=JobFinishedReceiptPayload( job_uuid=str(self.job_uuid), From 5d7533fb45132339808537f03242fd4bf91b9bba Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 13:24:07 +0200 Subject: [PATCH 06/99] migrations --- .../mv_protocol/migrations/0001_initial.py | 89 +++++++++++++++++++ .../mv_protocol/migrations/__init__.py | 0 2 files changed, 89 insertions(+) create mode 100644 compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py create mode 100644 compute_horde/compute_horde/mv_protocol/migrations/__init__.py diff --git a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py new file mode 100644 index 000000000..b3e44e851 --- /dev/null +++ b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py @@ -0,0 +1,89 @@ +# Generated by Django 4.2.16 on 2024-10-10 05:45 + +import compute_horde.executor_class +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="JobFinishedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ( + "miner_signature", + models.CharField(blank=True, max_length=256, null=True), + ), + ("time_started", models.DateTimeField()), + ("time_took_us", models.BigIntegerField()), + ("score_str", models.CharField(max_length=256)), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="JobStartedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ( + "miner_signature", + models.CharField(blank=True, max_length=256, null=True), + ), + ( + "executor_class", + models.CharField( + default=compute_horde.executor_class.ExecutorClass[ + "spin_up_4min__gpu_24gb" + ], + max_length=255, + ), + ), + ("time_accepted", models.DateTimeField()), + ("max_timeout", models.IntegerField()), + ], + options={ + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="jobstartedreceipt", + constraint=models.UniqueConstraint( + fields=("job_uuid",), name="unique_jobstartedreceipt_job_uuid" + ), + ), + migrations.AddConstraint( + model_name="jobfinishedreceipt", + constraint=models.UniqueConstraint( + fields=("job_uuid",), name="unique_jobfinishedreceipt_job_uuid" + ), + ), + ] diff --git a/compute_horde/compute_horde/mv_protocol/migrations/__init__.py b/compute_horde/compute_horde/mv_protocol/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From 3ff62a46776251f31ffae9b03f20b925aa3459a0 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 13:33:45 +0200 Subject: [PATCH 07/99] custom exceptions for unsigned receipts --- compute_horde/compute_horde/mv_protocol/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/mv_protocol/models.py index 21e8bcd0d..193e1409c 100644 --- a/compute_horde/compute_horde/mv_protocol/models.py +++ b/compute_horde/compute_horde/mv_protocol/models.py @@ -7,6 +7,10 @@ from compute_horde.receipts import Receipt +class ReceiptNotSigned(Exception): + pass + + class AbstractReceipt(models.Model): job_uuid = models.UUIDField() validator_hotkey = models.CharField(max_length=256) @@ -34,7 +38,7 @@ class JobStartedReceipt(AbstractReceipt): def to_receipt(self) -> Receipt: if self.miner_signature is None: - raise ValueError("Miner signature is required") + raise ReceiptNotSigned("Miner signature is required") return Receipt( payload=JobStartedReceiptPayload( @@ -64,9 +68,9 @@ def time_took(self): def score(self): return float(self.score_str) - def to_receipt(self): + def to_receipt(self) -> Receipt: if self.miner_signature is None: - raise ValueError("Miner signature is required") + raise ReceiptNotSigned("Miner signature is required") return Receipt( payload=JobFinishedReceiptPayload( From 8afe653467a0b7c218dde7cc687b1d29cf1ebf83 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:09:55 +0200 Subject: [PATCH 08/99] miner: use common receipt models --- .../src/compute_horde_miner/miner/admin.py | 4 -- .../miner_consumer/validator_interface.py | 11 ++- .../src/compute_horde_miner/miner/models.py | 69 ------------------- .../src/compute_horde_miner/miner/tasks.py | 17 ++++- .../miner/tests/test_migration.py | 2 +- miner/app/src/compute_horde_miner/settings.py | 1 + 6 files changed, 21 insertions(+), 83 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/admin.py b/miner/app/src/compute_horde_miner/miner/admin.py index ffa27d075..db4db58af 100644 --- a/miner/app/src/compute_horde_miner/miner/admin.py +++ b/miner/app/src/compute_horde_miner/miner/admin.py @@ -5,8 +5,6 @@ AcceptedJob, Validator, ValidatorBlacklist, - JobFinishedReceipt, - JobStartedReceipt, ) @@ -65,6 +63,4 @@ class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): admin.site.register(AcceptedJob, admin_class=AcceptedJobReadOnlyAdmin) admin.site.register(Validator, admin_class=ValidatorReadOnlyAdmin) -admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) -admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) admin.site.register(ValidatorBlacklist) diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index 51e4229e3..55f32bf7c 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -7,7 +7,8 @@ import bittensor from compute_horde.mv_protocol import miner_requests, validator_requests -from compute_horde.mv_protocol.validator_requests import BaseValidatorRequest +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.mv_protocol.validator_requests import BaseValidatorRequest, V0JobStartedReceiptRequest from django.conf import settings from django.utils import timezone @@ -27,8 +28,6 @@ ) from compute_horde_miner.miner.models import ( AcceptedJob, - JobFinishedReceipt, - JobStartedReceipt, Validator, ValidatorBlacklist, ) @@ -388,11 +387,11 @@ async def handle_job_started_receipt(self, msg: validator_requests.V0JobStartedR return await JobStartedReceipt.objects.acreate( - validator_signature=msg.signature, - miner_signature=get_miner_signature(msg), job_uuid=msg.payload.job_uuid, - miner_hotkey=msg.payload.miner_hotkey, validator_hotkey=msg.payload.validator_hotkey, + miner_hotkey=msg.payload.miner_hotkey, + validator_signature=msg.signature, + miner_signature=get_miner_signature(msg), executor_class=msg.payload.executor_class, time_accepted=msg.payload.time_accepted, max_timeout=msg.payload.max_timeout, diff --git a/miner/app/src/compute_horde_miner/miner/models.py b/miner/app/src/compute_horde_miner/miner/models.py index 178f2f2d0..42d27a2da 100644 --- a/miner/app/src/compute_horde_miner/miner/models.py +++ b/miner/app/src/compute_horde_miner/miner/models.py @@ -1,14 +1,7 @@ from collections.abc import Iterable -from datetime import timedelta from enum import Enum from typing import Self -from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass -from compute_horde.mv_protocol.validator_requests import ( - JobFinishedReceiptPayload, - JobStartedReceiptPayload, -) -from compute_horde.receipts import Receipt from django.core.serializers.json import DjangoJSONEncoder from django.db import models @@ -92,65 +85,3 @@ async def get_not_reported(cls, validator: Validator) -> Iterable[Self]: result_reported_to_validator__isnull=True, ) ] - - -class AbstractReceipt(models.Model): - validator_signature = models.CharField(max_length=256) - miner_signature = models.CharField(max_length=256) - - # payload fields - job_uuid = models.UUIDField() - miner_hotkey = models.CharField(max_length=256) - validator_hotkey = models.CharField(max_length=256) - - def __str__(self): - return f"uuid: {self.job_uuid}" - - class Meta: - abstract = True - - -class JobFinishedReceipt(AbstractReceipt): - time_started = models.DateTimeField() - time_took_us = models.BigIntegerField() - score_str = models.CharField(max_length=256) - - def time_took(self): - return timedelta(microseconds=self.time_took_us) - - def score(self): - return float(self.score_str) - - def to_receipt(self): - return Receipt( - payload=JobFinishedReceiptPayload( - job_uuid=str(self.job_uuid), - miner_hotkey=self.miner_hotkey, - validator_hotkey=self.validator_hotkey, - time_started=self.time_started, - time_took_us=self.time_took_us, - score_str=self.score_str, - ), - validator_signature=self.validator_signature, - miner_signature=self.miner_signature, - ) - - -class JobStartedReceipt(AbstractReceipt): - executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) - time_accepted = models.DateTimeField() - max_timeout = models.IntegerField() - - def to_receipt(self): - return Receipt( - payload=JobStartedReceiptPayload( - job_uuid=str(self.job_uuid), - miner_hotkey=self.miner_hotkey, - validator_hotkey=self.validator_hotkey, - executor_class=ExecutorClass(self.executor_class), - time_accepted=self.time_accepted, - max_timeout=self.max_timeout, - ), - validator_signature=self.validator_signature, - miner_signature=self.miner_signature, - ) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index e5d3bc465..bcb8e6afe 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -2,6 +2,7 @@ from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt, ReceiptNotSigned from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, @@ -14,7 +15,7 @@ from compute_horde_miner.celery import app from compute_horde_miner.miner import quasi_axon -from compute_horde_miner.miner.models import JobFinishedReceipt, JobStartedReceipt, Validator +from compute_horde_miner.miner.models import Validator from compute_horde_miner.miner.receipt_store.current import receipts_store logger = get_task_logger(__name__) @@ -72,12 +73,22 @@ def prepare_receipts(): job_started_receipts = JobStartedReceipt.objects.order_by("time_accepted").filter( time_accepted__gt=now() - RECEIPTS_MAX_SERVED_PERIOD ) - receipts += [jr.to_receipt() for jr in job_started_receipts] + for job_started_receipt in job_started_receipts: + try: + receipts.append(job_started_receipt.to_receipt()) + except Exception as e: + logger.error(f"Skipping job started receipt for job {job_started_receipt.job_uuid}: {e}") job_finished_receipts = JobFinishedReceipt.objects.order_by("time_started").filter( time_started__gt=now() - RECEIPTS_MAX_SERVED_PERIOD ) - receipts += [jr.to_receipt() for jr in job_finished_receipts] + for job_finished_receipt in job_finished_receipts: + try: + receipts.append(job_finished_receipt.to_receipt()) + except Exception as e: + logger.error(f"Skipping job finished receipt for job {job_finished_receipt.job_uuid}: {e}") + + logger.info(f"Stored receipts: {len(receipts)}") receipts_store.store(receipts) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 7a2327ffd..816495145 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -2,6 +2,7 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS +from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, @@ -10,7 +11,6 @@ from django.utils.timezone import now from pytest_mock import MockerFixture -from compute_horde_miner.miner.models import JobFinishedReceipt, JobStartedReceipt from compute_horde_miner.miner.tasks import announce_address_and_port, get_receipts_from_old_miner diff --git a/miner/app/src/compute_horde_miner/settings.py b/miner/app/src/compute_horde_miner/settings.py index 59967e76a..89abda41e 100644 --- a/miner/app/src/compute_horde_miner/settings.py +++ b/miner/app/src/compute_horde_miner/settings.py @@ -78,6 +78,7 @@ def wrapped(*args, **kwargs): "django_extensions", "django_probes", "constance", + "compute_horde.mv_protocol", "compute_horde_miner.miner", "compute_horde_miner.miner.admin_config.MinerAdminConfig", ] From b76fa8003f1ad379a158aa593dc1232f6b4067bf Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:31:52 +0200 Subject: [PATCH 09/99] validator: use common receipt models --- .../src/compute_horde_validator/settings.py | 1 + .../validator/admin.py | 4 --- .../validator/models.py | 33 ------------------- .../validator/synthetic_jobs/batch_run.py | 5 +-- .../validator/tasks.py | 3 +- .../validator/tests/test_receipts.py | 3 +- 6 files changed, 6 insertions(+), 43 deletions(-) diff --git a/validator/app/src/compute_horde_validator/settings.py b/validator/app/src/compute_horde_validator/settings.py index 990a9634f..12b986690 100644 --- a/validator/app/src/compute_horde_validator/settings.py +++ b/validator/app/src/compute_horde_validator/settings.py @@ -68,6 +68,7 @@ def wrapped(*args, **kwargs): "django_extensions", "django_probes", "constance", + "compute_horde.mv_protocol", "compute_horde_validator.validator", "compute_horde_validator.validator.admin_config.ValidatorAdminConfig", "rangefilter", diff --git a/validator/app/src/compute_horde_validator/validator/admin.py b/validator/app/src/compute_horde_validator/validator/admin.py index 0325b215b..43e7ad06d 100644 --- a/validator/app/src/compute_horde_validator/validator/admin.py +++ b/validator/app/src/compute_horde_validator/validator/admin.py @@ -9,8 +9,6 @@ SyntheticJob, MinerBlacklist, AdminJobRequest, - JobFinishedReceipt, - JobStartedReceipt, SystemEvent, Weights, Prompt, @@ -188,8 +186,6 @@ class PromptAdmin(ReadOnlyAdmin): admin.site.register(Miner, admin_class=MinerReadOnlyAdmin) admin.site.register(SyntheticJob, admin_class=JobReadOnlyAdmin) admin.site.register(OrganicJob, admin_class=JobReadOnlyAdmin) -admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) -admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) admin.site.register(MinerBlacklist) admin.site.register(AdminJobRequest, admin_class=AdminJobRequestAddOnlyAdmin) admin.site.register(SystemEvent, admin_class=SystemEventAdmin) diff --git a/validator/app/src/compute_horde_validator/validator/models.py b/validator/app/src/compute_horde_validator/validator/models.py index 037da4145..416e9a03b 100644 --- a/validator/app/src/compute_horde_validator/validator/models.py +++ b/validator/app/src/compute_horde_validator/validator/models.py @@ -240,39 +240,6 @@ def __str__(self): return f"uuid: {self.uuid} - miner hotkey: {self.miner.hotkey}" -class AbstractReceipt(models.Model): - job_uuid = models.UUIDField() - miner_hotkey = models.CharField(max_length=256) - validator_hotkey = models.CharField(max_length=256) - - class Meta: - abstract = True - constraints = [ - UniqueConstraint(fields=["job_uuid"], name="unique_%(class)s_job_uuid"), - ] - - def __str__(self): - return f"job_uuid: {self.job_uuid}" - - -class JobFinishedReceipt(AbstractReceipt): - time_started = models.DateTimeField() - time_took_us = models.BigIntegerField() - score_str = models.CharField(max_length=256) - - def time_took(self): - return timedelta(microseconds=self.time_took_us) - - def score(self): - return float(self.score_str) - - -class JobStartedReceipt(AbstractReceipt): - executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) - time_accepted = models.DateTimeField() - max_timeout = models.IntegerField() - - def get_random_salt() -> list[int]: return list(urandom(8)) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index d52515194..0cd5eb4a4 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -35,6 +35,7 @@ V0JobFinishedRequest, V0MachineSpecsRequest, ) +from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, JobFinishedReceiptPayload, @@ -53,8 +54,6 @@ from compute_horde_validator.validator.dynamic_config import get_miner_max_executors_per_class from compute_horde_validator.validator.models import ( - JobFinishedReceipt, - JobStartedReceipt, Miner, MinerManifest, PromptSample, @@ -1513,6 +1512,7 @@ def _db_persist(ctx: BatchContext) -> None: job_uuid=started_payload.job_uuid, miner_hotkey=started_payload.miner_hotkey, validator_hotkey=started_payload.validator_hotkey, + validator_signature=job.job_started_receipt.signature, executor_class=started_payload.executor_class, time_accepted=started_payload.time_accepted, max_timeout=started_payload.max_timeout, @@ -1529,6 +1529,7 @@ def _db_persist(ctx: BatchContext) -> None: job_uuid=finished_payload.job_uuid, miner_hotkey=finished_payload.miner_hotkey, validator_hotkey=finished_payload.validator_hotkey, + validator_signature=job.job_finished_receipt.signature, time_started=finished_payload.time_started, time_took_us=finished_payload.time_took_us, score_str=finished_payload.score_str, diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 12bc6a679..6a6638b20 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,6 +19,7 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config +from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, @@ -38,8 +39,6 @@ from compute_horde_validator.validator.metagraph_client import get_miner_axon_info from compute_horde_validator.validator.models import ( Cycle, - JobFinishedReceipt, - JobStartedReceipt, OrganicJob, Prompt, PromptSample, diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 7e040c664..85624a232 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -3,6 +3,7 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS +from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, @@ -11,8 +12,6 @@ from django.utils.timezone import now from compute_horde_validator.validator.models import ( - JobFinishedReceipt, - JobStartedReceipt, SystemEvent, ) from compute_horde_validator.validator.tasks import fetch_receipts From a451ff3a6f9f7d53f864a15f47f1d998962ffd68 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:32:04 +0200 Subject: [PATCH 10/99] migrations --- ...inishedreceipt_delete_jobstartedreceipt.py | 19 +++++++++++++++++++ ...inishedreceipt_delete_jobstartedreceipt.py | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py create mode 100644 validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py diff --git a/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py b/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py new file mode 100644 index 000000000..f768d338b --- /dev/null +++ b/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.16 on 2024-10-10 11:53 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('miner', '0008_jobstartedreceipt_and_more'), + ] + + operations = [ + migrations.DeleteModel( + name='JobFinishedReceipt', + ), + migrations.DeleteModel( + name='JobStartedReceipt', + ), + ] diff --git a/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py b/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py new file mode 100644 index 000000000..994e9884b --- /dev/null +++ b/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.15 on 2024-10-09 17:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('validator', '0038_alter_systemevent_subtype_alter_systemevent_type'), + ] + + operations = [ + migrations.DeleteModel( + name='JobFinishedReceipt', + ), + migrations.DeleteModel( + name='JobStartedReceipt', + ), + ] From 7b3818cbb05c0cc4471247b66334961a6c092fb8 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:37:33 +0200 Subject: [PATCH 11/99] ruff --- compute_horde/compute_horde/mv_protocol/apps.py | 2 +- .../mv_protocol/migrations/0001_initial.py | 3 ++- compute_horde/compute_horde/mv_protocol/models.py | 5 ++++- compute_horde/settings.py | 4 +--- ...lete_jobfinishedreceipt_delete_jobstartedreceipt.py | 7 +++---- .../miner/miner_consumer/validator_interface.py | 4 +++- miner/app/src/compute_horde_miner/miner/tasks.py | 10 +++++++--- .../compute_horde_miner/miner/tests/test_migration.py | 2 +- ...lete_jobfinishedreceipt_delete_jobstartedreceipt.py | 7 +++---- .../src/compute_horde_validator/validator/models.py | 1 - .../validator/synthetic_jobs/batch_run.py | 2 +- .../app/src/compute_horde_validator/validator/tasks.py | 2 +- .../validator/tests/test_receipts.py | 2 +- 13 files changed, 28 insertions(+), 23 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/apps.py b/compute_horde/compute_horde/mv_protocol/apps.py index 16cf6561b..6f8d071cb 100644 --- a/compute_horde/compute_horde/mv_protocol/apps.py +++ b/compute_horde/compute_horde/mv_protocol/apps.py @@ -8,4 +8,4 @@ class ComputeHordeMVProtocolConfig(AppConfig): name = "compute_horde.mv_protocol" verbose_name = "Miner-Validator protocol dependencies" - default_auto_field = "django.db.models.BigAutoField" \ No newline at end of file + default_auto_field = "django.db.models.BigAutoField" diff --git a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py index b3e44e851..f977b0a39 100644 --- a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py +++ b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.16 on 2024-10-10 05:45 -import compute_horde.executor_class from django.db import migrations, models +import compute_horde.executor_class + class Migration(migrations.Migration): initial = True diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/mv_protocol/models.py index 193e1409c..040997e55 100644 --- a/compute_horde/compute_horde/mv_protocol/models.py +++ b/compute_horde/compute_horde/mv_protocol/models.py @@ -3,7 +3,10 @@ from django.db import models from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass -from compute_horde.mv_protocol.validator_requests import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.mv_protocol.validator_requests import ( + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) from compute_horde.receipts import Receipt diff --git a/compute_horde/settings.py b/compute_horde/settings.py index c29570020..8fe544535 100644 --- a/compute_horde/settings.py +++ b/compute_horde/settings.py @@ -1,3 +1 @@ -INSTALLED_APPS = [ - "compute_horde.mv_protocol" -] \ No newline at end of file +INSTALLED_APPS = ["compute_horde.mv_protocol"] diff --git a/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py b/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py index f768d338b..f8e924ff3 100644 --- a/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py +++ b/miner/app/src/compute_horde_miner/miner/migrations/0009_delete_jobfinishedreceipt_delete_jobstartedreceipt.py @@ -4,16 +4,15 @@ class Migration(migrations.Migration): - dependencies = [ - ('miner', '0008_jobstartedreceipt_and_more'), + ("miner", "0008_jobstartedreceipt_and_more"), ] operations = [ migrations.DeleteModel( - name='JobFinishedReceipt', + name="JobFinishedReceipt", ), migrations.DeleteModel( - name='JobStartedReceipt', + name="JobStartedReceipt", ), ] diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index 55f32bf7c..e38b61a15 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -8,7 +8,9 @@ import bittensor from compute_horde.mv_protocol import miner_requests, validator_requests from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.mv_protocol.validator_requests import BaseValidatorRequest, V0JobStartedReceiptRequest +from compute_horde.mv_protocol.validator_requests import ( + BaseValidatorRequest, +) from django.conf import settings from django.utils import timezone diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index bcb8e6afe..e3c9855fe 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -2,7 +2,7 @@ from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt, ReceiptNotSigned +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, @@ -77,7 +77,9 @@ def prepare_receipts(): try: receipts.append(job_started_receipt.to_receipt()) except Exception as e: - logger.error(f"Skipping job started receipt for job {job_started_receipt.job_uuid}: {e}") + logger.error( + f"Skipping job started receipt for job {job_started_receipt.job_uuid}: {e}" + ) job_finished_receipts = JobFinishedReceipt.objects.order_by("time_started").filter( time_started__gt=now() - RECEIPTS_MAX_SERVED_PERIOD @@ -86,7 +88,9 @@ def prepare_receipts(): try: receipts.append(job_finished_receipt.to_receipt()) except Exception as e: - logger.error(f"Skipping job finished receipt for job {job_finished_receipt.job_uuid}: {e}") + logger.error( + f"Skipping job finished receipt for job {job_finished_receipt.job_uuid}: {e}" + ) logger.info(f"Stored receipts: {len(receipts)}") diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 816495145..db5f87b7f 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -2,7 +2,7 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, diff --git a/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py b/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py index 994e9884b..30da800d1 100644 --- a/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py +++ b/validator/app/src/compute_horde_validator/validator/migrations/0039_delete_jobfinishedreceipt_delete_jobstartedreceipt.py @@ -4,16 +4,15 @@ class Migration(migrations.Migration): - dependencies = [ - ('validator', '0038_alter_systemevent_subtype_alter_systemevent_type'), + ("validator", "0038_alter_systemevent_subtype_alter_systemevent_type"), ] operations = [ migrations.DeleteModel( - name='JobFinishedReceipt', + name="JobFinishedReceipt", ), migrations.DeleteModel( - name='JobStartedReceipt', + name="JobStartedReceipt", ), ] diff --git a/validator/app/src/compute_horde_validator/validator/models.py b/validator/app/src/compute_horde_validator/validator/models.py index 416e9a03b..6e6a8b3de 100644 --- a/validator/app/src/compute_horde_validator/validator/models.py +++ b/validator/app/src/compute_horde_validator/validator/models.py @@ -1,7 +1,6 @@ import logging import shlex import uuid -from datetime import timedelta from os import urandom from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 0cd5eb4a4..895327b5b 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -35,7 +35,7 @@ V0JobFinishedRequest, V0MachineSpecsRequest, ) -from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, JobFinishedReceiptPayload, diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 6a6638b20..2e7fa8f0a 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,7 +19,7 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 85624a232..17f017649 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -3,7 +3,7 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.models import JobStartedReceipt, JobFinishedReceipt +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, From 620ffee1cb052e1eccbfd9dc15f2908ea084aba0 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:40:02 +0200 Subject: [PATCH 12/99] library: configure mypy <-> django --- compute_horde/README.md | 2 +- compute_horde/{ => compute_horde}/settings.py | 0 compute_horde/mypy.ini | 6 ++++++ 3 files changed, 7 insertions(+), 1 deletion(-) rename compute_horde/{ => compute_horde}/settings.py (100%) diff --git a/compute_horde/README.md b/compute_horde/README.md index 7d0b070c1..139a3a57d 100644 --- a/compute_horde/README.md +++ b/compute_horde/README.md @@ -14,5 +14,5 @@ INSTALLED_APPS = [ ## Migrations To make new migrations after doing some changes in the model files, run: ```shell -DJANGO_SETTINGS_MODULE=settings pdm run django-admin makemigrations +DJANGO_SETTINGS_MODULE=compute_horde.settings pdm run django-admin makemigrations ``` diff --git a/compute_horde/settings.py b/compute_horde/compute_horde/settings.py similarity index 100% rename from compute_horde/settings.py rename to compute_horde/compute_horde/settings.py diff --git a/compute_horde/mypy.ini b/compute_horde/mypy.ini index b2384fb23..0a95c8673 100644 --- a/compute_horde/mypy.ini +++ b/compute_horde/mypy.ini @@ -1,4 +1,7 @@ [mypy] +plugins = + mypy_django_plugin.main + exclude = ^(noxfile\.py|manage\.py|tests\/.*|compute_horde\/test_base\/.*)$ strict = true @@ -16,3 +19,6 @@ disallow_subclassing_any = False disallow_incomplete_defs = False disallow_untyped_defs = False disallow_untyped_calls = False + +[mypy.plugins.django-stubs] +django_settings_module = "compute_horde.settings" \ No newline at end of file From e3801a453671d93ba4a128503d634cb6b8b3c323 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 14:54:03 +0200 Subject: [PATCH 13/99] rename UniqueConstraint to avoid conflict with old models --- .../compute_horde/mv_protocol/migrations/0001_initial.py | 8 +++++--- compute_horde/compute_horde/mv_protocol/models.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py index f977b0a39..b90f65249 100644 --- a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py +++ b/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-10-10 05:45 +# Generated by Django 4.2.16 on 2024-10-10 07:53 from django.db import migrations, models @@ -78,13 +78,15 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="jobstartedreceipt", constraint=models.UniqueConstraint( - fields=("job_uuid",), name="unique_jobstartedreceipt_job_uuid" + fields=("job_uuid",), + name="mv_protocol_unique_jobstartedreceipt_job_uuid", ), ), migrations.AddConstraint( model_name="jobfinishedreceipt", constraint=models.UniqueConstraint( - fields=("job_uuid",), name="unique_jobfinishedreceipt_job_uuid" + fields=("job_uuid",), + name="mv_protocol_unique_jobfinishedreceipt_job_uuid", ), ), ] diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/mv_protocol/models.py index 040997e55..e3dad9f32 100644 --- a/compute_horde/compute_horde/mv_protocol/models.py +++ b/compute_horde/compute_horde/mv_protocol/models.py @@ -24,7 +24,9 @@ class AbstractReceipt(models.Model): class Meta: abstract = True constraints = [ - models.UniqueConstraint(fields=["job_uuid"], name="unique_%(class)s_job_uuid"), + models.UniqueConstraint( + fields=["job_uuid"], name="mv_protocol_unique_%(class)s_job_uuid" + ), ] def __str__(self): From d655a68ffe07848d7e2b838f2a4ca21b0e333a21 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:06:01 +0200 Subject: [PATCH 14/99] ci: check for missing library migrations --- .github/workflows/library_ci.yml | 2 ++ compute_horde/noxfile.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/library_ci.yml b/.github/workflows/library_ci.yml index 90a22cd31..d8056b3a7 100644 --- a/.github/workflows/library_ci.yml +++ b/.github/workflows/library_ci.yml @@ -28,6 +28,8 @@ jobs: run: python -m pip install --upgrade nox 'pdm>=2.12,<3' - name: Run linters run: nox -vs lint + - name: Check for missing migrations + run: nox -vs check_missing_migrations type_check: runs-on: ubuntu-latest defaults: diff --git a/compute_horde/noxfile.py b/compute_horde/noxfile.py index 3d4e2e791..c5eda7568 100644 --- a/compute_horde/noxfile.py +++ b/compute_horde/noxfile.py @@ -45,6 +45,14 @@ def type_check(session): session.run("mypy", "--config-file", "mypy.ini", ".", *session.posargs) +@nox.session(python=PYTHON_VERSION) +def check_missing_migrations(session): + install(session, "check_missing_migrations") + session.run("django-admin", "makemigrations", "--dry-run", "--check", env={ + "DJANGO_SETTINGS_MODULE": "compute_horde.settings" + }) + + @nox.session(python=PYTHON_VERSION) def test(session): install(session, "test") From 89285c8cfed4c02d4d8bab0b169ea49a1b7eec91 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:09:19 +0200 Subject: [PATCH 15/99] rufffffffffff --- compute_horde/noxfile.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compute_horde/noxfile.py b/compute_horde/noxfile.py index c5eda7568..6a11a9e5d 100644 --- a/compute_horde/noxfile.py +++ b/compute_horde/noxfile.py @@ -48,9 +48,13 @@ def type_check(session): @nox.session(python=PYTHON_VERSION) def check_missing_migrations(session): install(session, "check_missing_migrations") - session.run("django-admin", "makemigrations", "--dry-run", "--check", env={ - "DJANGO_SETTINGS_MODULE": "compute_horde.settings" - }) + session.run( + "django-admin", + "makemigrations", + "--dry-run", + "--check", + env={"DJANGO_SETTINGS_MODULE": "compute_horde.settings"}, + ) @nox.session(python=PYTHON_VERSION) From c662308680492485c51f4f81b6606bd992f0b5cd Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:28:46 +0200 Subject: [PATCH 16/99] re-add admin pages for receipts --- .../compute_horde/mv_protocol/admin.py | 44 +++++++++++++++++++ .../compute_horde/mv_protocol/apps.py | 2 +- .../validator/admin.py | 24 ---------- 3 files changed, 45 insertions(+), 25 deletions(-) create mode 100644 compute_horde/compute_horde/mv_protocol/admin.py diff --git a/compute_horde/compute_horde/mv_protocol/admin.py b/compute_horde/compute_horde/mv_protocol/admin.py new file mode 100644 index 000000000..d15327d5d --- /dev/null +++ b/compute_horde/compute_horde/mv_protocol/admin.py @@ -0,0 +1,44 @@ +from django.contrib import admin # noqa + +from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt + + +class AddOnlyAdmin(admin.ModelAdmin): + def has_change_permission(self, *args, **kwargs): + return False + + def has_delete_permission(self, *args, **kwargs): + return False + + +class ReadOnlyAdmin(AddOnlyAdmin): + def has_add_permission(self, *args, **kwargs): + return False + + +class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): + list_display = [ + "job_uuid", + "miner_hotkey", + "validator_hotkey", + "executor_class", + "time_accepted", + "max_timeout", + ] + ordering = ["-time_accepted"] + + +class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): + list_display = [ + "job_uuid", + "miner_hotkey", + "validator_hotkey", + "score", + "time_started", + "time_took", + ] + ordering = ["-time_started"] + + +admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) +admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) diff --git a/compute_horde/compute_horde/mv_protocol/apps.py b/compute_horde/compute_horde/mv_protocol/apps.py index 6f8d071cb..b8b364bb5 100644 --- a/compute_horde/compute_horde/mv_protocol/apps.py +++ b/compute_horde/compute_horde/mv_protocol/apps.py @@ -7,5 +7,5 @@ class ComputeHordeMVProtocolConfig(AppConfig): name = "compute_horde.mv_protocol" - verbose_name = "Miner-Validator protocol dependencies" + verbose_name = "Miner-Validator communication" default_auto_field = "django.db.models.BigAutoField" diff --git a/validator/app/src/compute_horde_validator/validator/admin.py b/validator/app/src/compute_horde_validator/validator/admin.py index 43e7ad06d..89952999e 100644 --- a/validator/app/src/compute_horde_validator/validator/admin.py +++ b/validator/app/src/compute_horde_validator/validator/admin.py @@ -118,30 +118,6 @@ class SystemEventAdmin(ReadOnlyAdmin): ordering = ["-timestamp"] -class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): - list_display = [ - "job_uuid", - "miner_hotkey", - "validator_hotkey", - "executor_class", - "time_accepted", - "max_timeout", - ] - ordering = ["-time_accepted"] - - -class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): - list_display = [ - "job_uuid", - "miner_hotkey", - "validator_hotkey", - "score", - "time_started", - "time_took", - ] - ordering = ["-time_started"] - - class WeightsReadOnlyAdmin(ReadOnlyAdmin): list_display = ["block", "created_at", "revealed_at"] ordering = ["-created_at"] From 64db23152ee4c4e4e83d4ca7391ed36e454dbb7f Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:35:13 +0200 Subject: [PATCH 17/99] library: ignore admin in mypy --- compute_horde/compute_horde/mv_protocol/admin.py | 7 ++----- compute_horde/mypy.ini | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/admin.py b/compute_horde/compute_horde/mv_protocol/admin.py index d15327d5d..80be7efb0 100644 --- a/compute_horde/compute_horde/mv_protocol/admin.py +++ b/compute_horde/compute_horde/mv_protocol/admin.py @@ -2,16 +2,13 @@ from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt - -class AddOnlyAdmin(admin.ModelAdmin): +class ReadOnlyAdmin(admin.ModelAdmin): def has_change_permission(self, *args, **kwargs): return False def has_delete_permission(self, *args, **kwargs): return False - -class ReadOnlyAdmin(AddOnlyAdmin): def has_add_permission(self, *args, **kwargs): return False @@ -40,5 +37,5 @@ class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): ordering = ["-time_started"] -admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) +admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) diff --git a/compute_horde/mypy.ini b/compute_horde/mypy.ini index 0a95c8673..0bbb9f093 100644 --- a/compute_horde/mypy.ini +++ b/compute_horde/mypy.ini @@ -2,7 +2,7 @@ plugins = mypy_django_plugin.main -exclude = ^(noxfile\.py|manage\.py|tests\/.*|compute_horde\/test_base\/.*)$ +exclude = ^(noxfile\.py|manage\.py|tests\/.*|compute_horde\/test_base\/.*|.*\/admin\.py)$ strict = true From 2270e3bca660d8fdf162b09e0952de84df347804 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:36:44 +0200 Subject: [PATCH 18/99] lint --- compute_horde/compute_horde/mv_protocol/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compute_horde/compute_horde/mv_protocol/admin.py b/compute_horde/compute_horde/mv_protocol/admin.py index 80be7efb0..43a51ed29 100644 --- a/compute_horde/compute_horde/mv_protocol/admin.py +++ b/compute_horde/compute_horde/mv_protocol/admin.py @@ -2,6 +2,7 @@ from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt + class ReadOnlyAdmin(admin.ModelAdmin): def has_change_permission(self, *args, **kwargs): return False From d7e2ef0bd18e4e405f2decb8b26341efe0e7ee03 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 15:55:27 +0200 Subject: [PATCH 19/99] common admin mixins --- compute_horde/compute_horde/base/admin.py | 17 ++++++ .../compute_horde/mv_protocol/admin.py | 16 +----- .../src/compute_horde_miner/miner/admin.py | 22 ++----- .../validator/admin.py | 57 ++++++++----------- 4 files changed, 48 insertions(+), 64 deletions(-) create mode 100644 compute_horde/compute_horde/base/admin.py diff --git a/compute_horde/compute_horde/base/admin.py b/compute_horde/compute_horde/base/admin.py new file mode 100644 index 000000000..7aac89957 --- /dev/null +++ b/compute_horde/compute_horde/base/admin.py @@ -0,0 +1,17 @@ +class AddOnlyAdminMixin: + def has_change_permission(self, *args, **kwargs): + return False + + def has_delete_permission(self, *args, **kwargs): + return False + + +class ReadOnlyAdminMixin: + def has_change_permission(self, *args, **kwargs): + return False + + def has_delete_permission(self, *args, **kwargs): + return False + + def has_add_permission(self, *args, **kwargs): + return False diff --git a/compute_horde/compute_horde/mv_protocol/admin.py b/compute_horde/compute_horde/mv_protocol/admin.py index 43a51ed29..a7a6dd058 100644 --- a/compute_horde/compute_horde/mv_protocol/admin.py +++ b/compute_horde/compute_horde/mv_protocol/admin.py @@ -1,20 +1,10 @@ from django.contrib import admin # noqa +from compute_horde.base.admin import ReadOnlyAdminMixin from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt -class ReadOnlyAdmin(admin.ModelAdmin): - def has_change_permission(self, *args, **kwargs): - return False - - def has_delete_permission(self, *args, **kwargs): - return False - - def has_add_permission(self, *args, **kwargs): - return False - - -class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): +class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "job_uuid", "miner_hotkey", @@ -26,7 +16,7 @@ class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): ordering = ["-time_accepted"] -class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): +class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "job_uuid", "miner_hotkey", diff --git a/miner/app/src/compute_horde_miner/miner/admin.py b/miner/app/src/compute_horde_miner/miner/admin.py index db4db58af..b6659b83f 100644 --- a/miner/app/src/compute_horde_miner/miner/admin.py +++ b/miner/app/src/compute_horde_miner/miner/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin # noqa from django.contrib.admin import register, AdminSite # noqa +from compute_horde.base.admin import ReadOnlyAdminMixin from compute_horde_miner.miner.models import ( AcceptedJob, Validator, @@ -15,24 +16,11 @@ admin.site.index_template = "admin/miner_index.html" -class ReadOnlyAdmin(admin.ModelAdmin): - change_form_template = "admin/read_only_view.html" - - def has_add_permission(self, *args, **kwargs): - return False - - def has_change_permission(self, *args, **kwargs): - return False - - def has_delete_permission(self, *args, **kwargs): - return False - - -class ValidatorReadOnlyAdmin(ReadOnlyAdmin): +class ValidatorReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): search_fields = ["public_key"] -class AcceptedJobReadOnlyAdmin(ReadOnlyAdmin): +class AcceptedJobReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "job_uuid", "validator", @@ -45,7 +33,7 @@ class AcceptedJobReadOnlyAdmin(ReadOnlyAdmin): ordering = ["-created_at"] -class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): +class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "job_uuid", "validator_hotkey", @@ -56,7 +44,7 @@ class JobStartedReceiptsReadOnlyAdmin(ReadOnlyAdmin): ordering = ["-time_accepted"] -class JobFinishedReceiptsReadOnlyAdmin(ReadOnlyAdmin): +class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = ["job_uuid", "validator_hotkey", "score", "time_started", "time_took"] ordering = ["-time_started"] diff --git a/validator/app/src/compute_horde_validator/validator/admin.py b/validator/app/src/compute_horde_validator/validator/admin.py index 89952999e..0a397157b 100644 --- a/validator/app/src/compute_horde_validator/validator/admin.py +++ b/validator/app/src/compute_horde_validator/validator/admin.py @@ -1,24 +1,26 @@ -from django.contrib import admin # noqa -from django.contrib import messages # noqa -from django.utils.safestring import mark_safe # noqa +from compute_horde.base.admin import AddOnlyAdminMixin, ReadOnlyAdminMixin +from compute_horde.executor_class import EXECUTOR_CLASS from django import forms +from django.contrib import ( + admin, # noqa + messages, # noqa +) +from django.utils.safestring import mark_safe # noqa +from rangefilter.filters import DateTimeRangeFilter from compute_horde_validator.validator.models import ( + AdminJobRequest, Miner, - OrganicJob, - SyntheticJob, MinerBlacklist, - AdminJobRequest, - SystemEvent, - Weights, + OrganicJob, Prompt, - PromptSeries, PromptSample, + PromptSeries, SolveWorkload, + SyntheticJob, + SystemEvent, + Weights, ) # noqa -from rangefilter.filters import DateTimeRangeFilter - -from compute_horde.executor_class import EXECUTOR_CLASS from compute_horde_validator.validator.tasks import trigger_run_admin_job_request # noqa admin.site.site_header = "ComputeHorde Validator Administration" @@ -28,19 +30,6 @@ admin.site.index_template = "admin/validator_index.html" -class AddOnlyAdmin(admin.ModelAdmin): - def has_change_permission(self, *args, **kwargs): - return False - - def has_delete_permission(self, *args, **kwargs): - return False - - -class ReadOnlyAdmin(AddOnlyAdmin): - def has_add_permission(self, *args, **kwargs): - return False - - class AdminJobRequestForm(forms.ModelForm): executor_class = forms.ChoiceField() @@ -68,7 +57,7 @@ def __init__(self, *args, **kwargs): self.fields["executor_class"].choices = [(name, name) for name in EXECUTOR_CLASS] -class AdminJobRequestAddOnlyAdmin(AddOnlyAdmin): +class AdminJobRequestAddOnlyAdmin(admin.ModelAdmin, AddOnlyAdminMixin): form = AdminJobRequestForm exclude = ["env"] # not used ? list_display = ["uuid", "executor_class", "docker_image", "use_gpu", "miner", "created_at"] @@ -88,13 +77,13 @@ def save_model(self, request, obj, form, change): messages.add_message(request, messages.INFO, mark_safe(msg)) -class JobReadOnlyAdmin(ReadOnlyAdmin): +class JobReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = ["job_uuid", "miner", "executor_class", "status", "updated_at"] search_fields = ["job_uuid", "miner__hotkey"] ordering = ["-updated_at"] -class MinerReadOnlyAdmin(ReadOnlyAdmin): +class MinerReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): change_form_template = "admin/read_only_view.html" search_fields = ["hotkey"] @@ -112,18 +101,18 @@ def get_search_results(self, request, queryset, search_term): return queryset, use_distinct -class SystemEventAdmin(ReadOnlyAdmin): +class SystemEventAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = ["type", "subtype", "timestamp"] list_filter = ["type", "subtype", ("timestamp", DateTimeRangeFilter)] ordering = ["-timestamp"] -class WeightsReadOnlyAdmin(ReadOnlyAdmin): +class WeightsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = ["block", "created_at", "revealed_at"] ordering = ["-created_at"] -class PromptSeriesAdmin(ReadOnlyAdmin): +class PromptSeriesAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "series_uuid", "s3_url", @@ -132,7 +121,7 @@ class PromptSeriesAdmin(ReadOnlyAdmin): ] -class SolveWorkloadAdmin(ReadOnlyAdmin): +class SolveWorkloadAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "workload_uuid", "seed", @@ -142,7 +131,7 @@ class SolveWorkloadAdmin(ReadOnlyAdmin): ] -class PromptSampleAdmin(ReadOnlyAdmin): +class PromptSampleAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "pk", "series", @@ -152,7 +141,7 @@ class PromptSampleAdmin(ReadOnlyAdmin): ] -class PromptAdmin(ReadOnlyAdmin): +class PromptAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): list_display = [ "pk", "sample", From 69089ab2bd484b71c7a3540958a407eca3e1d0ee Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 18:37:52 +0200 Subject: [PATCH 20/99] Use DockerRunOptionsPreset where applicable --- compute_horde/compute_horde/mv_protocol/validator_requests.py | 3 ++- .../executor/management/commands/run_executor.py | 4 +++- .../validator/synthetic_jobs/generator/base.py | 3 ++- .../validator/synthetic_jobs/generator/gpu_hashcat.py | 3 ++- .../validator/tests/test_synthetic_jobs/mock_generator.py | 3 ++- .../src/compute_horde_validator/validator/tests/test_utils.py | 3 ++- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 59b53cde9..dc25891b1 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -7,6 +7,7 @@ import pydantic from pydantic import field_serializer, model_validator +from ..base.docker import DockerRunOptionsPreset from ..base.output_upload import OutputUpload # noqa from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin @@ -69,7 +70,7 @@ class V0JobRequest(BaseValidatorRequest, JobMixin): executor_class: ExecutorClass | None = None docker_image_name: str | None = None raw_script: str | None = None - docker_run_options_preset: str + docker_run_options_preset: DockerRunOptionsPreset docker_run_cmd: list[str] volume: Volume | None = None output_upload: OutputUpload | None = None diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py index 9fcc96037..57bc120f0 100644 --- a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py +++ b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py @@ -16,6 +16,8 @@ import httpx import pydantic + +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import ( InlineVolume, MultiVolume, @@ -63,7 +65,7 @@ class RunConfigManager: @classmethod - def preset_to_docker_run_args(cls, preset: str) -> list[str]: + def preset_to_docker_run_args(cls, preset: DockerRunOptionsPreset) -> list[str]: if preset == "none": return [] elif preset == "nvidia_all": diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/base.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/base.py index 4f15a3455..6b4624cbd 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/base.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/base.py @@ -1,6 +1,7 @@ import abc import uuid +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.output_upload import OutputUpload from compute_horde.base.volume import Volume from compute_horde.executor_class import ExecutorClass @@ -30,7 +31,7 @@ def base_docker_image_name(self) -> str: ... def docker_image_name(self) -> str: ... @abc.abstractmethod - def docker_run_options_preset(self) -> str: ... + def docker_run_options_preset(self) -> DockerRunOptionsPreset: ... def docker_run_cmd(self) -> list[str]: return [] diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/gpu_hashcat.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/gpu_hashcat.py index 3ccec77ca..933ef0529 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/gpu_hashcat.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/generator/gpu_hashcat.py @@ -1,4 +1,5 @@ from asgiref.sync import sync_to_async +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import InlineVolume, Volume from compute_horde.mv_protocol.miner_requests import V0JobFinishedRequest @@ -89,7 +90,7 @@ def docker_image_name(self) -> str: else: raise RuntimeError(f"No docker_image for weights_version: {self.weights_version}") - def docker_run_options_preset(self) -> str: + def docker_run_options_preset(self) -> DockerRunOptionsPreset: return "nvidia_all" def docker_run_cmd(self) -> list[str]: diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py index d5d8a07e7..e1985ffbe 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py @@ -1,5 +1,6 @@ import uuid +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import InlineVolume, Volume from compute_horde.executor_class import ExecutorClass from compute_horde.mv_protocol.miner_requests import ( @@ -32,7 +33,7 @@ def base_docker_image_name(self) -> str: def docker_image_name(self) -> str: return "mock" - def docker_run_options_preset(self) -> str: + def docker_run_options_preset(self) -> DockerRunOptionsPreset: return "mock" def docker_run_cmd(self) -> list[str]: diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py index c1483885c..1fadfd1a9 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py @@ -7,6 +7,7 @@ import bittensor import pytest from asgiref.sync import sync_to_async +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import InlineVolume, Volume from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass from compute_horde.mv_protocol.miner_requests import ( @@ -70,7 +71,7 @@ def base_docker_image_name(self) -> str: def docker_image_name(self) -> str: return "mock" - def docker_run_options_preset(self) -> str: + def docker_run_options_preset(self) -> DockerRunOptionsPreset: return "mock" def docker_run_cmd(self) -> list[str]: From 79045593f07d0b3fc9f39e4252e4b029997ff019 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Thu, 10 Oct 2024 18:43:45 +0200 Subject: [PATCH 21/99] ruff & typing --- compute_horde/compute_horde/em_protocol/miner_requests.py | 3 ++- .../executor/management/commands/run_executor.py | 1 - .../compute_horde_miner/miner/miner_consumer/layer_utils.py | 3 ++- .../validator/organic_jobs/miner_driver.py | 5 ++++- .../validator/tests/test_synthetic_jobs/mock_generator.py | 2 +- .../compute_horde_validator/validator/tests/test_utils.py | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compute_horde/compute_horde/em_protocol/miner_requests.py b/compute_horde/compute_horde/em_protocol/miner_requests.py index ce106dac3..2125fe8b9 100644 --- a/compute_horde/compute_horde/em_protocol/miner_requests.py +++ b/compute_horde/compute_horde/em_protocol/miner_requests.py @@ -3,6 +3,7 @@ from pydantic import model_validator +from ..base.docker import DockerRunOptionsPreset from ..base.output_upload import OutputUpload, OutputUploadType # noqa from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin @@ -36,7 +37,7 @@ class V0JobRequest(BaseMinerRequest, JobMixin): message_type: RequestType = RequestType.V0RunJobRequest docker_image_name: str | None = None raw_script: str | None = None - docker_run_options_preset: str + docker_run_options_preset: DockerRunOptionsPreset docker_run_cmd: list[str] volume: Volume | None = None output_upload: OutputUpload | None = None diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py index 57bc120f0..1209d2083 100644 --- a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py +++ b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py @@ -16,7 +16,6 @@ import httpx import pydantic - from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import ( InlineVolume, diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/layer_utils.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/layer_utils.py index 59a8ccf6b..78298f580 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/layer_utils.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/layer_utils.py @@ -4,6 +4,7 @@ import pydantic from channels.generic.websocket import AsyncWebsocketConsumer +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.output_upload import OutputUpload from compute_horde.base.volume import Volume from compute_horde.mv_protocol import validator_requests @@ -29,7 +30,7 @@ class JobRequest(pydantic.BaseModel): job_uuid: str docker_image_name: str | None = None raw_script: str | None = None - docker_run_options_preset: str + docker_run_options_preset: DockerRunOptionsPreset docker_run_cmd: list[str] volume: Volume | None = None output_upload: OutputUpload | None = None diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index ab7030fb9..8680fcc70 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -5,6 +5,7 @@ from functools import partial from typing import Literal +from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.output_upload import ZipAndHttpPutUpload from compute_horde.base.volume import InlineVolume, Volume, ZipUrlVolume from compute_horde.executor_class import ExecutorClass @@ -235,7 +236,9 @@ async def handle_send_error_event(msg: str): else: raise ValueError(f"Unexpected msg from miner {miner_client.miner_name}: {msg}") - docker_run_options_preset = "nvidia_all" if job_request.use_gpu else "none" + docker_run_options_preset: DockerRunOptionsPreset = ( + "nvidia_all" if job_request.use_gpu else "none" + ) if isinstance(job_request, V0FacilitatorJobRequest | AdminJobRequest): if job_request.output_url: diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py index e1985ffbe..e2cb29bdc 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/mock_generator.py @@ -34,7 +34,7 @@ def docker_image_name(self) -> str: return "mock" def docker_run_options_preset(self) -> DockerRunOptionsPreset: - return "mock" + return "none" def docker_run_cmd(self) -> list[str]: return ["mock"] diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py index 1fadfd1a9..cacc49e40 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py @@ -72,7 +72,7 @@ def docker_image_name(self) -> str: return "mock" def docker_run_options_preset(self) -> DockerRunOptionsPreset: - return "mock" + return "none" def docker_run_cmd(self) -> list[str]: return ["mock"] From 1ae630fe34a1566476e803939df4883500246a33 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Fri, 11 Oct 2024 12:10:54 +0200 Subject: [PATCH 22/99] create receipts django app for the receipts --- compute_horde/README.md | 4 +-- .../compute_horde/mv_protocol/__init__.py | 1 - .../compute_horde/receipts/__init__.py | 1 + .../{mv_protocol => receipts}/admin.py | 2 +- .../{mv_protocol => receipts}/apps.py | 6 ++-- .../migrations/0001_initial.py | 4 +-- .../migrations/__init__.py | 0 .../{mv_protocol => receipts}/models.py | 6 ++-- .../compute_horde/receipts/pydantic.py | 31 +++++++++++++++++++ .../{receipts.py => receipts/transfer.py} | 29 ++++------------- compute_horde/pyproject.toml | 1 + compute_horde/tests/conftest.py | 2 +- compute_horde/tests/test_receipts.py | 6 +++- .../miner_consumer/validator_interface.py | 2 +- .../miner/receipt_store/base.py | 2 +- .../miner/receipt_store/local.py | 2 +- .../src/compute_horde_miner/miner/tasks.py | 4 +-- .../miner/tests/test_migration.py | 4 +-- miner/app/src/compute_horde_miner/settings.py | 2 +- .../src/compute_horde_validator/settings.py | 2 +- .../validator/synthetic_jobs/batch_run.py | 2 +- .../validator/tasks.py | 4 +-- .../validator/tests/test_receipts.py | 4 +-- 23 files changed, 69 insertions(+), 52 deletions(-) create mode 100644 compute_horde/compute_horde/receipts/__init__.py rename compute_horde/compute_horde/{mv_protocol => receipts}/admin.py (90%) rename compute_horde/compute_horde/{mv_protocol => receipts}/apps.py (51%) rename compute_horde/compute_horde/{mv_protocol => receipts}/migrations/0001_initial.py (95%) rename compute_horde/compute_horde/{mv_protocol => receipts}/migrations/__init__.py (100%) rename compute_horde/compute_horde/{mv_protocol => receipts}/models.py (94%) create mode 100644 compute_horde/compute_horde/receipts/pydantic.py rename compute_horde/compute_horde/{receipts.py => receipts/transfer.py} (80%) diff --git a/compute_horde/README.md b/compute_horde/README.md index 139a3a57d..1d5f7c7ab 100644 --- a/compute_horde/README.md +++ b/compute_horde/README.md @@ -1,12 +1,12 @@ # compute-horde ## Common django models -This library contains common Django models for the Validator <-> Miner communication. +This library contains common Django models for the receipts. To use them, update your `INSTALLED_APPS`: ```python INSTALLED_APPS = [ ..., - 'compute_horde.mv_protocol', + 'compute_horde.receipts', ..., ] ``` diff --git a/compute_horde/compute_horde/mv_protocol/__init__.py b/compute_horde/compute_horde/mv_protocol/__init__.py index 823734570..e69de29bb 100644 --- a/compute_horde/compute_horde/mv_protocol/__init__.py +++ b/compute_horde/compute_horde/mv_protocol/__init__.py @@ -1 +0,0 @@ -default_app_config = "compute_horde.mv_protocol.apps.ComputeHordeMVProtocolConfig" diff --git a/compute_horde/compute_horde/receipts/__init__.py b/compute_horde/compute_horde/receipts/__init__.py new file mode 100644 index 000000000..4fc57ee42 --- /dev/null +++ b/compute_horde/compute_horde/receipts/__init__.py @@ -0,0 +1 @@ +default_app_config = "compute_horde.receipts.apps.ComputeHordeReceiptsConfig" diff --git a/compute_horde/compute_horde/mv_protocol/admin.py b/compute_horde/compute_horde/receipts/admin.py similarity index 90% rename from compute_horde/compute_horde/mv_protocol/admin.py rename to compute_horde/compute_horde/receipts/admin.py index a7a6dd058..712a15f79 100644 --- a/compute_horde/compute_horde/mv_protocol/admin.py +++ b/compute_horde/compute_horde/receipts/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin # noqa from compute_horde.base.admin import ReadOnlyAdminMixin -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): diff --git a/compute_horde/compute_horde/mv_protocol/apps.py b/compute_horde/compute_horde/receipts/apps.py similarity index 51% rename from compute_horde/compute_horde/mv_protocol/apps.py rename to compute_horde/compute_horde/receipts/apps.py index b8b364bb5..31ebcb37b 100644 --- a/compute_horde/compute_horde/mv_protocol/apps.py +++ b/compute_horde/compute_horde/receipts/apps.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) -class ComputeHordeMVProtocolConfig(AppConfig): - name = "compute_horde.mv_protocol" - verbose_name = "Miner-Validator communication" +class ComputeHordeReceiptsConfig(AppConfig): + name = "compute_horde.receipts" + verbose_name = "Receipts" default_auto_field = "django.db.models.BigAutoField" diff --git a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py b/compute_horde/compute_horde/receipts/migrations/0001_initial.py similarity index 95% rename from compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py rename to compute_horde/compute_horde/receipts/migrations/0001_initial.py index b90f65249..468677598 100644 --- a/compute_horde/compute_horde/mv_protocol/migrations/0001_initial.py +++ b/compute_horde/compute_horde/receipts/migrations/0001_initial.py @@ -79,14 +79,14 @@ class Migration(migrations.Migration): model_name="jobstartedreceipt", constraint=models.UniqueConstraint( fields=("job_uuid",), - name="mv_protocol_unique_jobstartedreceipt_job_uuid", + name="receipts_unique_jobstartedreceipt_job_uuid", ), ), migrations.AddConstraint( model_name="jobfinishedreceipt", constraint=models.UniqueConstraint( fields=("job_uuid",), - name="mv_protocol_unique_jobfinishedreceipt_job_uuid", + name="receipts_unique_jobfinishedreceipt_job_uuid", ), ), ] diff --git a/compute_horde/compute_horde/mv_protocol/migrations/__init__.py b/compute_horde/compute_horde/receipts/migrations/__init__.py similarity index 100% rename from compute_horde/compute_horde/mv_protocol/migrations/__init__.py rename to compute_horde/compute_horde/receipts/migrations/__init__.py diff --git a/compute_horde/compute_horde/mv_protocol/models.py b/compute_horde/compute_horde/receipts/models.py similarity index 94% rename from compute_horde/compute_horde/mv_protocol/models.py rename to compute_horde/compute_horde/receipts/models.py index e3dad9f32..c69d40630 100644 --- a/compute_horde/compute_horde/mv_protocol/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -7,7 +7,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt +from compute_horde.receipts.pydantic import Receipt class ReceiptNotSigned(Exception): @@ -24,9 +24,7 @@ class AbstractReceipt(models.Model): class Meta: abstract = True constraints = [ - models.UniqueConstraint( - fields=["job_uuid"], name="mv_protocol_unique_%(class)s_job_uuid" - ), + models.UniqueConstraint(fields=["job_uuid"], name="receipts_unique_%(class)s_job_uuid"), ] def __str__(self): diff --git a/compute_horde/compute_horde/receipts/pydantic.py b/compute_horde/compute_horde/receipts/pydantic.py new file mode 100644 index 000000000..7d8cfc94d --- /dev/null +++ b/compute_horde/compute_horde/receipts/pydantic.py @@ -0,0 +1,31 @@ +import enum +import logging + +import bittensor +import pydantic + +from compute_horde.mv_protocol.validator_requests import ( + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) + +logger = logging.getLogger(__name__) + + +class ReceiptType(enum.Enum): + JobStartedReceipt = "JobStartedReceipt" + JobFinishedReceipt = "JobFinishedReceipt" + + +class Receipt(pydantic.BaseModel): + payload: JobStartedReceiptPayload | JobFinishedReceiptPayload + validator_signature: str + miner_signature: str + + def verify_miner_signature(self): + miner_keypair = bittensor.Keypair(ss58_address=self.payload.miner_hotkey) + return miner_keypair.verify(self.payload.blob_for_signing(), self.miner_signature) + + def verify_validator_signature(self): + validator_keypair = bittensor.Keypair(ss58_address=self.payload.validator_hotkey) + return validator_keypair.verify(self.payload.blob_for_signing(), self.validator_signature) diff --git a/compute_horde/compute_horde/receipts.py b/compute_horde/compute_horde/receipts/transfer.py similarity index 80% rename from compute_horde/compute_horde/receipts.py rename to compute_horde/compute_horde/receipts/transfer.py index 4444beee5..e803e1485 100644 --- a/compute_horde/compute_horde/receipts.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -1,41 +1,24 @@ import contextlib import csv import datetime -import enum import io import logging import shutil import tempfile -import bittensor import pydantic import requests -from .executor_class import ExecutorClass -from .mv_protocol.validator_requests import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.executor_class import ExecutorClass +from compute_horde.mv_protocol.validator_requests import ( + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) +from compute_horde.receipts.pydantic import Receipt, ReceiptType logger = logging.getLogger(__name__) -class ReceiptType(enum.Enum): - JobStartedReceipt = "JobStartedReceipt" - JobFinishedReceipt = "JobFinishedReceipt" - - -class Receipt(pydantic.BaseModel): - payload: JobStartedReceiptPayload | JobFinishedReceiptPayload - validator_signature: str - miner_signature: str - - def verify_miner_signature(self): - miner_keypair = bittensor.Keypair(ss58_address=self.payload.miner_hotkey) - return miner_keypair.verify(self.payload.blob_for_signing(), self.miner_signature) - - def verify_validator_signature(self): - validator_keypair = bittensor.Keypair(ss58_address=self.payload.validator_hotkey) - return validator_keypair.verify(self.payload.blob_for_signing(), self.validator_signature) - - class ReceiptFetchError(Exception): pass diff --git a/compute_horde/pyproject.toml b/compute_horde/pyproject.toml index b4f5c431e..641efa7cc 100644 --- a/compute_horde/pyproject.toml +++ b/compute_horde/pyproject.toml @@ -23,6 +23,7 @@ distribution = true [tool.pdm.build] includes = ["compute_horde"] +excludes = ["compute_horde/settings.py"] [tool.pdm.version] source = "scm" diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index 771d3856a..5d69c0f72 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -9,7 +9,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt +from compute_horde.receipts.pydantic import Receipt @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index 77178d716..5e0028425 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -7,7 +7,11 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt, ReceiptFetchError, ReceiptType, get_miner_receipts +from compute_horde.receipts.pydantic import ( + Receipt, + ReceiptType, +) +from compute_horde.receipts.transfer import ReceiptFetchError, get_miner_receipts def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index e38b61a15..ee74158cd 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -7,10 +7,10 @@ import bittensor from compute_horde.mv_protocol import miner_requests, validator_requests -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( BaseValidatorRequest, ) +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from django.conf import settings from django.utils import timezone diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/base.py b/miner/app/src/compute_horde_miner/miner/receipt_store/base.py index 940de8080..faadfdbc3 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/base.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/base.py @@ -1,6 +1,6 @@ import abc -from compute_horde.receipts import Receipt +from compute_horde.receipts.pydantic import Receipt class BaseReceiptStore(metaclass=abc.ABCMeta): diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py index 6a4328c68..664fca239 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py @@ -8,7 +8,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt, ReceiptType +from compute_horde.receipts.pydantic import Receipt, ReceiptType from django.conf import settings from compute_horde_miner.miner.receipt_store.base import BaseReceiptStore diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index e3c9855fe..8049a77f5 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -2,12 +2,12 @@ from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import get_miner_receipts +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import get_validators from constance import config from django.conf import settings diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index db5f87b7f..7cde30e31 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -2,12 +2,12 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.pydantic import Receipt from django.utils.timezone import now from pytest_mock import MockerFixture diff --git a/miner/app/src/compute_horde_miner/settings.py b/miner/app/src/compute_horde_miner/settings.py index 89abda41e..d097bf58c 100644 --- a/miner/app/src/compute_horde_miner/settings.py +++ b/miner/app/src/compute_horde_miner/settings.py @@ -78,7 +78,7 @@ def wrapped(*args, **kwargs): "django_extensions", "django_probes", "constance", - "compute_horde.mv_protocol", + "compute_horde.receipts", "compute_horde_miner.miner", "compute_horde_miner.miner.admin_config.MinerAdminConfig", ] diff --git a/validator/app/src/compute_horde_validator/settings.py b/validator/app/src/compute_horde_validator/settings.py index 12b986690..845426982 100644 --- a/validator/app/src/compute_horde_validator/settings.py +++ b/validator/app/src/compute_horde_validator/settings.py @@ -68,7 +68,7 @@ def wrapped(*args, **kwargs): "django_extensions", "django_probes", "constance", - "compute_horde.mv_protocol", + "compute_horde.receipts", "compute_horde_validator.validator", "compute_horde_validator.validator.admin_config.ValidatorAdminConfig", "rangefilter", diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 895327b5b..ecda12a22 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -35,7 +35,6 @@ V0JobFinishedRequest, V0MachineSpecsRequest, ) -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, JobFinishedReceiptPayload, @@ -46,6 +45,7 @@ V0JobRequest, V0JobStartedReceiptRequest, ) +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.transport import AbstractTransport, WSTransport from compute_horde.transport.base import TransportConnectionError from django.conf import settings diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 2e7fa8f0a..cc06e621e 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,12 +19,12 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import get_miner_receipts +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import ValidatorListError, get_validators from constance import config from django.conf import settings diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 17f017649..2f24434e6 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -3,12 +3,12 @@ import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts import Receipt +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.pydantic import Receipt from django.utils.timezone import now from compute_horde_validator.validator.models import ( From 5dc03b9f886c0805c1c3336a3c532b8cf4e718e5 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Fri, 11 Oct 2024 12:49:33 +0200 Subject: [PATCH 23/99] attempt to fix integration tests --- pdm.lock | 987 ++++++++++++++++++++------------------- validator/pdm.lock | 16 +- validator/pyproject.toml | 2 +- 3 files changed, 513 insertions(+), 492 deletions(-) diff --git a/pdm.lock b/pdm.lock index ec21e19b9..866206883 100644 --- a/pdm.lock +++ b/pdm.lock @@ -4,51 +4,55 @@ [metadata] groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] -lock_version = "4.4.1" -content_hash = "sha256:55cf121a9b42c1794822718e8abcc8bc4db76659b4c16765e7e450d3b6aa409b" +lock_version = "4.5.0" +content_hash = "sha256:fe31d35c711691d77c770334dc9d1c6b7062ce9447cd7d64bb9ffd9e8d1dd1a4" + +[[metadata.targets]] +requires_python = "==3.11.*" [[package]] name = "aiohappyeyeballs" -version = "2.4.0" +version = "2.4.3" requires_python = ">=3.8" summary = "Happy Eyeballs for asyncio" groups = ["dev"] files = [ - {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"}, - {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"}, + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, ] [[package]] name = "aiohttp" -version = "3.10.5" +version = "3.10.10" requires_python = ">=3.8" summary = "Async http client/server framework (asyncio)" groups = ["dev"] dependencies = [ "aiohappyeyeballs>=2.3.0", "aiosignal>=1.1.2", + "async-timeout<5.0,>=4.0; python_version < \"3.11\"", "attrs>=17.3.0", "frozenlist>=1.1.1", "multidict<7.0,>=4.5", - "yarl<2.0,>=1.0", + "yarl<2.0,>=1.12.0", ] files = [ - {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"}, - {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"}, - {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"}, - {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"}, - {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"}, - {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"}, - {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"}, - {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"}, - {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"}, - {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"}, - {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"}, - {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"}, - {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"}, - {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"}, - {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"}, - {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, + {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, + {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, + {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, ] [[package]] @@ -85,6 +89,9 @@ version = "0.7.0" requires_python = ">=3.8" summary = "Reusable constraint types to use with typing.Annotated" groups = ["dev"] +dependencies = [ + "typing-extensions>=4.0.0; python_version < \"3.9\"", +] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -92,34 +99,35 @@ files = [ [[package]] name = "ansible" -version = "6.7.0" -requires_python = ">=3.8" +version = "8.5.0" +requires_python = ">=3.9" summary = "Radically simple IT automation" groups = ["dev"] dependencies = [ - "ansible-core~=2.13.7", + "ansible-core~=2.15.5", ] files = [ - {file = "ansible-6.7.0-py3-none-any.whl", hash = "sha256:3cda6e67b1d42516f64ce376bb94c5186ff33d215d155432be5b3c2ec60bf112"}, - {file = "ansible-6.7.0.tar.gz", hash = "sha256:c188f3ac8a8583794aadcff0bea87895ead58c19d6f244cd0c342562706e176c"}, + {file = "ansible-8.5.0-py3-none-any.whl", hash = "sha256:2749032e26b0dbc9a694528b85fd89e7f950b8c7b53606f17dd997f23ac7cc88"}, + {file = "ansible-8.5.0.tar.gz", hash = "sha256:327c509bdaf5cdb2489d85c09d2c107e9432f9874c8bb5c0702a731160915f2d"}, ] [[package]] name = "ansible-core" -version = "2.13.13" -requires_python = ">=3.8" +version = "2.15.12" +requires_python = ">=3.9" summary = "Radically simple IT automation" groups = ["dev"] dependencies = [ "PyYAML>=5.1", "cryptography", + "importlib-resources<5.1,>=5.0; python_version < \"3.10\"", "jinja2>=3.0.0", "packaging", - "resolvelib<0.9.0,>=0.5.3", + "resolvelib<1.1.0,>=0.5.3", ] files = [ - {file = "ansible-core-2.13.13.tar.gz", hash = "sha256:7ad2d8c0a5fa4a59de1809a5f96d2dbf511189c834116f5c72aec9730b51074b"}, - {file = "ansible_core-2.13.13-py3-none-any.whl", hash = "sha256:f50220254b8e13a79b68e68e759f5bf89f3f3584c907737985a017c699b1c3b6"}, + {file = "ansible_core-2.15.12-py3-none-any.whl", hash = "sha256:390edd603420122f7cb1c470d8d1f8bdbbd795a1844dd03c1917db21935aecb9"}, + {file = "ansible_core-2.15.12.tar.gz", hash = "sha256:5fde82cd3928d9857ad880782c644f27d3168b0f25321d5a8d6befa524aa1818"}, ] [[package]] @@ -137,17 +145,19 @@ files = [ [[package]] name = "anyio" -version = "4.4.0" -requires_python = ">=3.8" +version = "4.6.0" +requires_python = ">=3.9" summary = "High level compatibility layer for multiple asynchronous event loop implementations" groups = ["dev"] dependencies = [ + "exceptiongroup>=1.0.2; python_version < \"3.11\"", "idna>=2.8", "sniffio>=1.1", + "typing-extensions>=4.1; python_version < \"3.11\"", ] files = [ - {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, - {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, + {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, + {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, ] [[package]] @@ -164,13 +174,13 @@ files = [ [[package]] name = "argcomplete" -version = "3.5.0" +version = "3.5.1" requires_python = ">=3.8" summary = "Bash tab completion for argparse" groups = ["dev"] files = [ - {file = "argcomplete-3.5.0-py3-none-any.whl", hash = "sha256:d4bcf3ff544f51e16e54228a7ac7f486ed70ebf2ecfe49a63a91171c76bf029b"}, - {file = "argcomplete-3.5.0.tar.gz", hash = "sha256:4349400469dccfb7950bb60334a680c58d88699bff6159df61251878dc6bf74b"}, + {file = "argcomplete-3.5.1-py3-none-any.whl", hash = "sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363"}, + {file = "argcomplete-3.5.1.tar.gz", hash = "sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4"}, ] [[package]] @@ -179,6 +189,9 @@ version = "3.8.1" requires_python = ">=3.8" summary = "ASGI specs, helper code, and adapters" groups = ["dev"] +dependencies = [ + "typing-extensions>=4; python_version < \"3.11\"", +] files = [ {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"}, {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, @@ -191,6 +204,7 @@ summary = "Annotate AST trees with source code positions" groups = ["dev"] dependencies = [ "six>=1.12.0", + "typing; python_version < \"3.5\"", ] files = [ {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, @@ -204,6 +218,9 @@ requires_python = ">=3.7" summary = "Timeout context manager for asyncio programs" groups = ["dev"] marker = "python_full_version <= \"3.11.2\"" +dependencies = [ + "typing-extensions>=3.6.5; python_version < \"3.8\"", +] files = [ {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, @@ -215,6 +232,9 @@ version = "24.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" groups = ["dev"] +dependencies = [ + "importlib-metadata; python_version < \"3.8\"", +] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -243,6 +263,9 @@ version = "24.8.1" requires_python = ">=3.8" summary = "Self-service finite-state machines for the programmer on the go." groups = ["dev"] +dependencies = [ + "typing-extensions; python_version < \"3.10\"", +] files = [ {file = "Automat-24.8.1-py3-none-any.whl", hash = "sha256:bf029a7bc3da1e2c24da2343e7598affaa9f10bf0ab63ff808566ce90551e02a"}, {file = "automat-24.8.1.tar.gz", hash = "sha256:b34227cf63f6325b8ad2399ede780675083e439b20c323d376373d8ee6306d88"}, @@ -282,28 +305,28 @@ files = [ [[package]] name = "billiard" -version = "4.2.0" +version = "4.2.1" requires_python = ">=3.7" summary = "Python multiprocessing fork with improvements and bugfixes" groups = ["dev"] files = [ - {file = "billiard-4.2.0-py3-none-any.whl", hash = "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d"}, - {file = "billiard-4.2.0.tar.gz", hash = "sha256:9a3c3184cb275aa17a732f93f65b20c525d3d9f253722d26a82194803ade5a2c"}, + {file = "billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb"}, + {file = "billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f"}, ] [[package]] name = "bittensor" -version = "7.3.1" +version = "7.4.0" requires_python = ">=3.9" summary = "bittensor" groups = ["dev"] dependencies = [ - "PyNaCl<=1.5.0,>=1.3.0", + "PyNaCl~=1.3", "aiohttp~=3.9", "ansible-vault~=2.1", - "ansible~=6.7", + "ansible~=8.5.0", "backoff", - "certifi~=2024.2.2", + "certifi~=2024.7.4", "colorama~=0.4.6", "cryptography~=42.0.5", "ddt~=1.6.0", @@ -314,28 +337,62 @@ dependencies = [ "munch~=2.5.0", "nest-asyncio", "netaddr", - "numpy", + "numpy~=1.26", "packaging", "password-strength", "pycryptodome<4.0.0,>=3.18.0", "pydantic<3,>=2.3", "python-Levenshtein", - "python-statemachine~=2.1.2", + "python-statemachine~=2.1", "pyyaml", "requests", "retry", "rich", "scalecodec==1.2.11", + "setuptools~=70.0.0", "shtab~=1.6.5", "substrate-interface~=1.7.9", "termcolor", "tqdm", - "uvicorn<=0.30", + "uvicorn", "wheel", ] files = [ - {file = "bittensor-7.3.1-py3-none-any.whl", hash = "sha256:67d2f375ca53b31e49b1802c6100de3c66df6d760ef562a3ddff63db408ce80b"}, - {file = "bittensor-7.3.1.tar.gz", hash = "sha256:0eb8de326c1f644d8b660e81919301f6ddccb0a824bce8f94c1b1a844624c26b"}, + {file = "bittensor-7.4.0-py3-none-any.whl", hash = "sha256:fefa7336f2a7f0dc1edea53a5b1296f95503d9e1cc01cb00a445e6c7afc10d1c"}, + {file = "bittensor-7.4.0.tar.gz", hash = "sha256:235b3f5bb3d93f098a41a4f63d7676bd548debb0bb58f02a104704c0beb20ea2"}, +] + +[[package]] +name = "boto3" +version = "1.35.38" +requires_python = ">=3.8" +summary = "The AWS SDK for Python" +groups = ["dev"] +dependencies = [ + "botocore<1.36.0,>=1.35.38", + "jmespath<2.0.0,>=0.7.1", + "s3transfer<0.11.0,>=0.10.0", +] +files = [ + {file = "boto3-1.35.38-py3-none-any.whl", hash = "sha256:234a475fe56b65e99b4f5cfff50adaac6b23d39558d6b55137bbf1e50dd0ef08"}, + {file = "boto3-1.35.38.tar.gz", hash = "sha256:90c8cddc4a08c8040057ad44c7468ff82fea9fe8b6517db5ff01a9b2900299cc"}, +] + +[[package]] +name = "botocore" +version = "1.35.38" +requires_python = ">=3.8" +summary = "Low-level, data-driven core of boto 3." +groups = ["dev"] +dependencies = [ + "jmespath<2.0.0,>=0.7.1", + "python-dateutil<3.0.0,>=2.1", + "urllib3!=2.2.0,<3,>=1.25.4; python_version >= \"3.10\"", + "urllib3<1.27,>=1.25.4; python_version < \"3.10\"", +] +files = [ + {file = "botocore-1.35.38-py3-none-any.whl", hash = "sha256:2eb17d32fa2d3bb5d475132a83564d28e3acc2161534f24b75a54418a1d51359"}, + {file = "botocore-1.35.38.tar.gz", hash = "sha256:55d9305c44e5ba29476df456120fa4fb919f03f066afa82f2ae400485e7465f4"}, ] [[package]] @@ -345,11 +402,13 @@ requires_python = ">=3.8" summary = "Distributed Task Queue." groups = ["dev"] dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", "billiard<5.0,>=4.2.0", "click-didyoumean>=0.3.0", "click-plugins>=1.1.1", "click-repl>=0.2.0", "click<9.0,>=8.1.2", + "importlib-metadata>=3.6; python_version < \"3.8\"", "kombu<6.0,>=5.3.4", "python-dateutil>=2.8.2", "tzdata>=2022.7", @@ -362,18 +421,18 @@ files = [ [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.7.4" requires_python = ">=3.6" summary = "Python package for providing Mozilla's CA Bundle." groups = ["dev"] files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] name = "cffi" -version = "1.17.0" +version = "1.17.1" requires_python = ">=3.8" summary = "Foreign Function Interface for Python calling C code." groups = ["dev"] @@ -381,19 +440,19 @@ dependencies = [ "pycparser", ] files = [ - {file = "cffi-1.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720"}, - {file = "cffi-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb"}, - {file = "cffi-1.17.0-cp311-cp311-win32.whl", hash = "sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9"}, - {file = "cffi-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0"}, - {file = "cffi-1.17.0.tar.gz", hash = "sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [[package]] @@ -446,28 +505,28 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.3.2" +version = "3.4.0" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." groups = ["dev"] files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] [[package]] @@ -478,6 +537,7 @@ summary = "Composable command line interface toolkit" groups = ["dev"] dependencies = [ "colorama; platform_system == \"Windows\"", + "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -553,7 +613,7 @@ files = [ [[package]] name = "compute-horde" -version = "0.0.10.dev136" +version = "0.0.11.dev120" requires_python = "==3.11.*" editable = true path = "./compute_horde" @@ -611,21 +671,13 @@ files = [ {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, ] [[package]] name = "cytoolz" -version = "0.12.3" -requires_python = ">=3.7" +version = "1.0.0" +requires_python = ">=3.8" summary = "Cython implementation of Toolz: High performance functional utilities" groups = ["dev"] marker = "implementation_name == \"cpython\"" @@ -633,41 +685,21 @@ dependencies = [ "toolz>=0.8.0", ] files = [ - {file = "cytoolz-0.12.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ac4f2fb38bbc67ff1875b7d2f0f162a247f43bd28eb7c9d15e6175a982e558d"}, - {file = "cytoolz-0.12.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0cf1e1e96dd86829a0539baf514a9c8473a58fbb415f92401a68e8e52a34ecd5"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08a438701c6141dd34eaf92e9e9a1f66e23a22f7840ef8a371eba274477de85d"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6b6f11b0d7ed91be53166aeef2a23a799e636625675bb30818f47f41ad31821"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7fde09384d23048a7b4ac889063761e44b89a0b64015393e2d1d21d5c1f534a"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d3bfe45173cc8e6c76206be3a916d8bfd2214fb2965563e288088012f1dabfc"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27513a5d5b6624372d63313574381d3217a66e7a2626b056c695179623a5cb1a"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d294e5e81ff094fe920fd545052ff30838ea49f9e91227a55ecd9f3ca19774a0"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:727b01a2004ddb513496507a695e19b5c0cfebcdfcc68349d3efd92a1c297bf4"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:fe1e1779a39dbe83f13886d2b4b02f8c4b10755e3c8d9a89b630395f49f4f406"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:de74ef266e2679c3bf8b5fc20cee4fc0271ba13ae0d9097b1491c7a9bcadb389"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e04d22049233394e0b08193aca9737200b4a2afa28659d957327aa780ddddf2"}, - {file = "cytoolz-0.12.3-cp311-cp311-win32.whl", hash = "sha256:20d36430d8ac809186736fda735ee7d595b6242bdb35f69b598ef809ebfa5605"}, - {file = "cytoolz-0.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:780c06110f383344d537f48d9010d79fa4f75070d214fc47f389357dd4f010b6"}, - {file = "cytoolz-0.12.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:55f9bd1ae6c2a27eda5abe2a0b65a83029d2385c5a1da7b8ef47af5905d7e905"}, - {file = "cytoolz-0.12.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2d271393c378282727f1231d40391ae93b93ddc0997448acc21dd0cb6a1e56d"}, - {file = "cytoolz-0.12.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee98968d6a66ee83a8ceabf31182189ab5d8598998c8ce69b6d5843daeb2db60"}, - {file = "cytoolz-0.12.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01cfb8518828c1189200c02a5010ea404407fb18fd5589e29c126e84bbeadd36"}, - {file = "cytoolz-0.12.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:456395d7aec01db32bf9e6db191d667347c78d8d48e77234521fa1078f60dabb"}, - {file = "cytoolz-0.12.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cd88028bb897fba99ddd84f253ca6bef73ecb7bdf3f3cf25bc493f8f97d3c7c5"}, - {file = "cytoolz-0.12.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59b19223e7f7bd7a73ec3aa6fdfb73b579ff09c2bc0b7d26857eec2d01a58c76"}, - {file = "cytoolz-0.12.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a79d72b08048a0980a59457c239555f111ac0c8bdc140c91a025f124104dbb4"}, - {file = "cytoolz-0.12.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dd70141b32b717696a72b8876e86bc9c6f8eff995c1808e299db3541213ff82"}, - {file = "cytoolz-0.12.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a1445c91009eb775d479e88954c51d0b4cf9a1e8ce3c503c2672d17252882647"}, - {file = "cytoolz-0.12.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ca6a9a9300d5bda417d9090107c6d2b007683efc59d63cc09aca0e7930a08a85"}, - {file = "cytoolz-0.12.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be6feb903d2a08a4ba2e70e950e862fd3be9be9a588b7c38cee4728150a52918"}, - {file = "cytoolz-0.12.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92b6f43f086e5a965d33d62a145ae121b4ccb6e0789ac0acc895ce084fec8c65"}, - {file = "cytoolz-0.12.3-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:534fa66db8564d9b13872d81d54b6b09ae592c585eb826aac235bd6f1830f8ad"}, - {file = "cytoolz-0.12.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:fea649f979def23150680de1bd1d09682da3b54932800a0f90f29fc2a6c98ba8"}, - {file = "cytoolz-0.12.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a447247ed312dd64e3a8d9483841ecc5338ee26d6e6fbd29cd373ed030db0240"}, - {file = "cytoolz-0.12.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba3f843aa89f35467b38c398ae5b980a824fdbdb94065adc6ec7c47a0a22f4c7"}, - {file = "cytoolz-0.12.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:582c22f97a380211fb36a7b65b1beeb84ea11d82015fa84b054be78580390082"}, - {file = "cytoolz-0.12.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47feb089506fc66e1593cd9ade3945693a9d089a445fbe9a11385cab200b9f22"}, - {file = "cytoolz-0.12.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ba9002d2f043943744a9dc8e50a47362bcb6e6f360dc0a1abcb19642584d87bb"}, - {file = "cytoolz-0.12.3.tar.gz", hash = "sha256:4503dc59f4ced53a54643272c61dc305d1dbbfbd7d6bdf296948de9f34c3a282"}, + {file = "cytoolz-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dffc22fd2c91be64dbdbc462d0786f8e8ac9a275cfa1869a1084d1867d4f67e0"}, + {file = "cytoolz-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a99e7e29274e293f4ffe20e07f76c2ac753a78f1b40c1828dfc54b2981b2f6c4"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c507a3e0a45c41d66b43f96797290d75d1e7a8549aa03a4a6b8854fdf3f7b8d8"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:643a593ec272ef7429099e1182a22f64ec2696c00d295d2a5be390db1b7ff176"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ce38e2e42cbae30446190c59b92a8a9029e1806fd79eaf88f48b0fe33003893"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810a6a168b8c5ecb412fbae3dd6f7ed6c6253a63caf4174ee9794ebd29b2224f"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ce8a2a85c0741c1b19b16e6782c4a5abc54c3caecda66793447112ab2fa9884"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ea4ac72e6b830861035c4c7999af8e55813f57c6d1913a3d93cc4a6babc27bf7"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a09cdfb21dfb38aa04df43e7546a41f673377eb5485da88ceb784e327ec7603b"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:658dd85deb375ff7af990a674e5c9058cef1c9d1f5dc89bc87b77be499348144"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9715d1ff5576919d10b68f17241375f6a1eec8961c25b78a83e6ef1487053f39"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f370a1f1f1afc5c1c8cc5edc1cfe0ba444263a0772af7ce094be8e734f41769d"}, + {file = "cytoolz-1.0.0-cp311-cp311-win32.whl", hash = "sha256:dbb2ec1177dca700f3db2127e572da20de280c214fc587b2a11c717fc421af56"}, + {file = "cytoolz-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0983eee73df86e54bb4a79fcc4996aa8b8368fdbf43897f02f9c3bf39c4dc4fb"}, + {file = "cytoolz-1.0.0.tar.gz", hash = "sha256:eb453b30182152f9917a5189b7d99046b6ce90cdf8aeb0feff4b2683e600defd"}, ] [[package]] @@ -691,6 +723,9 @@ name = "ddt" version = "1.6.0" summary = "Data-Driven/Decorated Tests" groups = ["dev"] +dependencies = [ + "enum34; python_version < \"3\"", +] files = [ {file = "ddt-1.6.0-py2.py3-none-any.whl", hash = "sha256:e3c93b961a108b4f4d5a6c7f2263513d928baf3bb5b32af8e1c804bfb041141d"}, {file = "ddt-1.6.0.tar.gz", hash = "sha256:f71b348731b8c78c3100bffbd951a769fbd439088d1fdbb3841eee019af80acd"}, @@ -709,28 +744,29 @@ files = [ [[package]] name = "distlib" -version = "0.3.8" +version = "0.3.9" summary = "Distribution utilities" groups = ["dev"] files = [ - {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, - {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, + {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, + {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, ] [[package]] name = "django" -version = "4.2.15" +version = "4.2.16" requires_python = ">=3.8" summary = "A high-level Python web framework that encourages rapid development and clean, pragmatic design." groups = ["dev"] dependencies = [ "asgiref<4,>=3.6.0", + "backports-zoneinfo; python_version < \"3.9\"", "sqlparse>=0.3.1", "tzdata; sys_platform == \"win32\"", ] files = [ - {file = "Django-4.2.15-py3-none-any.whl", hash = "sha256:61ee4a130efb8c451ef3467c67ca99fdce400fedd768634efc86a68c18d80d30"}, - {file = "Django-4.2.15.tar.gz", hash = "sha256:c77f926b81129493961e19c0e02188f8d07c112a1162df69bfab178ae447f94a"}, + {file = "Django-4.2.16-py3-none-any.whl", hash = "sha256:1ddc333a16fc139fd253035a1606bb24261951bbc3a6ca256717fa06cc41a898"}, + {file = "Django-4.2.16.tar.gz", hash = "sha256:6f1616c2786c408ce86ab7e10f792b8f15742f7b7b7460243929cb371e7f1dad"}, ] [[package]] @@ -939,6 +975,7 @@ requires_python = ">=3.7,<4" summary = "eth-utils: Common utility functions for python code that interacts with Ethereum" groups = ["dev"] dependencies = [ + "cached-property<2,>=1.5.2; python_version < \"3.8\"", "cytoolz>=0.10.1; implementation_name == \"cpython\"", "eth-hash>=0.3.1", "eth-typing>=3.0.0", @@ -962,13 +999,13 @@ files = [ [[package]] name = "executing" -version = "2.0.1" -requires_python = ">=3.5" +version = "2.1.0" +requires_python = ">=3.8" summary = "Get the currently executing AST node of a frame, and other information" groups = ["dev"] files = [ - {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, - {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, + {file = "executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf"}, + {file = "executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"}, ] [[package]] @@ -1021,13 +1058,13 @@ files = [ [[package]] name = "filelock" -version = "3.15.4" +version = "3.16.1" requires_python = ">=3.8" summary = "A platform independent file lock." groups = ["dev"] files = [ - {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, - {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [[package]] @@ -1104,6 +1141,9 @@ version = "0.14.0" requires_python = ">=3.7" summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" groups = ["dev"] +dependencies = [ + "typing-extensions; python_version < \"3.8\"", +] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1111,7 +1151,7 @@ files = [ [[package]] name = "httpcore" -version = "1.0.5" +version = "1.0.6" requires_python = ">=3.8" summary = "A minimal low-level HTTP client." groups = ["dev"] @@ -1120,8 +1160,8 @@ dependencies = [ "h11<0.15,>=0.13", ] files = [ - {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, - {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, + {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, + {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, ] [[package]] @@ -1144,13 +1184,13 @@ files = [ [[package]] name = "humanize" -version = "4.10.0" -requires_python = ">=3.8" +version = "4.11.0" +requires_python = ">=3.9" summary = "Python humanize utilities" groups = ["dev"] files = [ - {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, - {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, + {file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"}, + {file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"}, ] [[package]] @@ -1161,6 +1201,7 @@ summary = "A featureful, immutable, and correct URL for Python." groups = ["dev"] dependencies = [ "idna>=2.5", + "typing; python_version < \"3.5\"", ] files = [ {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"}, @@ -1169,13 +1210,13 @@ files = [ [[package]] name = "idna" -version = "3.7" -requires_python = ">=3.5" +version = "3.10" +requires_python = ">=3.6" summary = "Internationalized Domain Names in Applications (IDNA)" groups = ["dev"] files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] [[package]] @@ -1186,6 +1227,7 @@ summary = "A small library that versions your Python projects." groups = ["dev"] dependencies = [ "setuptools>=61.0", + "tomli; python_version < \"3.11\"", ] files = [ {file = "incremental-24.7.2-py3-none-any.whl", hash = "sha256:8cb2c3431530bec48ad70513931a760f446ad6c25e8333ca5d95e24b0ed7b8fe"}, @@ -1222,6 +1264,7 @@ dependencies = [ "pygments>=2.4.0", "stack-data", "traitlets>=5", + "typing-extensions; python_version < \"3.10\"", ] files = [ {file = "ipython-8.14.0-py3-none-any.whl", hash = "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"}, @@ -1256,58 +1299,60 @@ files = [ {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, ] +[[package]] +name = "jmespath" +version = "1.0.1" +requires_python = ">=3.7" +summary = "JSON Matching Expressions" +groups = ["dev"] +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "kombu" -version = "5.4.0" +version = "5.4.2" requires_python = ">=3.8" summary = "Messaging library for Python." groups = ["dev"] dependencies = [ "amqp<6.0.0,>=5.1.1", + "backports-zoneinfo[tzdata]>=0.2.1; python_version < \"3.9\"", + "typing-extensions==4.12.2; python_version < \"3.10\"", + "tzdata; python_version >= \"3.9\"", "vine==5.1.0", ] files = [ - {file = "kombu-5.4.0-py3-none-any.whl", hash = "sha256:c8dd99820467610b4febbc7a9e8a0d3d7da2d35116b67184418b51cc520ea6b6"}, - {file = "kombu-5.4.0.tar.gz", hash = "sha256:ad200a8dbdaaa2bbc5f26d2ee7d707d9a1fded353a0f4bd751ce8c7d9f449c60"}, + {file = "kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763"}, + {file = "kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf"}, ] [[package]] name = "levenshtein" -version = "0.25.1" -requires_python = ">=3.8" +version = "0.26.0" +requires_python = ">=3.9" summary = "Python extension for computing string edit distances and similarities." groups = ["dev"] dependencies = [ - "rapidfuzz<4.0.0,>=3.8.0", -] -files = [ - {file = "Levenshtein-0.25.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f57d9cf06dac55c2d2f01f0d06e32acc074ab9a902921dc8fddccfb385053ad5"}, - {file = "Levenshtein-0.25.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:22b60c6d791f4ca67a3686b557ddb2a48de203dae5214f220f9dddaab17f44bb"}, - {file = "Levenshtein-0.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d0444ee62eccf1e6cedc7c5bc01a9face6ff70cc8afa3f3ca9340e4e16f601a4"}, - {file = "Levenshtein-0.25.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e8758be8221a274c83924bae8dd8f42041792565a3c3bdd3c10e3f9b4a5f94e"}, - {file = "Levenshtein-0.25.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:147221cfb1d03ed81d22fdd2a4c7fc2112062941b689e027a30d2b75bbced4a3"}, - {file = "Levenshtein-0.25.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a454d5bc4f4a289f5471418788517cc122fcc00d5a8aba78c54d7984840655a2"}, - {file = "Levenshtein-0.25.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c25f3778bbac78286bef2df0ca80f50517b42b951af0a5ddaec514412f79fac"}, - {file = "Levenshtein-0.25.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:181486cf465aff934694cc9a19f3898a1d28025a9a5f80fc1608217e7cd1c799"}, - {file = "Levenshtein-0.25.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b8db9f672a5d150706648b37b044dba61f36ab7216c6a121cebbb2899d7dfaa3"}, - {file = "Levenshtein-0.25.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f2a69fe5ddea586d439f9a50d0c51952982f6c0db0e3573b167aa17e6d1dfc48"}, - {file = "Levenshtein-0.25.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:3b684675a3bd35efa6997856e73f36c8a41ef62519e0267dcbeefd15e26cae71"}, - {file = "Levenshtein-0.25.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:cc707ef7edb71f6bf8339198b929ead87c022c78040e41668a4db68360129cef"}, - {file = "Levenshtein-0.25.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:41512c436b8c691326e2d07786d906cba0e92b5e3f455bf338befb302a0ca76d"}, - {file = "Levenshtein-0.25.1-cp311-cp311-win32.whl", hash = "sha256:2a3830175c01ade832ba0736091283f14a6506a06ffe8c846f66d9fbca91562f"}, - {file = "Levenshtein-0.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:9e0af4e6e023e0c8f79af1d1ca5f289094eb91201f08ad90f426d71e4ae84052"}, - {file = "Levenshtein-0.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:38e5d9a1d737d7b49fa17d6a4c03a0359288154bf46dc93b29403a9dd0cd1a7d"}, - {file = "Levenshtein-0.25.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e9e060ef3925a68aeb12276f0e524fb1264592803d562ec0306c7c3f5c68eae0"}, - {file = "Levenshtein-0.25.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f84b84049318d44722db307c448f9dcb8d27c73525a378e901189a94889ba61"}, - {file = "Levenshtein-0.25.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07e23fdf330cb185a0c7913ca5bd73a189dfd1742eae3a82e31ed8688b191800"}, - {file = "Levenshtein-0.25.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d06958e4a81ea0f0b2b7768a2ad05bcd50a9ad04c4d521dd37d5730ff12decdc"}, - {file = "Levenshtein-0.25.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2ea7c34ec22b2fce21299b0caa6dde6bdebafcc2970e265853c9cfea8d1186da"}, - {file = "Levenshtein-0.25.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fddc0ccbdd94f57aa32e2eb3ac8310d08df2e175943dc20b3e1fc7a115850af4"}, - {file = "Levenshtein-0.25.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d52249cb3448bfe661d3d7db3a6673e835c7f37b30b0aeac499a1601bae873d"}, - {file = "Levenshtein-0.25.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8dd4c201b15f8c1e612f9074335392c8208ac147acbce09aff04e3974bf9b16"}, - {file = "Levenshtein-0.25.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23a4d95ce9d44161c7aa87ab76ad6056bc1093c461c60c097054a46dc957991f"}, - {file = "Levenshtein-0.25.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:65eea8a9c33037b23069dca4b3bc310e3c28ca53f60ec0c958d15c0952ba39fa"}, - {file = "Levenshtein-0.25.1.tar.gz", hash = "sha256:2df14471c778c75ffbd59cb64bbecfd4b0ef320ef9f80e4804764be7d5678980"}, + "rapidfuzz<4.0.0,>=3.9.0", +] +files = [ + {file = "levenshtein-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7aabafb951b96ca6e0f981b1edb3ec81b41c010b7437758e275393768fa84453"}, + {file = "levenshtein-0.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cefa552c5190e912f0fe39b62a5b08597d1256f330ed2c459ba724947458282"}, + {file = "levenshtein-0.26.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a0ed02c8aaef0903b72fe0da88d9d24d7964b07dbc123997e549ac165efad8d"}, + {file = "levenshtein-0.26.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5d21d3b08ceb7e544fae04897e211e43fb3500c9b3a8e74d08468b015c7270d"}, + {file = "levenshtein-0.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77422c5da5cfd8455a8835329d965e24250b0f0c1398e0a6362879f00d18f27c"}, + {file = "levenshtein-0.26.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4931c8263e06edbece310b1f8e03bfcb74f211863a85058b46cdf8460a4136af"}, + {file = "levenshtein-0.26.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3e265812db8b04e6ae159751c7a82d6e0e5025223bd330fc9104a8a5beeeb7cf"}, + {file = "levenshtein-0.26.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8c58cb6c9f90f9b11d6b478e2da6ac1f0bcb5ea9608a5611088d30f782ee5920"}, + {file = "levenshtein-0.26.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:754669e9b82795cfc3ca0d70f2e715b58ff4d0f7e7f4e77fc6539543439ae22c"}, + {file = "levenshtein-0.26.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:830b1993e3e945b213a6b73ceca8b555147a6ecd7323e4959b80dee35abfc7fc"}, + {file = "levenshtein-0.26.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2b3aa9a0b844ddaeb6f5317eb4e85b5748901cf40c9a9b0d3a8bf76ef9d3cccc"}, + {file = "levenshtein-0.26.0-cp311-cp311-win32.whl", hash = "sha256:07ffd78569ca80adfd55172156faac35eb12ccd375d6d51ba4512b0346337cbf"}, + {file = "levenshtein-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:1bf886afed2304e0022c33ed644afb719411cce4d4af11ba5bb040f05d9f00c1"}, + {file = "levenshtein-0.26.0-cp311-cp311-win_arm64.whl", hash = "sha256:c68b6e6c74ce9056370559196177b9e514ba20611a1ce9545dcd366d8a97cc60"}, + {file = "levenshtein-0.26.0.tar.gz", hash = "sha256:960b020d96bbd348400d6ff5c16290adee49f0ae2d42707a550a3b4f7d092abe"}, ] [[package]] @@ -1326,22 +1371,22 @@ files = [ [[package]] name = "markupsafe" -version = "2.1.5" -requires_python = ">=3.7" +version = "3.0.1" +requires_python = ">=3.9" summary = "Safely add untrusted strings to HTML/XML markup." groups = ["dev"] files = [ - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, + {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, ] [[package]] @@ -1403,34 +1448,34 @@ dependencies = [ [[package]] name = "more-itertools" -version = "10.4.0" +version = "10.5.0" requires_python = ">=3.8" summary = "More routines for operating on iterables, beyond itertools" groups = ["dev"] files = [ - {file = "more-itertools-10.4.0.tar.gz", hash = "sha256:fe0e63c4ab068eac62410ab05cccca2dc71ec44ba8ef29916a0090df061cf923"}, - {file = "more_itertools-10.4.0-py3-none-any.whl", hash = "sha256:0f7d9f83a0a8dcfa8a2694a770590d98a67ea943e3d9f5298309a484758c4e27"}, + {file = "more-itertools-10.5.0.tar.gz", hash = "sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6"}, + {file = "more_itertools-10.5.0-py3-none-any.whl", hash = "sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef"}, ] [[package]] name = "msgpack" -version = "1.0.8" +version = "1.1.0" requires_python = ">=3.8" summary = "MessagePack serializer" groups = ["dev"] files = [ - {file = "msgpack-1.0.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9517004e21664f2b5a5fd6333b0731b9cf0817403a941b393d89a2f1dc2bd836"}, - {file = "msgpack-1.0.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d16a786905034e7e34098634b184a7d81f91d4c3d246edc6bd7aefb2fd8ea6ad"}, - {file = "msgpack-1.0.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2872993e209f7ed04d963e4b4fbae72d034844ec66bc4ca403329db2074377b"}, - {file = "msgpack-1.0.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c330eace3dd100bdb54b5653b966de7f51c26ec4a7d4e87132d9b4f738220ba"}, - {file = "msgpack-1.0.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b5c044f3eff2a6534768ccfd50425939e7a8b5cf9a7261c385de1e20dcfc85"}, - {file = "msgpack-1.0.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1876b0b653a808fcd50123b953af170c535027bf1d053b59790eebb0aeb38950"}, - {file = "msgpack-1.0.8-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dfe1f0f0ed5785c187144c46a292b8c34c1295c01da12e10ccddfc16def4448a"}, - {file = "msgpack-1.0.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3528807cbbb7f315bb81959d5961855e7ba52aa60a3097151cb21956fbc7502b"}, - {file = "msgpack-1.0.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e2f879ab92ce502a1e65fce390eab619774dda6a6ff719718069ac94084098ce"}, - {file = "msgpack-1.0.8-cp311-cp311-win32.whl", hash = "sha256:26ee97a8261e6e35885c2ecd2fd4a6d38252246f94a2aec23665a4e66d066305"}, - {file = "msgpack-1.0.8-cp311-cp311-win_amd64.whl", hash = "sha256:eadb9f826c138e6cf3c49d6f8de88225a3c0ab181a9b4ba792e006e5292d150e"}, - {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa"}, + {file = "msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59"}, + {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6"}, + {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5"}, + {file = "msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88"}, + {file = "msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"}, + {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, ] [[package]] @@ -1449,28 +1494,31 @@ files = [ [[package]] name = "multidict" -version = "6.0.5" -requires_python = ">=3.7" +version = "6.1.0" +requires_python = ">=3.8" summary = "multidict implementation" groups = ["dev"] +dependencies = [ + "typing-extensions>=4.1.0; python_version < \"3.11\"", +] files = [ - {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, - {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, - {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, - {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, - {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, - {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, - {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"}, + {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"}, + {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, ] [[package]] @@ -1517,7 +1565,9 @@ groups = ["dev"] dependencies = [ "argcomplete<4.0,>=1.9.4", "colorlog<7.0.0,>=2.6.1", + "importlib-metadata; python_version < \"3.8\"", "packaging>=20.9", + "typing-extensions>=3.7.4; python_version < \"3.8\"", "virtualenv>=14", ] files = [ @@ -1527,25 +1577,20 @@ files = [ [[package]] name = "numpy" -version = "2.1.0" -requires_python = ">=3.10" +version = "1.26.4" +requires_python = ">=3.9" summary = "Fundamental package for array computing in Python" groups = ["dev"] files = [ - {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, - {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, - {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, - {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1602,6 +1647,9 @@ name = "pickleshare" version = "0.7.5" summary = "Tiny 'shelve'-like database with concurrency support" groups = ["dev"] +dependencies = [ + "pathlib2; python_version in \"2.6 2.7 3.2 3.3\"", +] files = [ {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, @@ -1609,13 +1657,13 @@ files = [ [[package]] name = "platformdirs" -version = "4.2.2" +version = "4.3.6" requires_python = ">=3.8" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." groups = ["dev"] files = [ - {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, - {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [[package]] @@ -1642,7 +1690,7 @@ files = [ [[package]] name = "prompt-toolkit" -version = "3.0.47" +version = "3.0.48" requires_python = ">=3.7.0" summary = "Library for building powerful interactive command lines in Python" groups = ["dev"] @@ -1650,8 +1698,35 @@ dependencies = [ "wcwidth", ] files = [ - {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"}, - {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"}, + {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, + {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, +] + +[[package]] +name = "propcache" +version = "0.2.0" +requires_python = ">=3.8" +summary = "Accelerated property cache" +groups = ["dev"] +files = [ + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, + {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, + {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, + {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, + {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, ] [[package]] @@ -1742,9 +1817,6 @@ files = [ {file = "py_ed25519_zebra_bindings-1.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2e715341b4927f6735ed7113644c0a5362310df4ddad1f938b5040c85884db15"}, {file = "py_ed25519_zebra_bindings-1.0.1-cp311-none-win32.whl", hash = "sha256:21498379d5e85d97a9633b7cf6362b4d187c7575ab8633c3ba6c99b1dcb83358"}, {file = "py_ed25519_zebra_bindings-1.0.1-cp311-none-win_amd64.whl", hash = "sha256:58e7d56a6f565fc044d313ec429b782150366a39ada973051dde60f1363abd9b"}, - {file = "py_ed25519_zebra_bindings-1.0.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:670f01572012aabd02436de78bd880a63691031f7758289a05529368e8af5ac6"}, - {file = "py_ed25519_zebra_bindings-1.0.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aae847c456757dc8e5022c0f5af64db07baaf8e0d60e0931912e92c1e9ea532d"}, - {file = "py_ed25519_zebra_bindings-1.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98450820e82e4fd446e2b9fd7b449e8b4cb645fa4c053d4e0a598a9c43ed8209"}, {file = "py_ed25519_zebra_bindings-1.0.1.tar.gz", hash = "sha256:0062f189e1c8672ba94676cedb346fae4c33a0cabaf12e75a1aedcf9db47403b"}, ] @@ -1762,26 +1834,23 @@ files = [ {file = "py_sr25519_bindings-0.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7b56b5cbbfb36b41ddfa462989a03386590ac036f3a755ef64fffeb2fed88654"}, {file = "py_sr25519_bindings-0.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8f06ea3237e06666e3a4ff4719b4fba415472943831b229428753c37d5ecd1b4"}, {file = "py_sr25519_bindings-0.2.0-cp311-none-win_amd64.whl", hash = "sha256:d62af30b2022f5fa787e46c06823c35a21abe791bf55012f498f9ba8e4baabc8"}, - {file = "py_sr25519_bindings-0.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38db0ee90bd676b9df7ddd03fcb2113b5a5e9d9c984d82426728acc0e9d54277"}, - {file = "py_sr25519_bindings-0.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dfe767069d5c5e8a313e77b6bd681ea4f6b5988b09b6b4c9399e255fe4a7c53"}, - {file = "py_sr25519_bindings-0.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8951d1a6e310a682a6253d547e44a9e7a606476dbc18dea3f121d98bdb81042"}, {file = "py_sr25519_bindings-0.2.0.tar.gz", hash = "sha256:0c2fe92b7cdcebf6c5611a90054f8ba6ea90b68b8832896d2dc565537bc40b0c"}, ] [[package]] name = "pyasn1" -version = "0.6.0" +version = "0.6.1" requires_python = ">=3.8" summary = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" groups = ["dev"] files = [ - {file = "pyasn1-0.6.0-py2.py3-none-any.whl", hash = "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"}, - {file = "pyasn1-0.6.0.tar.gz", hash = "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c"}, + {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, + {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, ] [[package]] name = "pyasn1-modules" -version = "0.4.0" +version = "0.4.1" requires_python = ">=3.8" summary = "A collection of ASN.1-based protocols modules" groups = ["dev"] @@ -1789,8 +1858,8 @@ dependencies = [ "pyasn1<0.7.0,>=0.4.6", ] files = [ - {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"}, - {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"}, + {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, + {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, ] [[package]] @@ -1806,53 +1875,44 @@ files = [ [[package]] name = "pycryptodome" -version = "3.20.0" -requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "3.21.0" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" summary = "Cryptographic library for Python" groups = ["dev"] files = [ - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, - {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:2480ec2c72438430da9f601ebc12c518c093c13111a5c1644c82cdfc2e50b1e4"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:de18954104667f565e2fbb4783b56667f30fb49c4d79b346f52a29cb198d5b6b"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de4b7263a33947ff440412339cb72b28a5a4c769b5c1ca19e33dd6cd1dcec6e"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0714206d467fc911042d01ea3a1847c847bc10884cf674c82e12915cfe1649f8"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d85c1b613121ed3dbaa5a97369b3b757909531a959d229406a75b912dd51dd1"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8898a66425a57bcf15e25fc19c12490b87bd939800f39a03ea2de2aea5e3611a"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:932c905b71a56474bff8a9c014030bc3c882cee696b448af920399f730a650c2"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18caa8cfbc676eaaf28613637a89980ad2fd96e00c564135bf90bc3f0b34dd93"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win32.whl", hash = "sha256:280b67d20e33bb63171d55b1067f61fbd932e0b1ad976b3a184303a3dad22764"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b7aa25fc0baa5b1d95b7633af4f5f1838467f1815442b22487426f94e0d66c53"}, + {file = "pycryptodome-3.21.0.tar.gz", hash = "sha256:f7787e0d469bdae763b876174cf2e6c0f7be79808af26b1da96f1a64bcf47297"}, ] [[package]] name = "pydantic" -version = "2.8.2" +version = "2.9.2" requires_python = ">=3.8" summary = "Data validation using Python type hints" groups = ["dev"] dependencies = [ - "annotated-types>=0.4.0", - "pydantic-core==2.20.1", + "annotated-types>=0.6.0", + "pydantic-core==2.23.4", + "typing-extensions>=4.12.2; python_version >= \"3.13\"", "typing-extensions>=4.6.1; python_version < \"3.13\"", ] files = [ - {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, - {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [[package]] name = "pydantic-core" -version = "2.20.1" +version = "2.23.4" requires_python = ">=3.8" summary = "Core functionality for Pydantic validation and serialization" groups = ["dev"] @@ -1860,35 +1920,19 @@ dependencies = [ "typing-extensions!=4.7.0,>=4.6.0", ] files = [ - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, - {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, - {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, - {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, ] [[package]] @@ -1940,33 +1984,35 @@ files = [ [[package]] name = "pytest" -version = "8.3.2" +version = "8.3.3" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["dev"] dependencies = [ "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", "iniconfig", "packaging", "pluggy<2,>=1.5", + "tomli>=1; python_version < \"3.11\"", ] files = [ - {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, - {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, ] [[package]] name = "pytest-asyncio" -version = "0.23.8" +version = "0.24.0" requires_python = ">=3.8" summary = "Pytest support for asyncio" groups = ["dev"] dependencies = [ - "pytest<9,>=7.0.0", + "pytest<9,>=8.2", ] files = [ - {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, - {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, + {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, + {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, ] [[package]] @@ -2000,37 +2046,37 @@ files = [ [[package]] name = "python-levenshtein" -version = "0.25.1" -requires_python = ">=3.8" +version = "0.26.0" +requires_python = ">=3.9" summary = "Python extension for computing string edit distances and similarities." groups = ["dev"] dependencies = [ - "Levenshtein==0.25.1", + "Levenshtein==0.26.0", ] files = [ - {file = "python-Levenshtein-0.25.1.tar.gz", hash = "sha256:b21e7efe83c8e8dc8260f2143b2393c6c77cb2956f0c53de6c4731c4d8006acc"}, - {file = "python_Levenshtein-0.25.1-py3-none-any.whl", hash = "sha256:654446d1ea4acbcc573d44c43775854834a7547e4cb2f79f638f738134d72037"}, + {file = "python_Levenshtein-0.26.0-py3-none-any.whl", hash = "sha256:1d808ba2f9df04aaea5eceba6e73734f2ffeba99d98d2a91078f32276cd041f4"}, + {file = "python_levenshtein-0.26.0.tar.gz", hash = "sha256:b454dd13708546649f1cba2a0f450dd98e7c1679a92e2d6f0a8b8c013c276e55"}, ] [[package]] name = "python-statemachine" -version = "2.1.2" -requires_python = ">=3.7,<3.13" +version = "2.3.6" +requires_python = ">=3.7" summary = "Python Finite State Machines made easy." groups = ["dev"] files = [ - {file = "python_statemachine-2.1.2-py3-none-any.whl", hash = "sha256:d7e369d5da5b9007cc7cf5eb7a1b169081e2f4b7d30b6415fc122858fb7696ec"}, - {file = "python_statemachine-2.1.2.tar.gz", hash = "sha256:0b0dd8b28738b53f14391b06d5072cd5e72259da5ae23574d3d4f5e6dd366663"}, + {file = "python_statemachine-2.3.6-py3-none-any.whl", hash = "sha256:0001b02cbe2f5b2420c423b5b3e3a33915447ac6d9735219c929e2378d454f5f"}, + {file = "python_statemachine-2.3.6.tar.gz", hash = "sha256:9cb4040ca7f2158d3cd46f36a77b420b6ef95a90223928a7f3cab232a70bd560"}, ] [[package]] name = "pytz" -version = "2024.1" +version = "2024.2" summary = "World timezone definitions, modern and historical" groups = ["dev"] files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] [[package]] @@ -2054,45 +2100,27 @@ files = [ [[package]] name = "rapidfuzz" -version = "3.9.6" -requires_python = ">=3.8" +version = "3.10.0" +requires_python = ">=3.9" summary = "rapid fuzzy string matching" groups = ["dev"] files = [ - {file = "rapidfuzz-3.9.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:52e4675f642fbc85632f691b67115a243cd4d2a47bdcc4a3d9a79e784518ff97"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1f93a2f13038700bd245b927c46a2017db3dcd4d4ff94687d74b5123689b873b"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b70500bca460264b8141d8040caee22e9cf0418c5388104ff0c73fb69ee28f"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1e037fb89f714a220f68f902fc6300ab7a33349f3ce8ffae668c3b3a40b0b06"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6792f66d59b86ccfad5e247f2912e255c85c575789acdbad8e7f561412ffed8a"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68d9cffe710b67f1969cf996983608cee4490521d96ea91d16bd7ea5dc80ea98"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63daaeeea76da17fa0bbe7fb05cba8ed8064bb1a0edf8360636557f8b6511961"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d214e063bffa13e3b771520b74f674b22d309b5720d4df9918ff3e0c0f037720"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ed443a2062460f44c0346cb9d269b586496b808c2419bbd6057f54061c9b9c75"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5b0c9b227ee0076fb2d58301c505bb837a290ae99ee628beacdb719f0626d749"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:82c9722b7dfaa71e8b61f8c89fed0482567fb69178e139fe4151fc71ed7df782"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c18897c95c0a288347e29537b63608a8f63a5c3cb6da258ac46fcf89155e723e"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-win32.whl", hash = "sha256:3e910cf08944da381159587709daaad9e59d8ff7bca1f788d15928f3c3d49c2a"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-win_amd64.whl", hash = "sha256:59c4a61fab676d37329fc3a671618a461bfeef53a4d0b8b12e3bc24a14e166f8"}, - {file = "rapidfuzz-3.9.6-cp311-cp311-win_arm64.whl", hash = "sha256:8b4afea244102332973377fddbe54ce844d0916e1c67a5123432291717f32ffa"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a65c2f63218ea2dedd56fc56361035e189ca123bd9c9ce63a9bef6f99540d681"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:680dc78a5f889d3b89f74824b89fe357f49f88ad10d2c121e9c3ad37bac1e4eb"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8ca862927a0b05bd825e46ddf82d0724ea44b07d898ef639386530bf9b40f15"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2116fa1fbff21fa52cd46f3cfcb1e193ba1d65d81f8b6e123193451cd3d6c15e"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dcb7d9afd740370a897c15da61d3d57a8d54738d7c764a99cedb5f746d6a003"}, - {file = "rapidfuzz-3.9.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1a5bd6401bb489e14cbb5981c378d53ede850b7cc84b2464cad606149cc4e17d"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:29fda70b9d03e29df6fc45cc27cbcc235534b1b0b2900e0a3ae0b43022aaeef5"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:88144f5f52ae977df9352029488326afadd7a7f42c6779d486d1f82d43b2b1f2"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:715aeaabafba2709b9dd91acb2a44bad59d60b4616ef90c08f4d4402a3bbca60"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af26ebd3714224fbf9bebbc27bdbac14f334c15f5d7043699cd694635050d6ca"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101bd2df438861a005ed47c032631b7857dfcdb17b82beeeb410307983aac61d"}, - {file = "rapidfuzz-3.9.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2185e8e29809b97ad22a7f99281d1669a89bdf5fa1ef4ef1feca36924e675367"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9e53c72d08f0e9c6e4a369e52df5971f311305b4487690c62e8dd0846770260c"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a0cb157162f0cdd62e538c7bd298ff669847fc43a96422811d5ab933f4c16c3a"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bb5ff2bd48132ed5e7fbb8f619885facb2e023759f2519a448b2c18afe07e5d"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dc37f601865e8407e3a8037ffbc3afe0b0f837b2146f7632bd29d087385babe"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a657eee4b94668faf1fa2703bdd803654303f7e468eb9ba10a664d867ed9e779"}, - {file = "rapidfuzz-3.9.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:51be6ab5b1d5bb32abd39718f2a5e3835502e026a8272d139ead295c224a6f5e"}, - {file = "rapidfuzz-3.9.6.tar.gz", hash = "sha256:5cf2a7d621e4515fee84722e93563bf77ff2cbe832a77a48b81f88f9e23b9e8d"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bb0013795b40db5cf361e6f21ee7cda09627cf294977149b50e217d7fe9a2f03"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:69ef5b363afff7150a1fbe788007e307b9802a2eb6ad92ed51ab94e6ad2674c6"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c582c46b1bb0b19f1a5f4c1312f1b640c21d78c371a6615c34025b16ee56369b"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:288f6f6e7410cacb115fb851f3f18bf0e4231eb3f6cb5bd1cec0e7b25c4d039d"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9e29a13d2fd9be3e7d8c26c7ef4ba60b5bc7efbc9dbdf24454c7e9ebba31768"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea2da0459b951ee461bd4e02b8904890bd1c4263999d291c5cd01e6620177ad4"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:457827ba82261aa2ae6ac06a46d0043ab12ba7216b82d87ae1434ec0f29736d6"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d350864269d56f51ab81ab750c9259ae5cad3152c0680baef143dcec92206a1"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a9b8f51e08c3f983d857c3889930af9ddecc768453822076683664772d87e374"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f3a6aa6e70fc27e4ff5c479f13cc9fc26a56347610f5f8b50396a0d344c5f55"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:803f255f10d63420979b1909ef976e7d30dec42025c9b067fc1d2040cc365a7e"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2026651761bf83a0f31495cc0f70840d5c0d54388f41316e3f9cb51bd85e49a5"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-win32.whl", hash = "sha256:4df75b3ebbb8cfdb9bf8b213b168620b88fd92d0c16a8bc9f9234630b282db59"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:f9f0bbfb6787b97c51516f3ccf97737d504db5d239ad44527673b81f598b84ab"}, + {file = "rapidfuzz-3.10.0-cp311-cp311-win_arm64.whl", hash = "sha256:10fdad800441b9c97d471a937ba7d42625f1b530db05e572f1cb7d401d95c893"}, + {file = "rapidfuzz-3.10.0.tar.gz", hash = "sha256:6b62af27e65bb39276a66533655a2fa3c60a487b03935721c45b7809527979be"}, ] [[package]] @@ -2103,6 +2131,8 @@ summary = "Python client for Redis database and key-value store" groups = ["dev"] dependencies = [ "async-timeout>=4.0.2; python_full_version <= \"3.11.2\"", + "importlib-metadata>=1.0; python_version < \"3.8\"", + "typing-extensions; python_version < \"3.8\"", ] files = [ {file = "redis-4.6.0-py3-none-any.whl", hash = "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c"}, @@ -2128,12 +2158,12 @@ files = [ [[package]] name = "resolvelib" -version = "0.8.1" +version = "1.0.1" summary = "Resolve abstract dependencies into concrete ones" groups = ["dev"] files = [ - {file = "resolvelib-0.8.1-py2.py3-none-any.whl", hash = "sha256:d9b7907f055c3b3a2cfc56c914ffd940122915826ff5fb5b1de0c99778f4de98"}, - {file = "resolvelib-0.8.1.tar.gz", hash = "sha256:c6ea56732e9fb6fca1b2acc2ccc68a0b6b8c566d8f3e78e0443310ede61dbd37"}, + {file = "resolvelib-1.0.1-py2.py3-none-any.whl", hash = "sha256:d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf"}, + {file = "resolvelib-1.0.1.tar.gz", hash = "sha256:04ce76cbd63fded2078ce224785da6ecd42b9564b1390793f64ddecbe997b309"}, ] [[package]] @@ -2152,17 +2182,32 @@ files = [ [[package]] name = "rich" -version = "13.7.1" -requires_python = ">=3.7.0" +version = "13.9.2" +requires_python = ">=3.8.0" summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" groups = ["dev"] dependencies = [ "markdown-it-py>=2.2.0", "pygments<3.0.0,>=2.13.0", + "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", +] +files = [ + {file = "rich-13.9.2-py3-none-any.whl", hash = "sha256:8c82a3d3f8dcfe9e734771313e606b39d8247bb6b826e196f4914b333b743cf1"}, + {file = "rich-13.9.2.tar.gz", hash = "sha256:51a2c62057461aaf7152b4d611168f93a9fc73068f8ded2790f29fe2b5366d0c"}, +] + +[[package]] +name = "s3transfer" +version = "0.10.3" +requires_python = ">=3.8" +summary = "An Amazon S3 Transfer Manager" +groups = ["dev"] +dependencies = [ + "botocore<2.0a.0,>=1.33.2", ] files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d"}, + {file = "s3transfer-0.10.3.tar.gz", hash = "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"}, ] [[package]] @@ -2188,7 +2233,9 @@ summary = "Python client for Sentry (https://sentry.io)" groups = ["dev"] dependencies = [ "certifi", + "urllib3>=1.25.7; python_version <= \"3.4\"", "urllib3>=1.26.11; python_version >= \"3.6\"", + "urllib3>=1.26.9; python_version == \"3.5\"", ] files = [ {file = "sentry-sdk-1.29.2.tar.gz", hash = "sha256:a99ee105384788c3f228726a88baf515fe7b5f1d2d0f215a03d194369f158df7"}, @@ -2214,13 +2261,13 @@ files = [ [[package]] name = "setuptools" -version = "73.0.0" +version = "70.0.0" requires_python = ">=3.8" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["dev"] files = [ - {file = "setuptools-73.0.0-py3-none-any.whl", hash = "sha256:f2bfcce7ae1784d90b04c57c2802e8649e1976530bb25dc72c2b078d3ecf4864"}, - {file = "setuptools-73.0.0.tar.gz", hash = "sha256:3c08705fadfc8c7c445cf4d98078f0fafb9225775b2b4e8447e40348f82597c0"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [[package]] @@ -2290,6 +2337,7 @@ summary = "The little ASGI library that shines." groups = ["dev"] dependencies = [ "anyio<5,>=3.4.0", + "typing-extensions>=3.10.0; python_version < \"3.10\"", ] files = [ {file = "starlette-0.37.2-py3-none-any.whl", hash = "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee"}, @@ -2298,7 +2346,7 @@ files = [ [[package]] name = "substrate-interface" -version = "1.7.10" +version = "1.7.11" requires_python = "<4,>=3.7" summary = "Library for interfacing with a Substrate node" groups = ["dev"] @@ -2308,7 +2356,7 @@ dependencies = [ "certifi>=2019.3.9", "ecdsa<1,>=0.17.0", "eth-keys<1,>=0.2.1", - "eth-utils<3,>=1.3.0", + "eth-utils<6,>=1.3.0", "idna<4,>=2.1.0", "py-bip39-bindings<1,>=0.1.9", "py-ed25519-zebra-bindings<2,>=1.0", @@ -2320,8 +2368,8 @@ dependencies = [ "xxhash<4,>=1.3.0", ] files = [ - {file = "substrate-interface-1.7.10.tar.gz", hash = "sha256:0dec0104abc16d01c3d22700253a84e67430b5e56c46efea71fea47063a8eaa4"}, - {file = "substrate_interface-1.7.10-py3-none-any.whl", hash = "sha256:4873e9f1b75375ed9fcdd12d7bca66c47ab0e9fbd532ec4f9538ceac0f0ab2f5"}, + {file = "substrate-interface-1.7.11.tar.gz", hash = "sha256:4caa5eacb9996edbe76ad12249521b3542bbd8d9d69b96734087201db1fef8f6"}, + {file = "substrate_interface-1.7.11-py3-none-any.whl", hash = "sha256:ce19bc97481769238ed23c752db985a3058637918693f2db6aeed2fab3756075"}, ] [[package]] @@ -2337,25 +2385,25 @@ files = [ [[package]] name = "termcolor" -version = "2.4.0" -requires_python = ">=3.8" +version = "2.5.0" +requires_python = ">=3.9" summary = "ANSI color formatting for output in terminal" groups = ["dev"] files = [ - {file = "termcolor-2.4.0-py3-none-any.whl", hash = "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63"}, - {file = "termcolor-2.4.0.tar.gz", hash = "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"}, + {file = "termcolor-2.5.0-py3-none-any.whl", hash = "sha256:37b17b5fc1e604945c2642c872a3764b5d547a48009871aea3edd3afa180afb8"}, + {file = "termcolor-2.5.0.tar.gz", hash = "sha256:998d8d27da6d48442e8e1f016119076b690d962507531df4890fcd2db2ef8a6f"}, ] [[package]] name = "toolz" -version = "0.12.1" -requires_python = ">=3.7" +version = "1.0.0" +requires_python = ">=3.8" summary = "List processing tools and functional utilities" groups = ["dev"] marker = "implementation_name == \"pypy\" or implementation_name == \"cpython\"" files = [ - {file = "toolz-0.12.1-py3-none-any.whl", hash = "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85"}, - {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"}, + {file = "toolz-1.0.0-py3-none-any.whl", hash = "sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236"}, + {file = "toolz-1.0.0.tar.gz", hash = "sha256:2c86e3d9a04798ac556793bced838816296a2f085017664e4995cb40a1047a02"}, ] [[package]] @@ -2465,39 +2513,40 @@ files = [ [[package]] name = "tzdata" -version = "2024.1" +version = "2024.2" requires_python = ">=2" summary = "Provider of IANA time zone data" groups = ["dev"] files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] [[package]] name = "urllib3" -version = "2.2.2" +version = "2.2.3" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." groups = ["dev"] files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [[package]] name = "uvicorn" -version = "0.30.0" +version = "0.31.1" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["dev"] dependencies = [ "click>=7.0", "h11>=0.8", + "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.30.0-py3-none-any.whl", hash = "sha256:78fa0b5f56abb8562024a59041caeb555c86e48d0efdd23c3fe7de7a4075bdab"}, - {file = "uvicorn-0.30.0.tar.gz", hash = "sha256:f678dec4fa3a39706bbf49b9ec5fc40049d42418716cea52b53f07828a60aa37"}, + {file = "uvicorn-0.31.1-py3-none-any.whl", hash = "sha256:adc42d9cac80cf3e51af97c1851648066841e7cfb6993a4ca8de29ac1548ed41"}, + {file = "uvicorn-0.31.1.tar.gz", hash = "sha256:f5167919867b161b7bcaf32646c6a94cdbd4c3aa2eb5c17d36bb9aa5cfd8c493"}, ] [[package]] @@ -2526,6 +2575,7 @@ summary = "" groups = ["dev"] dependencies = [ "Django~=4.2.4", + "boto3>=1.35.11", "celery~=5.3.1", "channels-redis>=4.2.0", "channels[daphne]==4.*", @@ -2541,6 +2591,7 @@ dependencies = [ "django-prometheus==2.3.1", "flower~=2.0.0", "gunicorn==20.1.0", + "httpx~=0.26.0", "ipython~=8.14.0", "nox==2023.4.22", "prometheus-client~=0.17.0", @@ -2565,18 +2616,19 @@ files = [ [[package]] name = "virtualenv" -version = "20.26.3" +version = "20.26.6" requires_python = ">=3.7" summary = "Virtual Python Environment builder" groups = ["dev"] dependencies = [ "distlib<1,>=0.3.7", "filelock<4,>=3.12.2", + "importlib-metadata>=6.6; python_version < \"3.8\"", "platformdirs<5,>=3.9.1", ] files = [ - {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, - {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, + {file = "virtualenv-20.26.6-py3-none-any.whl", hash = "sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2"}, + {file = "virtualenv-20.26.6.tar.gz", hash = "sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48"}, ] [[package]] @@ -2584,6 +2636,9 @@ name = "wcwidth" version = "0.2.13" summary = "Measures the displayed width of unicode strings in a terminal" groups = ["dev"] +dependencies = [ + "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"", +] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -2618,21 +2673,6 @@ files = [ {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, - {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, - {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, - {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, - {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, - {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, - {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] @@ -2670,62 +2710,43 @@ files = [ {file = "xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da"}, {file = "xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58"}, {file = "xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3"}, - {file = "xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c"}, - {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986"}, - {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6"}, - {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b"}, - {file = "xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da"}, - {file = "xxhash-3.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c"}, - {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae"}, - {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e"}, - {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57"}, - {file = "xxhash-3.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837"}, - {file = "xxhash-3.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692"}, - {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18"}, - {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514"}, - {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81"}, - {file = "xxhash-3.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1"}, - {file = "xxhash-3.5.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9"}, - {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1"}, - {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f"}, - {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0"}, - {file = "xxhash-3.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240"}, {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"}, ] [[package]] name = "yarl" -version = "1.9.4" -requires_python = ">=3.7" +version = "1.14.0" +requires_python = ">=3.8" summary = "Yet another URL library" groups = ["dev"] dependencies = [ "idna>=2.0", "multidict>=4.0", + "propcache>=0.2.0", ] files = [ - {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, - {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, - {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, - {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, - {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, - {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, - {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, + {file = "yarl-1.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:07f9eaf57719d6721ab15805d85f4b01a5b509a0868d7320134371bcb652152d"}, + {file = "yarl-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c14b504a74e58e2deb0378b3eca10f3d076635c100f45b113c18c770b4a47a50"}, + {file = "yarl-1.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:16a682a127930f3fc4e42583becca6049e1d7214bcad23520c590edd741d2114"}, + {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73bedd2be05f48af19f0f2e9e1353921ce0c83f4a1c9e8556ecdcf1f1eae4892"}, + {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3ab950f8814f3b7b5e3eebc117986f817ec933676f68f0a6c5b2137dd7c9c69"}, + {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b693c63e7e64b524f54aa4888403c680342d1ad0d97be1707c531584d6aeeb4f"}, + {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85cb3e40eaa98489f1e2e8b29f5ad02ee1ee40d6ce6b88d50cf0f205de1d9d2c"}, + {file = "yarl-1.14.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f24f08b6c9b9818fd80612c97857d28f9779f0d1211653ece9844fc7b414df2"}, + {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29a84a46ec3ebae7a1c024c055612b11e9363a8a23238b3e905552d77a2bc51b"}, + {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5cd5dad8366e0168e0fd23d10705a603790484a6dbb9eb272b33673b8f2cce72"}, + {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a152751af7ef7b5d5fa6d215756e508dd05eb07d0cf2ba51f3e740076aa74373"}, + {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3d569f877ed9a708e4c71a2d13d2940cb0791da309f70bd970ac1a5c088a0a92"}, + {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a615cad11ec3428020fb3c5a88d85ce1b5c69fd66e9fcb91a7daa5e855325dd"}, + {file = "yarl-1.14.0-cp311-cp311-win32.whl", hash = "sha256:bab03192091681d54e8225c53f270b0517637915d9297028409a2a5114ff4634"}, + {file = "yarl-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:985623575e5c4ea763056ffe0e2d63836f771a8c294b3de06d09480538316b13"}, + {file = "yarl-1.14.0-py3-none-any.whl", hash = "sha256:c8ed4034f0765f8861620c1f2f2364d2e58520ea288497084dae880424fc0d9f"}, + {file = "yarl-1.14.0.tar.gz", hash = "sha256:88c7d9d58aab0724b979ab5617330acb1c7030b79379c8138c1c8c94e121d1b3"}, ] [[package]] name = "zope-interface" -version = "7.0.1" +version = "7.1.0" requires_python = ">=3.8" summary = "Interfaces for Python" groups = ["dev"] @@ -2733,11 +2754,11 @@ dependencies = [ "setuptools", ] files = [ - {file = "zope.interface-7.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03bd5c0db82237bbc47833a8b25f1cc090646e212f86b601903d79d7e6b37031"}, - {file = "zope.interface-7.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f52050c6a10d4a039ec6f2c58e5b3ade5cc570d16cf9d102711e6b8413c90e6"}, - {file = "zope.interface-7.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af0b33f04677b57843d529b9257a475d2865403300b48c67654c40abac2f9f24"}, - {file = "zope.interface-7.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696c2a381fc7876b3056711717dba5eddd07c2c9e5ccd50da54029a1293b6e43"}, - {file = "zope.interface-7.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f89a420cf5a6f2aa7849dd59e1ff0e477f562d97cf8d6a1ee03461e1eec39887"}, - {file = "zope.interface-7.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:b59deb0ddc7b431e41d720c00f99d68b52cb9bd1d5605a085dc18f502fe9c47f"}, - {file = "zope.interface-7.0.1.tar.gz", hash = "sha256:f0f5fda7cbf890371a59ab1d06512da4f2c89a6ea194e595808123c863c38eff"}, + {file = "zope.interface-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ac20581fc6cd7c754f6dff0ae06fedb060fa0e9ea6309d8be8b2701d9ea51c4"}, + {file = "zope.interface-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:848b6fa92d7c8143646e64124ed46818a0049a24ecc517958c520081fd147685"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1ef1fdb6f014d5886b97e52b16d0f852364f447d2ab0f0c6027765777b6667"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bcff5c09d0215f42ba64b49205a278e44413d9bf9fa688fd9e42bfe472b5f4f"}, + {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07add15de0cc7e69917f7d286b64d54125c950aeb43efed7a5ea7172f000fbc1"}, + {file = "zope.interface-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:9940d5bc441f887c5f375ec62bcf7e7e495a2d5b1da97de1184a88fb567f06af"}, + {file = "zope_interface-7.1.0.tar.gz", hash = "sha256:3f005869a1a05e368965adb2075f97f8ee9a26c61898a9e52a9764d93774f237"}, ] diff --git a/validator/pdm.lock b/validator/pdm.lock index 0237c2d22..6722379c6 100644 --- a/validator/pdm.lock +++ b/validator/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "format", "lint", "security_check", "test", "type_check"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:338403a1363e8542b83f9c2827ca0b9572ccf61067288dd4bba20bb791e2c02c" +content_hash = "sha256:779dfd3e080b19c6bd4a37e4f28edb28b16b131af3f6e195de306166b88a24d0" [[metadata.targets]] requires_python = "==3.11.*" @@ -1288,7 +1288,7 @@ files = [ [[package]] name = "httpx" -version = "0.27.2" +version = "0.26.0" requires_python = ">=3.8" summary = "The next generation HTTP client." groups = ["default", "test"] @@ -1300,8 +1300,8 @@ dependencies = [ "sniffio", ] files = [ - {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, - {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [[package]] @@ -2216,17 +2216,17 @@ files = [ [[package]] name = "pytest-httpx" -version = "0.30.0" +version = "0.29.0" requires_python = ">=3.9" summary = "Send responses to httpx." groups = ["test"] dependencies = [ - "httpx==0.27.*", + "httpx==0.26.*", "pytest<9,>=7", ] files = [ - {file = "pytest-httpx-0.30.0.tar.gz", hash = "sha256:755b8edca87c974dd4f3605c374fda11db84631de3d163b99c0df5807023a19a"}, - {file = "pytest_httpx-0.30.0-py3-none-any.whl", hash = "sha256:6d47849691faf11d2532565d0c8e0e02b9f4ee730da31687feae315581d7520c"}, + {file = "pytest_httpx-0.29.0-py3-none-any.whl", hash = "sha256:7d6fd29042e7b98ed98199ded120bc8100c8078ca306952666e89bf8807b95ff"}, + {file = "pytest_httpx-0.29.0.tar.gz", hash = "sha256:ed08ed802e2b315b83cdd16f0b26cbb2b836c29e0fde5c18bc3105f1073e0332"}, ] [[package]] diff --git a/validator/pyproject.toml b/validator/pyproject.toml index 46fa99730..1025de5fd 100644 --- a/validator/pyproject.toml +++ b/validator/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "django-admin-rangefilter==0.12.4", "uvloop>=0.19.0", "boto3>=1.35.11", - "httpx>=0.27.2", + "httpx~=0.26.0", ] [build-system] From 500afcb3e96d40df204f8634f39797f8b8161e1f Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Fri, 11 Oct 2024 15:22:57 +0200 Subject: [PATCH 24/99] rename package --- compute_horde/compute_horde/receipts/models.py | 2 +- .../compute_horde/receipts/{pydantic.py => schemas.py} | 0 compute_horde/compute_horde/receipts/transfer.py | 2 +- compute_horde/tests/conftest.py | 2 +- compute_horde/tests/test_receipts.py | 2 +- miner/app/src/compute_horde_miner/miner/receipt_store/base.py | 2 +- miner/app/src/compute_horde_miner/miner/receipt_store/local.py | 2 +- miner/app/src/compute_horde_miner/miner/tests/test_migration.py | 2 +- .../compute_horde_validator/validator/tests/test_receipts.py | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename compute_horde/compute_horde/receipts/{pydantic.py => schemas.py} (100%) diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index c69d40630..b9166034a 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -7,7 +7,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.pydantic import Receipt +from compute_horde.receipts.schemas import Receipt class ReceiptNotSigned(Exception): diff --git a/compute_horde/compute_horde/receipts/pydantic.py b/compute_horde/compute_horde/receipts/schemas.py similarity index 100% rename from compute_horde/compute_horde/receipts/pydantic.py rename to compute_horde/compute_horde/receipts/schemas.py diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index e803e1485..071a468d2 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -14,7 +14,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.pydantic import Receipt, ReceiptType +from compute_horde.receipts.schemas import Receipt, ReceiptType logger = logging.getLogger(__name__) diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index 5d69c0f72..bde902a63 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -9,7 +9,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.pydantic import Receipt +from compute_horde.receipts.schemas import Receipt @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index 5e0028425..545f16a47 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -7,7 +7,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.pydantic import ( +from compute_horde.receipts.schemas import ( Receipt, ReceiptType, ) diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/base.py b/miner/app/src/compute_horde_miner/miner/receipt_store/base.py index faadfdbc3..395d53bd9 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/base.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/base.py @@ -1,6 +1,6 @@ import abc -from compute_horde.receipts.pydantic import Receipt +from compute_horde.receipts.schemas import Receipt class BaseReceiptStore(metaclass=abc.ABCMeta): diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py index 664fca239..2aad9143e 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py @@ -8,7 +8,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.pydantic import Receipt, ReceiptType +from compute_horde.receipts.schemas import Receipt, ReceiptType from django.conf import settings from compute_horde_miner.miner.receipt_store.base import BaseReceiptStore diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 7cde30e31..556195e33 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -7,7 +7,7 @@ JobStartedReceiptPayload, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.pydantic import Receipt +from compute_horde.receipts.schemas import Receipt from django.utils.timezone import now from pytest_mock import MockerFixture diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 2f24434e6..aa32769f0 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -8,7 +8,7 @@ JobStartedReceiptPayload, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.pydantic import Receipt +from compute_horde.receipts.schemas import Receipt from django.utils.timezone import now from compute_horde_validator.validator.models import ( From 41776907947f5b3ee228f96f7cec4b0fdda5673d Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Fri, 11 Oct 2024 17:17:30 +0200 Subject: [PATCH 25/99] reexport things from the dismembered "receipts" module. --- compute_horde/compute_horde/receipts/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compute_horde/compute_horde/receipts/__init__.py b/compute_horde/compute_horde/receipts/__init__.py index 4fc57ee42..385ed2933 100644 --- a/compute_horde/compute_horde/receipts/__init__.py +++ b/compute_horde/compute_horde/receipts/__init__.py @@ -1 +1,12 @@ +from compute_horde.receipts.schemas import Receipt, ReceiptType +from compute_horde.receipts.transfer import ReceiptFetchError, get_miner_receipts + default_app_config = "compute_horde.receipts.apps.ComputeHordeReceiptsConfig" + +# Reexported for compatibility. These were moved to submodules. +__all__ = [ + "Receipt", + "ReceiptType", + "get_miner_receipts", + "ReceiptFetchError", +] From 3a87505447c5033e6c5eacfb3bf7fdf14e2a4f4d Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 15 Oct 2024 21:07:24 +0200 Subject: [PATCH 26/99] added is_organic field to job started receipts --- .../compute_horde/miner_client/organic.py | 1 + .../mv_protocol/validator_requests.py | 1 + .../0002_jobstartedreceipt_is_organic.py | 18 ++++++++++++++++++ compute_horde/compute_horde/receipts/models.py | 2 ++ .../compute_horde/receipts/transfer.py | 1 + compute_horde/compute_horde/settings.py | 2 +- .../validator/synthetic_jobs/batch_run.py | 2 ++ 7 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 compute_horde/compute_horde/receipts/migrations/0002_jobstartedreceipt_is_organic.py diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index e6292dd3f..fc6fce626 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -222,6 +222,7 @@ def generate_job_started_receipt_message( executor_class=executor_class, time_accepted=time_accepted, max_timeout=max_timeout, + is_organic=True, ) return V0JobStartedReceiptRequest( payload=receipt_payload, diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index dc25891b1..d811c91a6 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -133,6 +133,7 @@ class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass time_accepted: datetime.datetime max_timeout: int # seconds + is_organic: bool = False @field_serializer("time_accepted") def serialize_dt(self, dt: datetime.datetime, _info): diff --git a/compute_horde/compute_horde/receipts/migrations/0002_jobstartedreceipt_is_organic.py b/compute_horde/compute_horde/receipts/migrations/0002_jobstartedreceipt_is_organic.py new file mode 100644 index 000000000..1ed6799e8 --- /dev/null +++ b/compute_horde/compute_horde/receipts/migrations/0002_jobstartedreceipt_is_organic.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-10-15 13:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("receipts", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="jobstartedreceipt", + name="is_organic", + field=models.BooleanField(default=False), + preserve_default=False, + ), + ] diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index b9166034a..9d6db2cb8 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -35,6 +35,7 @@ class JobStartedReceipt(AbstractReceipt): executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) time_accepted = models.DateTimeField() max_timeout = models.IntegerField() + is_organic = models.BooleanField() # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 objects: models.Manager["JobStartedReceipt"] @@ -51,6 +52,7 @@ def to_receipt(self) -> Receipt: executor_class=ExecutorClass(self.executor_class), time_accepted=self.time_accepted, max_timeout=self.max_timeout, + is_organic=self.is_organic, ), validator_signature=self.validator_signature, miner_signature=self.miner_signature, diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 071a468d2..825343ffe 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -56,6 +56,7 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: raw_receipt["time_accepted"] ), max_timeout=int(raw_receipt["max_timeout"]), + is_organic=raw_receipt.get("is_organic") == "True", ) case ReceiptType.JobFinishedReceipt: diff --git a/compute_horde/compute_horde/settings.py b/compute_horde/compute_horde/settings.py index 8fe544535..dc0ee5bb6 100644 --- a/compute_horde/compute_horde/settings.py +++ b/compute_horde/compute_horde/settings.py @@ -1 +1 @@ -INSTALLED_APPS = ["compute_horde.mv_protocol"] +INSTALLED_APPS = ["compute_horde.receipts"] diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index ecda12a22..bec9b29c8 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -697,6 +697,7 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: executor_class=ExecutorClass(job.executor_class), time_accepted=job.executor_response_time, max_timeout=max_timeout, + is_organic=False, ) job.job_started_receipt = V0JobStartedReceiptRequest( payload=payload, @@ -1516,6 +1517,7 @@ def _db_persist(ctx: BatchContext) -> None: executor_class=started_payload.executor_class, time_accepted=started_payload.time_accepted, max_timeout=started_payload.max_timeout, + is_organic=False, ) ) JobStartedReceipt.objects.bulk_create(job_started_receipts) From 3c3aa2a7b2af8524885a2154b24eafcb66e87a54 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski <152924171+kkowalski-reef@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:08:10 +0200 Subject: [PATCH 27/99] Update settings.py --- compute_horde/compute_horde/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/settings.py b/compute_horde/compute_horde/settings.py index 8fe544535..dc0ee5bb6 100644 --- a/compute_horde/compute_horde/settings.py +++ b/compute_horde/compute_horde/settings.py @@ -1 +1 @@ -INSTALLED_APPS = ["compute_horde.mv_protocol"] +INSTALLED_APPS = ["compute_horde.receipts"] From 3fa9db489fbc8d2097ed11b050382a2b05de55a3 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 15 Oct 2024 21:42:10 +0200 Subject: [PATCH 28/99] fix tests --- miner/app/src/compute_horde_miner/miner/tasks.py | 1 + validator/app/src/compute_horde_validator/validator/tasks.py | 1 + 2 files changed, 2 insertions(+) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index 8a3d987cc..b67639065 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -137,6 +137,7 @@ def get_receipts_from_old_miner(): executor_class=receipt.payload.executor_class, time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, + is_organic=receipt.payload.is_organic, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index f19bdcd14..b39c31893 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1036,6 +1036,7 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): executor_class=receipt.payload.executor_class, time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, + is_organic=receipt.payload.is_organic, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) From f9a977e439678dcc7daa4bc0509e6b4c8988a8b3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Oct 2024 00:23:26 +0600 Subject: [PATCH 29/99] Fix run_organic_job exception --- compute_horde/compute_horde/miner_client/organic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index e6292dd3f..94e9b0d9b 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -390,7 +390,7 @@ async def run_organic_job( except TimeoutError as exc: raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc if isinstance(executor_readiness_response, V0ExecutorFailedRequest): - raise OrganicJobError(FailureReason.EXECUTOR_FAILED, initial_response) + raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) await client.send_job_started_receipt_message( executor_class=job_details.executor_class, From bfe806fed9da7d9434add3c9c5e54aaa5b7041f1 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Oct 2024 00:25:00 +0600 Subject: [PATCH 30/99] Add notify hooks for success stages in run_organic_job --- .../compute_horde/miner_client/organic.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index 94e9b0d9b..2f410f9b3 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -145,6 +145,12 @@ async def notify_receipt_failure(self, comment: str) -> None: async def notify_send_failure(self, msg: str) -> None: """This method is called when sending messages to miner fails""" + async def notify_job_accepted(self, msg: V0AcceptJobRequest) -> None: + """This method is called when miner sends job accepted message""" + + async def notify_executor_ready(self, msg: V0ExecutorReadyRequest) -> None: + """This method is called when miner sends executor ready message""" + async def handle_manifest_request(self, msg: V0ExecutorManifestRequest) -> None: try: self.miner_manifest.set_result(msg.manifest) @@ -313,7 +319,7 @@ def __init__(self, reason: FailureReason, received: BaseRequest | None = None): self.received = received def __str__(self): - s = f"Organic job failed, {self.reason=}" + s = f"Organic job failed, received: {self.received_str()}" if self.received: s += f", {self.received=}" return s @@ -321,6 +327,11 @@ def __str__(self): def __repr__(self): return f"{type(self).__name__}: {str(self)}" + def received_str(self) -> str: + if not self.received: + return "" + return self.received.model_dump_json() + @dataclass class OrganicJobDetails: @@ -382,6 +393,8 @@ async def run_organic_job( if isinstance(initial_response, V0DeclineJobRequest): raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) + await client.notify_job_accepted(initial_response) + try: executor_readiness_response = await asyncio.wait_for( client.executor_ready_or_failed_future, @@ -392,6 +405,8 @@ async def run_organic_job( if isinstance(executor_readiness_response, V0ExecutorFailedRequest): raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) + await client.notify_executor_ready(executor_readiness_response) + await client.send_job_started_receipt_message( executor_class=job_details.executor_class, accepted_timestamp=time.time(), From f3f335dc9ab23434103527f9d65c51b6488cc03e Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Oct 2024 00:30:41 +0600 Subject: [PATCH 31/99] Remove duplicate code for organic jobs from validator --- .../validator/organic_jobs/miner_driver.py | 267 ++++++------------ .../validator/tests/test_miner_driver.py | 2 +- 2 files changed, 86 insertions(+), 183 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 8680fcc70..22a972562 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -1,25 +1,18 @@ -import asyncio -import contextlib import logging -import time +from collections.abc import Awaitable, Callable from functools import partial from typing import Literal -from compute_horde.base.docker import DockerRunOptionsPreset -from compute_horde.base.output_upload import ZipAndHttpPutUpload -from compute_horde.base.volume import InlineVolume, Volume, ZipUrlVolume +from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload +from compute_horde.base.volume import Volume, ZipUrlVolume from compute_horde.executor_class import ExecutorClass -from compute_horde.mv_protocol.miner_requests import ( - V0AcceptJobRequest, - V0DeclineJobRequest, - V0ExecutorFailedRequest, - V0ExecutorReadyRequest, - V0JobFailedRequest, - V0JobFinishedRequest, +from compute_horde.miner_client.organic import ( + FailureReason, + OrganicJobDetails, + OrganicJobError, + run_organic_job, ) -from compute_horde.mv_protocol.validator_requests import V0InitialJobRequest, V0JobRequest -from compute_horde.transport.base import TransportConnectionError -from compute_horde.utils import Timer +from compute_horde.mv_protocol.miner_requests import V0AcceptJobRequest, V0JobFailedRequest from django.conf import settings from pydantic import BaseModel, JsonValue @@ -29,8 +22,11 @@ OrganicJob, SystemEvent, ) -from compute_horde_validator.validator.organic_jobs.facilitator_api import V0FacilitatorJobRequest -from compute_horde_validator.validator.utils import get_dummy_inline_zip_volume +from compute_horde_validator.validator.organic_jobs.facilitator_api import ( + V0FacilitatorJobRequest, + V1FacilitatorJobRequest, +) +from compute_horde_validator.validator.organic_jobs.miner_client import MinerClient logger = logging.getLogger(__name__) @@ -92,24 +88,53 @@ async def save_job_execution_event( ) +async def _dummy_notify_callback(_: JobStatusUpdate) -> None: + pass + + async def execute_organic_job( - miner_client, - job, - job_request, + miner_client: MinerClient, + job: OrganicJob, + job_request: V0FacilitatorJobRequest | V1FacilitatorJobRequest | AdminJobRequest, total_job_timeout: int = 300, wait_timeout: int = 300, - notify_callback=None, + notify_callback: Callable[[JobStatusUpdate], Awaitable[None]] = _dummy_notify_callback, ): data = {"job_uuid": str(job.job_uuid), "miner_hotkey": miner_client.my_hotkey} save_event = partial(save_job_execution_event, data=data) - async def handle_send_error_event(msg: str): - await save_event(subtype=SystemEvent.EventSubType.MINER_SEND_ERROR, long_description=msg) + async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: + await notify_callback(JobStatusUpdate.from_job(job, "accepted", msg.message_type.value)) + + miner_client.notify_job_accepted = notify_job_accepted # type: ignore[method-assign] + + volume: Volume | None = None + output_upload: OutputUpload | None = None + if isinstance(job_request, V0FacilitatorJobRequest | AdminJobRequest): + if job_request.input_url: + volume = ZipUrlVolume(contents=str(job_request.input_url)) + if job_request.output_url: + output_upload = ZipAndHttpPutUpload(url=str(job_request.output_url)) + else: + volume = job_request.volume + output_upload = job_request.output_upload + + job_details = OrganicJobDetails( + job_uuid=str(job.job_uuid), + executor_class=ExecutorClass(job_request.executor_class), + docker_image=job_request.docker_image or None, + raw_script=job_request.raw_script or None, + docker_run_options_preset="nvidia_all" if job_request.use_gpu else "none", + docker_run_cmd=job_request.get_args(), + total_job_timeout=total_job_timeout, + volume=volume, + output=output_upload, + ) - async with contextlib.AsyncExitStack() as exit_stack: - try: - await exit_stack.enter_async_context(miner_client) - except TransportConnectionError as exc: + try: + stdout, stderr = await run_organic_job(miner_client, job_details, wait_timeout) + except OrganicJobError as exc: + if exc.reason == FailureReason.MINER_CONNECTION_FAILED: comment = f"Miner connection error: {exc}" job.status = OrganicJob.Status.FAILED job.comment = comment @@ -119,40 +144,8 @@ async def handle_send_error_event(msg: str): await save_event( subtype=SystemEvent.EventSubType.MINER_CONNECTION_ERROR, long_description=comment ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, status="failed")) - return - - job_timer = Timer(timeout=total_job_timeout) - - volume: Volume - if isinstance(job_request, V0FacilitatorJobRequest | AdminJobRequest): - if job_request.input_url: - volume = ZipUrlVolume(contents=str(job_request.input_url)) - else: - # TODO: after release it can be changed to None - with this line new protocol - # can be released in any order - volume = InlineVolume(contents=get_dummy_inline_zip_volume()) - else: - volume = job_request.volume - - await miner_client.send_model( - V0InitialJobRequest( - job_uuid=job.job_uuid, - executor_class=job_request.executor_class, - base_docker_image_name=job_request.docker_image or None, - timeout_seconds=total_job_timeout, - volume_type=volume.volume_type.value if volume else None, - ), - error_event_callback=handle_send_error_event, - ) - - try: - msg = await asyncio.wait_for( - miner_client.miner_accepting_or_declining_future, - timeout=min(job_timer.time_left(), wait_timeout), - ) - except TimeoutError: + await notify_callback(JobStatusUpdate.from_job(job, status="failed")) + elif exc.reason == FailureReason.INITIAL_RESPONSE_TIMED_OUT: comment = f"Miner {miner_client.miner_name} timed out while preparing executor for job {job.job_uuid} after {wait_timeout} seconds" job.status = OrganicJob.Status.FAILED job.comment = comment @@ -163,12 +156,9 @@ async def handle_send_error_event(msg: str): subtype=SystemEvent.EventSubType.JOB_NOT_STARTED, long_description=comment, ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, "failed")) - return - - if isinstance(msg, V0DeclineJobRequest): - comment = f"Miner {miner_client.miner_name} won't do job: {msg.model_dump_json()}" + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + elif exc.reason == FailureReason.JOB_DECLINED: + comment = f"Miner {miner_client.miner_name} won't do job: {exc.received_str()}" job.status = OrganicJob.Status.FAILED job.comment = comment await job.asave() @@ -178,20 +168,8 @@ async def handle_send_error_event(msg: str): subtype=SystemEvent.EventSubType.JOB_REJECTED, long_description=comment, ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, "rejected")) - return - elif isinstance(msg, V0AcceptJobRequest): - logger.debug(f"Miner {miner_client.miner_name} accepted job: {msg}") - else: - raise ValueError(f"Unexpected msg from miner {miner_client.miner_name}: {msg}") - - try: - msg = await asyncio.wait_for( - miner_client.executor_ready_or_failed_future, - timeout=min(job_timer.time_left(), wait_timeout), - ) - except TimeoutError: + await notify_callback(JobStatusUpdate.from_job(job, "rejected")) + elif exc.reason == FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT: comment = f"Miner {miner_client.miner_name} timed out while preparing executor for job {job.job_uuid} after {wait_timeout} seconds" job.status = OrganicJob.Status.FAILED job.comment = comment @@ -202,13 +180,10 @@ async def handle_send_error_event(msg: str): subtype=SystemEvent.EventSubType.JOB_NOT_STARTED, long_description=comment, ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, "failed")) - return - - if isinstance(msg, V0ExecutorFailedRequest): + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + elif exc.reason == FailureReason.EXECUTOR_FAILED: comment = ( - f"Miner {miner_client.miner_name} failed to start executor: {msg.model_dump_json()}" + f"Miner {miner_client.miner_name} failed to start executor: {exc.received_str()}" ) job.status = OrganicJob.Status.FAILED job.comment = comment @@ -219,59 +194,8 @@ async def handle_send_error_event(msg: str): subtype=SystemEvent.EventSubType.JOB_REJECTED, long_description=comment, ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, "failed")) - return - elif isinstance(msg, V0ExecutorReadyRequest): - logger.debug(f"Miner {miner_client.miner_name} ready for job: {msg}") - if notify_callback: - await notify_callback( - JobStatusUpdate.from_job(job, "accepted", msg.message_type.value) - ) - await miner_client.send_job_started_receipt_message( - executor_class=ExecutorClass(job.executor_class), - accepted_timestamp=time.time(), - max_timeout=int(job_timer.time_left()), - ) - else: - raise ValueError(f"Unexpected msg from miner {miner_client.miner_name}: {msg}") - - docker_run_options_preset: DockerRunOptionsPreset = ( - "nvidia_all" if job_request.use_gpu else "none" - ) - - if isinstance(job_request, V0FacilitatorJobRequest | AdminJobRequest): - if job_request.output_url: - output_upload = ZipAndHttpPutUpload( - url=str(job_request.output_url), - ) - else: - output_upload = None - else: - output_upload = job_request.output_upload - - await miner_client.send_model( - V0JobRequest( - job_uuid=job.job_uuid, - executor_class=job_request.executor_class, - docker_image_name=job_request.docker_image or None, - raw_script=job_request.raw_script or None, - docker_run_options_preset=docker_run_options_preset, - docker_run_cmd=job_request.get_args(), - volume=volume, # TODO: raw scripts - output_upload=output_upload, - ), - error_event_callback=handle_send_error_event, - ) - full_job_sent = time.time() - try: - msg = await asyncio.wait_for( - miner_client.miner_finished_or_failed_future, - timeout=job_timer.time_left(), - ) - time_took = miner_client.miner_finished_or_failed_timestamp - full_job_sent - logger.info(f"Miner took {time_took} seconds to finish {job.job_uuid}") - except TimeoutError: + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + elif exc.reason == FailureReason.FINAL_RESPONSE_TIMED_OUT: comment = f"Miner {miner_client.miner_name} timed out after {total_job_timeout} seconds" job.status = OrganicJob.Status.FAILED job.comment = comment @@ -281,50 +205,29 @@ async def handle_send_error_event(msg: str): await save_event( subtype=SystemEvent.EventSubType.JOB_EXECUTION_TIMEOUT, long_description=comment ) - if notify_callback: - await notify_callback(JobStatusUpdate.from_job(job, "failed")) - return - if isinstance(msg, V0JobFailedRequest): - comment = f"Miner {miner_client.miner_name} failed: {msg.model_dump_json()}" - job.stdout = msg.docker_process_stdout - job.stderr = msg.docker_process_stderr + await notify_callback(JobStatusUpdate.from_job(job, "failed")) + elif exc.reason == FailureReason.JOB_FAILED: + comment = f"Miner {miner_client.miner_name} failed: {exc.received_str()}" + if isinstance(exc.received, V0JobFailedRequest): + job.stdout = exc.received.docker_process_stdout + job.stderr = exc.received.docker_process_stderr job.status = OrganicJob.Status.FAILED job.comment = comment await job.asave() logger.info(comment) await save_event(subtype=SystemEvent.EventSubType.FAILURE, long_description=comment) - if notify_callback: - await notify_callback( - JobStatusUpdate.from_job(job, "failed", msg.message_type.value) - ) - return - elif isinstance(msg, V0JobFinishedRequest): - comment = f"Miner {miner_client.miner_name} finished: {msg.model_dump_json()}" - job.stdout = msg.docker_process_stdout - job.stderr = msg.docker_process_stderr - job.status = OrganicJob.Status.COMPLETED - job.comment = comment - await job.asave() - - logger.info(comment) - await save_event( - subtype=SystemEvent.EventSubType.SUCCESS, long_description=comment, success=True - ) - if notify_callback: - await notify_callback( - JobStatusUpdate.from_job(job, "completed", msg.message_type.value) - ) - await miner_client.send_job_finished_receipt_message( - started_timestamp=job_timer.start_time.timestamp(), - time_took_seconds=job_timer.passed_time(), - score=0, # no score for organic jobs (at least right now) - ) - return - else: - comment = f"Unexpected msg from miner {miner_client.miner_name}: {msg}" - logger.warning(comment) - await save_event( - subtype=SystemEvent.EventSubType.UNEXPECTED_MESSAGE, long_description=comment - ) - raise ValueError(comment) + await notify_callback(JobStatusUpdate.from_job(job, "failed", "V0JobFailedRequest")) + else: + comment = f"Miner {miner_client.miner_name} finished: {stdout=} {stderr=}" + job.stdout = stdout + job.stderr = stderr + job.status = OrganicJob.Status.COMPLETED + job.comment = comment + await job.asave() + + logger.info(comment) + await save_event( + subtype=SystemEvent.EventSubType.SUCCESS, long_description=comment, success=True + ) + await notify_callback(JobStatusUpdate.from_job(job, "completed", "V0JobFinishedRequest")) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index bad2cebae..6120d4d95 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -59,7 +59,7 @@ ), ( (V0AcceptJobRequest, V0ExecutorFailedRequest, None), - ["failed"], + ["accepted", "failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, False, From 92e6edd2a186d9bab2186cacc1d1af1ec870271d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 12 Oct 2024 01:00:55 +0600 Subject: [PATCH 32/99] =?UTF-8?q?mypy=20=C2=AF\=5F(=E3=83=84)=5F/=C2=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (╯°□°)╯︵ ┻━┻ --- .../validator/organic_jobs/miner_driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 22a972562..b28825088 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -77,7 +77,7 @@ def from_job(job: JobBase, status, message_type=None) -> "JobStatusUpdate": async def save_job_execution_event( subtype: str, long_description: str, data: JsonValue = None, success: bool = False -): +) -> None: await SystemEvent.objects.using(settings.DEFAULT_DB_ALIAS).acreate( type=SystemEvent.EventType.MINER_ORGANIC_JOB_SUCCESS if success @@ -99,8 +99,8 @@ async def execute_organic_job( total_job_timeout: int = 300, wait_timeout: int = 300, notify_callback: Callable[[JobStatusUpdate], Awaitable[None]] = _dummy_notify_callback, -): - data = {"job_uuid": str(job.job_uuid), "miner_hotkey": miner_client.my_hotkey} +) -> None: + data: JsonValue = {"job_uuid": str(job.job_uuid), "miner_hotkey": miner_client.my_hotkey} save_event = partial(save_job_execution_event, data=data) async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: From 3823241d9c7a098218b634b5232df09fafea4e69 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Oct 2024 14:13:36 +0600 Subject: [PATCH 33/99] Refactor volume and output_upload of organic jobs --- .../compute_horde_validator/validator/models.py | 14 ++++++++++++++ .../validator/organic_jobs/facilitator_api.py | 16 ++++++++++++++-- .../validator/organic_jobs/miner_driver.py | 17 ++--------------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/models.py b/validator/app/src/compute_horde_validator/validator/models.py index 6e6a8b3de..7ebec6261 100644 --- a/validator/app/src/compute_horde_validator/validator/models.py +++ b/validator/app/src/compute_horde_validator/validator/models.py @@ -3,6 +3,8 @@ import uuid from os import urandom +from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload +from compute_horde.base.volume import Volume, ZipUrlVolume from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from django.contrib.postgres.fields import ArrayField from django.db import models @@ -238,6 +240,18 @@ def get_args(self): def __str__(self): return f"uuid: {self.uuid} - miner hotkey: {self.miner.hotkey}" + @property + def volume(self) -> Volume | None: + if self.input_url: + return ZipUrlVolume(contents=self.input_url) + return None + + @property + def output_upload(self) -> OutputUpload | None: + if self.output_url: + return ZipAndHttpPutUpload(url=self.output_url) + return None + def get_random_salt() -> list[int]: return list(urandom(8)) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py index a0a7cb77c..fa2028e07 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py @@ -2,8 +2,8 @@ import bittensor import pydantic -from compute_horde.base.output_upload import OutputUpload -from compute_horde.base.volume import Volume +from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload +from compute_horde.base.volume import Volume, ZipUrlVolume from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass from pydantic import BaseModel, model_validator @@ -64,6 +64,18 @@ def validate_at_least_docker_image_or_raw_script(self) -> Self: raise ValueError("Expected at least one of `docker_image` or `raw_script`") return self + @property + def volume(self) -> Volume | None: + if self.input_url: + return ZipUrlVolume(contents=self.input_url) + return None + + @property + def output_upload(self) -> OutputUpload | None: + if self.output_url: + return ZipAndHttpPutUpload(url=self.output_url) + return None + class V1FacilitatorJobRequest(BaseModel, extra="forbid"): """Message sent from facilitator to validator to request a job execution""" diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index b28825088..4ca1ccf2a 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -3,8 +3,6 @@ from functools import partial from typing import Literal -from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload -from compute_horde.base.volume import Volume, ZipUrlVolume from compute_horde.executor_class import ExecutorClass from compute_horde.miner_client.organic import ( FailureReason, @@ -108,17 +106,6 @@ async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: miner_client.notify_job_accepted = notify_job_accepted # type: ignore[method-assign] - volume: Volume | None = None - output_upload: OutputUpload | None = None - if isinstance(job_request, V0FacilitatorJobRequest | AdminJobRequest): - if job_request.input_url: - volume = ZipUrlVolume(contents=str(job_request.input_url)) - if job_request.output_url: - output_upload = ZipAndHttpPutUpload(url=str(job_request.output_url)) - else: - volume = job_request.volume - output_upload = job_request.output_upload - job_details = OrganicJobDetails( job_uuid=str(job.job_uuid), executor_class=ExecutorClass(job_request.executor_class), @@ -127,8 +114,8 @@ async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: docker_run_options_preset="nvidia_all" if job_request.use_gpu else "none", docker_run_cmd=job_request.get_args(), total_job_timeout=total_job_timeout, - volume=volume, - output=output_upload, + volume=job_request.volume, + output=job_request.output_upload, ) try: From 53fa3f77d9c4d468a73fdac71ab5ba2ce242ec5d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Oct 2024 14:16:10 +0600 Subject: [PATCH 34/99] Refactor simplify organic jobs code --- .../validator/organic_jobs/miner_driver.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 4ca1ccf2a..036f25ec7 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -120,6 +120,19 @@ async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: try: stdout, stderr = await run_organic_job(miner_client, job_details, wait_timeout) + + comment = f"Miner {miner_client.miner_name} finished: {stdout=} {stderr=}" + job.stdout = stdout + job.stderr = stderr + job.status = OrganicJob.Status.COMPLETED + job.comment = comment + await job.asave() + + logger.info(comment) + await save_event( + subtype=SystemEvent.EventSubType.SUCCESS, long_description=comment, success=True + ) + await notify_callback(JobStatusUpdate.from_job(job, "completed", "V0JobFinishedRequest")) except OrganicJobError as exc: if exc.reason == FailureReason.MINER_CONNECTION_FAILED: comment = f"Miner connection error: {exc}" @@ -205,16 +218,3 @@ async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: logger.info(comment) await save_event(subtype=SystemEvent.EventSubType.FAILURE, long_description=comment) await notify_callback(JobStatusUpdate.from_job(job, "failed", "V0JobFailedRequest")) - else: - comment = f"Miner {miner_client.miner_name} finished: {stdout=} {stderr=}" - job.stdout = stdout - job.stderr = stderr - job.status = OrganicJob.Status.COMPLETED - job.comment = comment - await job.asave() - - logger.info(comment) - await save_event( - subtype=SystemEvent.EventSubType.SUCCESS, long_description=comment, success=True - ) - await notify_callback(JobStatusUpdate.from_job(job, "completed", "V0JobFinishedRequest")) From 276b3fadd0a4b290f25d30ffe9f83bc15649076a Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Oct 2024 14:46:51 +0600 Subject: [PATCH 35/99] Add TODO for organic job method-assign issue --- .../validator/organic_jobs/miner_driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 036f25ec7..a4d9057c0 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -105,6 +105,7 @@ async def notify_job_accepted(msg: V0AcceptJobRequest) -> None: await notify_callback(JobStatusUpdate.from_job(job, "accepted", msg.message_type.value)) miner_client.notify_job_accepted = notify_job_accepted # type: ignore[method-assign] + # TODO: remove method assignment above and properly handle notify_* cases job_details = OrganicJobDetails( job_uuid=str(job.job_uuid), From 8dfa8d85500ed5157f8680159f8694d33540f091 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 16 Oct 2024 21:29:29 +0600 Subject: [PATCH 36/99] Move V0InitialJobRequest to bottom of file (for easier review) --- .../mv_protocol/validator_requests.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index dc25891b1..0bb4e6dec 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -50,21 +50,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0InitialJobRequest(BaseValidatorRequest, JobMixin): - message_type: RequestType = RequestType.V0InitialJobRequest - executor_class: ExecutorClass | None = None - base_docker_image_name: str | None = None - timeout_seconds: int | None = None - volume: Volume | None = None - volume_type: VolumeType | None = None - - @model_validator(mode="after") - def validate_volume_or_volume_type(self) -> Self: - if bool(self.volume) and bool(self.volume_type): - raise ValueError("Expected either `volume` or `volume_type`, got both") - return self - - class V0JobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0JobRequest executor_class: ExecutorClass | None = None @@ -146,3 +131,18 @@ class V0JobStartedReceiptRequest(BaseValidatorRequest): def blob_for_signing(self): return self.payload.blob_for_signing() + + +class V0InitialJobRequest(BaseValidatorRequest, JobMixin): + message_type: RequestType = RequestType.V0InitialJobRequest + executor_class: ExecutorClass | None = None + base_docker_image_name: str | None = None + timeout_seconds: int | None = None + volume: Volume | None = None + volume_type: VolumeType | None = None + + @model_validator(mode="after") + def validate_volume_or_volume_type(self) -> Self: + if bool(self.volume) and bool(self.volume_type): + raise ValueError("Expected either `volume` or `volume_type`, got both") + return self From 3f28a4c22b3ab4651afc64e7b58b281279546cc3 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 02:05:44 +0600 Subject: [PATCH 37/99] Allow sending job started receipt with initial job request --- .../compute_horde/mv_protocol/validator_requests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 0bb4e6dec..5cf368860 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -116,10 +116,11 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass - time_accepted: datetime.datetime + time_accepted: datetime.datetime | None max_timeout: int # seconds + ttl: int | None = None # seconds - @field_serializer("time_accepted") + @field_serializer("time_accepted", when_used="unless-none") def serialize_dt(self, dt: datetime.datetime, _info): return dt.isoformat() @@ -141,6 +142,9 @@ class V0InitialJobRequest(BaseValidatorRequest, JobMixin): volume: Volume | None = None volume_type: VolumeType | None = None + job_started_receipt_payload: JobStartedReceiptPayload | None = None + job_started_receipt_signature: str | None = None + @model_validator(mode="after") def validate_volume_or_volume_type(self) -> Self: if bool(self.volume) and bool(self.volume_type): From 972ba4bc975ed7b6208b8060ae53787053b4b087 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:01:49 +0600 Subject: [PATCH 38/99] Add timestamp field to all receipts --- .../compute_horde/mv_protocol/validator_requests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 5cf368860..1cc65d50d 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -81,6 +81,11 @@ class ReceiptPayload(pydantic.BaseModel): job_uuid: str miner_hotkey: str validator_hotkey: str + timestamp: datetime.datetime # when the receipt was generated + + @field_serializer("timestamp") + def serialize_timestamp(self, dt: datetime.datetime, _info): + return dt.isoformat() def blob_for_signing(self): # pydantic v2 does not support sort_keys anymore. From b40f833dac9a4be74ab48a92043ddef933af84a4 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:02:44 +0600 Subject: [PATCH 39/99] Handle empty string cases for None --- .../mv_protocol/validator_requests.py | 8 ++++---- compute_horde/compute_horde/utils.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 1cc65d50d..85a2017a1 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -2,7 +2,7 @@ import enum import json import re -from typing import Self +from typing import Annotated, Self import pydantic from pydantic import field_serializer, model_validator @@ -12,7 +12,7 @@ from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass -from ..utils import MachineSpecs, _json_dumps_default +from ..utils import MachineSpecs, _json_dumps_default, empty_string_none SAFE_DOMAIN_REGEX = re.compile(r".*") @@ -121,9 +121,9 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass - time_accepted: datetime.datetime | None + time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds - ttl: int | None = None # seconds + ttl: Annotated[int | None, empty_string_none] = None # seconds @field_serializer("time_accepted", when_used="unless-none") def serialize_dt(self, dt: datetime.datetime, _info): diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 1b95e343e..60d6d71c8 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -3,6 +3,7 @@ import bittensor import pydantic +from pydantic import BeforeValidator from substrateinterface.exceptions import SubstrateRequestException if TYPE_CHECKING: @@ -71,3 +72,16 @@ def time_left(self): if self.timeout is None: raise ValueError("timeout was not specified") return self.timeout - self.passed_time() + + +def _empty_string_none(value: Any) -> Any: + """ + Converts value to None if it is empty-string, otherwise returns the same value. + Intended to be used with pydantic validators. + """ + if value == "": + return None + return value + + +empty_string_none = BeforeValidator(_empty_string_none) From 24be520908391be1cc3fdb0790a737f825136820 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 17 Oct 2024 04:15:03 +0600 Subject: [PATCH 40/99] Add JobStillRunningReceipt --- .../mv_protocol/validator_requests.py | 14 ++++++++++++++ compute_horde/compute_horde/receipts/schemas.py | 4 +++- compute_horde/compute_horde/receipts/transfer.py | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 85a2017a1..ce6344236 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -24,6 +24,7 @@ class RequestType(enum.Enum): V0JobRequest = "V0JobRequest" V0JobFinishedReceiptRequest = "V0JobFinishedReceiptRequest" V0JobStartedReceiptRequest = "V0JobStartedReceiptRequest" + V0JobStillRunningReceiptRequest = "V0JobStillRunningReceiptRequest" GenericError = "GenericError" @@ -155,3 +156,16 @@ def validate_volume_or_volume_type(self) -> Self: if bool(self.volume) and bool(self.volume_type): raise ValueError("Expected either `volume` or `volume_type`, got both") return self + + +class JobStillRunningReceiptPayload(ReceiptPayload): + pass + + +class V0JobStillRunningReceiptRequest(BaseValidatorRequest): + message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest + payload: JobStillRunningReceiptPayload + signature: str + + def blob_for_signing(self): + return self.payload.blob_for_signing() diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 7d8cfc94d..af74ea5df 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -7,6 +7,7 @@ from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + JobStillRunningReceiptPayload, ) logger = logging.getLogger(__name__) @@ -15,10 +16,11 @@ class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" JobFinishedReceipt = "JobFinishedReceipt" + JobStillRunningReceipt = "JobStillRunningReceipt" class Receipt(pydantic.BaseModel): - payload: JobStartedReceiptPayload | JobFinishedReceiptPayload + payload: JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload validator_signature: str miner_signature: str diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 071a468d2..58013eec1 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -13,6 +13,7 @@ from compute_horde.mv_protocol.validator_requests import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + JobStillRunningReceiptPayload, ) from compute_horde.receipts.schemas import Receipt, ReceiptType @@ -43,7 +44,11 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: for raw_receipt in csv_reader: try: receipt_type = ReceiptType(raw_receipt["type"]) - receipt_payload: JobStartedReceiptPayload | JobFinishedReceiptPayload + receipt_payload: ( + JobStartedReceiptPayload + | JobFinishedReceiptPayload + | JobStillRunningReceiptPayload + ) match receipt_type: case ReceiptType.JobStartedReceipt: @@ -70,6 +75,14 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: score_str=raw_receipt["score_str"], ) + case ReceiptType.JobStillRunningReceipt: + receipt_payload = JobStillRunningReceiptPayload( + job_uuid=raw_receipt["job_uuid"], + miner_hotkey=raw_receipt["miner_hotkey"], + validator_hotkey=raw_receipt["validator_hotkey"], + timestamp=datetime.datetime.fromisoformat(raw_receipt["timestamp"]), + ) + receipt = Receipt( payload=receipt_payload, validator_signature=raw_receipt["validator_signature"], From d566d8fbf78f39339020ea7ddd2c654cf544a256 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:42:51 +0600 Subject: [PATCH 41/99] Move receipt payloads from validator_requests.py to schemas.py --- .../compute_horde/miner_client/organic.py | 3 +- .../mv_protocol/validator_requests.py | 60 +++--------------- .../compute_horde/receipts/models.py | 4 +- .../compute_horde/receipts/schemas.py | 61 +++++++++++++++++-- .../compute_horde/receipts/transfer.py | 5 +- compute_horde/tests/conftest.py | 4 +- compute_horde/tests/test_receipts.py | 5 +- 7 files changed, 73 insertions(+), 69 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index 2f410f9b3..a0855dd4c 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -34,14 +34,13 @@ ) from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, - JobFinishedReceiptPayload, - JobStartedReceiptPayload, V0AuthenticateRequest, V0InitialJobRequest, V0JobFinishedReceiptRequest, V0JobRequest, V0JobStartedReceiptRequest, ) +from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload from compute_horde.transport import AbstractTransport, TransportConnectionError, WSTransport from compute_horde.utils import MachineSpecs, Timer diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index ce6344236..d9c2c48d1 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -1,18 +1,22 @@ -import datetime import enum import json import re -from typing import Annotated, Self +from typing import Self import pydantic -from pydantic import field_serializer, model_validator +from pydantic import model_validator from ..base.docker import DockerRunOptionsPreset from ..base.output_upload import OutputUpload # noqa from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass -from ..utils import MachineSpecs, _json_dumps_default, empty_string_none +from ..receipts.schemas import ( + JobFinishedReceiptPayload, + JobStartedReceiptPayload, + JobStillRunningReceiptPayload, +) +from ..utils import MachineSpecs SAFE_DOMAIN_REGEX = re.compile(r".*") @@ -78,39 +82,6 @@ class GenericError(BaseValidatorRequest): details: str | None = None -class ReceiptPayload(pydantic.BaseModel): - job_uuid: str - miner_hotkey: str - validator_hotkey: str - timestamp: datetime.datetime # when the receipt was generated - - @field_serializer("timestamp") - def serialize_timestamp(self, dt: datetime.datetime, _info): - return dt.isoformat() - - def blob_for_signing(self): - # pydantic v2 does not support sort_keys anymore. - return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) - - -class JobFinishedReceiptPayload(ReceiptPayload): - time_started: datetime.datetime - time_took_us: int # micro-seconds - score_str: str - - @property - def time_took(self): - return datetime.timedelta(microseconds=self.time_took_us) - - @property - def score(self): - return float(self.score_str) - - @field_serializer("time_started") - def serialize_dt(self, dt: datetime.datetime, _info): - return dt.isoformat() - - class V0JobFinishedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobFinishedReceiptRequest payload: JobFinishedReceiptPayload @@ -120,17 +91,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class JobStartedReceiptPayload(ReceiptPayload): - executor_class: ExecutorClass - time_accepted: Annotated[datetime.datetime | None, empty_string_none] - max_timeout: int # seconds - ttl: Annotated[int | None, empty_string_none] = None # seconds - - @field_serializer("time_accepted", when_used="unless-none") - def serialize_dt(self, dt: datetime.datetime, _info): - return dt.isoformat() - - class V0JobStartedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobStartedReceiptRequest payload: JobStartedReceiptPayload @@ -158,10 +118,6 @@ def validate_volume_or_volume_type(self) -> Self: return self -class JobStillRunningReceiptPayload(ReceiptPayload): - pass - - class V0JobStillRunningReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest payload: JobStillRunningReceiptPayload diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index b9166034a..642f7f37b 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -3,11 +3,11 @@ from django.db import models from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt class ReceiptNotSigned(Exception): diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index af74ea5df..627499b6f 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,22 +1,71 @@ +import datetime import enum +import json import logging +from typing import Annotated import bittensor import pydantic +from pydantic import field_serializer -from compute_horde.mv_protocol.validator_requests import ( - JobFinishedReceiptPayload, - JobStartedReceiptPayload, - JobStillRunningReceiptPayload, -) +from compute_horde.executor_class import ExecutorClass +from compute_horde.utils import _json_dumps_default, empty_string_none logger = logging.getLogger(__name__) class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" - JobFinishedReceipt = "JobFinishedReceipt" JobStillRunningReceipt = "JobStillRunningReceipt" + JobFinishedReceipt = "JobFinishedReceipt" + + +class ReceiptPayload(pydantic.BaseModel): + job_uuid: str + miner_hotkey: str + validator_hotkey: str + timestamp: datetime.datetime # when the receipt was generated + + @field_serializer("timestamp") + def serialize_timestamp(self, dt: datetime.datetime, _info): + return dt.isoformat() + + def blob_for_signing(self): + # pydantic v2 does not support sort_keys anymore. + return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) + + +class JobStartedReceiptPayload(ReceiptPayload): + executor_class: ExecutorClass + time_accepted: Annotated[datetime.datetime | None, empty_string_none] + max_timeout: int # seconds + ttl: Annotated[int | None, empty_string_none] = None # seconds + + @field_serializer("time_accepted", when_used="unless-none") + def serialize_dt(self, dt: datetime.datetime, _info): + return dt.isoformat() + + +class JobStillRunningReceiptPayload(ReceiptPayload): + pass + + +class JobFinishedReceiptPayload(ReceiptPayload): + time_started: datetime.datetime + time_took_us: int # micro-seconds + score_str: str + + @property + def time_took(self): + return datetime.timedelta(microseconds=self.time_took_us) + + @property + def score(self): + return float(self.score_str) + + @field_serializer("time_started") + def serialize_dt(self, dt: datetime.datetime, _info): + return dt.isoformat() class Receipt(pydantic.BaseModel): diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 58013eec1..a2d3df727 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -10,12 +10,13 @@ import requests from compute_horde.executor_class import ExecutorClass -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, JobStillRunningReceiptPayload, + Receipt, + ReceiptType, ) -from compute_horde.receipts.schemas import Receipt, ReceiptType logger = logging.getLogger(__name__) diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index bde902a63..cc4fd69b7 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -5,11 +5,11 @@ from bittensor import Keypair from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index 545f16a47..ef412e4fd 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -3,11 +3,10 @@ import pytest -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, -) -from compute_horde.receipts.schemas import ( + JobStillRunningReceiptPayload, Receipt, ReceiptType, ) From 28c35db88d4b55d92124052854b92da579c7b664 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:53:35 +0600 Subject: [PATCH 42/99] Add receipt_type discriminator to receipt payloads --- compute_horde/compute_horde/receipts/schemas.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 627499b6f..a002bc6ca 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -6,7 +6,7 @@ import bittensor import pydantic -from pydantic import field_serializer +from pydantic import Field, field_serializer from compute_horde.executor_class import ExecutorClass from compute_horde.utils import _json_dumps_default, empty_string_none @@ -21,6 +21,7 @@ class ReceiptType(enum.Enum): class ReceiptPayload(pydantic.BaseModel): + receipt_type: ReceiptType job_uuid: str miner_hotkey: str validator_hotkey: str @@ -36,6 +37,7 @@ def blob_for_signing(self): class JobStartedReceiptPayload(ReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds @@ -47,10 +49,11 @@ def serialize_dt(self, dt: datetime.datetime, _info): class JobStillRunningReceiptPayload(ReceiptPayload): - pass + receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt class JobFinishedReceiptPayload(ReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds score_str: str @@ -68,8 +71,14 @@ def serialize_dt(self, dt: datetime.datetime, _info): return dt.isoformat() +ReceiptPayload = Annotated[ + JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, + Field(discriminator="receipt_type"), +] + + class Receipt(pydantic.BaseModel): - payload: JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload + payload: ReceiptPayload validator_signature: str miner_signature: str From bc8d687bb717b59a268e53482c050f52a62bedba Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 01:59:15 +0600 Subject: [PATCH 43/99] Misc changes in schemas.py --- .../compute_horde/receipts/schemas.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index a002bc6ca..7b91deaa6 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,18 +1,14 @@ import datetime import enum import json -import logging from typing import Annotated import bittensor -import pydantic -from pydantic import Field, field_serializer +from pydantic import Field, field_serializer, BaseModel from compute_horde.executor_class import ExecutorClass from compute_horde.utils import _json_dumps_default, empty_string_none -logger = logging.getLogger(__name__) - class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" @@ -20,7 +16,7 @@ class ReceiptType(enum.Enum): JobFinishedReceipt = "JobFinishedReceipt" -class ReceiptPayload(pydantic.BaseModel): +class BaseReceiptPayload(BaseModel): receipt_type: ReceiptType job_uuid: str miner_hotkey: str @@ -36,7 +32,7 @@ def blob_for_signing(self): return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) -class JobStartedReceiptPayload(ReceiptPayload): +class JobStartedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass time_accepted: Annotated[datetime.datetime | None, empty_string_none] @@ -44,15 +40,15 @@ class JobStartedReceiptPayload(ReceiptPayload): ttl: Annotated[int | None, empty_string_none] = None # seconds @field_serializer("time_accepted", when_used="unless-none") - def serialize_dt(self, dt: datetime.datetime, _info): + def serialize_time_accepted(self, dt: datetime.datetime, _info): return dt.isoformat() -class JobStillRunningReceiptPayload(ReceiptPayload): +class JobStillRunningReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt -class JobFinishedReceiptPayload(ReceiptPayload): +class JobFinishedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds @@ -67,7 +63,7 @@ def score(self): return float(self.score_str) @field_serializer("time_started") - def serialize_dt(self, dt: datetime.datetime, _info): + def serialize_time_started(self, dt: datetime.datetime, _info): return dt.isoformat() @@ -77,7 +73,7 @@ def serialize_dt(self, dt: datetime.datetime, _info): ] -class Receipt(pydantic.BaseModel): +class Receipt(BaseModel): payload: ReceiptPayload validator_signature: str miner_signature: str From 17a2e1811fbd7b8f4596ecf490780b8b0f29ea66 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 02:05:30 +0600 Subject: [PATCH 44/99] Use pydantic serialization of datetime objects in receipts --- .../compute_horde/receipts/schemas.py | 18 +++--------------- compute_horde/compute_horde/utils.py | 7 ------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 7b91deaa6..775e980ea 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -4,10 +4,10 @@ from typing import Annotated import bittensor -from pydantic import Field, field_serializer, BaseModel +from pydantic import Field, BaseModel from compute_horde.executor_class import ExecutorClass -from compute_horde.utils import _json_dumps_default, empty_string_none +from compute_horde.utils import empty_string_none class ReceiptType(enum.Enum): @@ -23,13 +23,9 @@ class BaseReceiptPayload(BaseModel): validator_hotkey: str timestamp: datetime.datetime # when the receipt was generated - @field_serializer("timestamp") - def serialize_timestamp(self, dt: datetime.datetime, _info): - return dt.isoformat() - def blob_for_signing(self): # pydantic v2 does not support sort_keys anymore. - return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) + return json.dumps(self.model_dump(mode="json"), sort_keys=True) class JobStartedReceiptPayload(BaseReceiptPayload): @@ -39,10 +35,6 @@ class JobStartedReceiptPayload(BaseReceiptPayload): max_timeout: int # seconds ttl: Annotated[int | None, empty_string_none] = None # seconds - @field_serializer("time_accepted", when_used="unless-none") - def serialize_time_accepted(self, dt: datetime.datetime, _info): - return dt.isoformat() - class JobStillRunningReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt @@ -62,10 +54,6 @@ def time_took(self): def score(self): return float(self.score_str) - @field_serializer("time_started") - def serialize_time_started(self, dt: datetime.datetime, _info): - return dt.isoformat() - ReceiptPayload = Annotated[ JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 60d6d71c8..1be8f2a8d 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -53,13 +53,6 @@ def get_validators(netuid=12, network="finney", block: int | None = None) -> lis return neurons[:VALIDATORS_LIMIT] -def _json_dumps_default(obj): - if isinstance(obj, datetime.datetime): - return obj.isoformat() - - raise TypeError - - class Timer: def __init__(self, timeout=None): self.start_time = datetime.datetime.now() From 8edf43dc22e4d5ef7f55bd07ae0012f4c8c05921 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 02:13:36 +0600 Subject: [PATCH 45/99] Use JobStartedReceiptPayload only in V0InitialJobRequest Also remove V0JobStartedReceiptRequest --- .../mv_protocol/validator_requests.py | 14 ++------------ compute_horde/compute_horde/receipts/schemas.py | 4 +--- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index d9c2c48d1..5ed2e0a50 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -91,15 +91,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0JobStartedReceiptRequest(BaseValidatorRequest): - message_type: RequestType = RequestType.V0JobStartedReceiptRequest - payload: JobStartedReceiptPayload - signature: str - - def blob_for_signing(self): - return self.payload.blob_for_signing() - - class V0InitialJobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0InitialJobRequest executor_class: ExecutorClass | None = None @@ -107,9 +98,8 @@ class V0InitialJobRequest(BaseValidatorRequest, JobMixin): timeout_seconds: int | None = None volume: Volume | None = None volume_type: VolumeType | None = None - - job_started_receipt_payload: JobStartedReceiptPayload | None = None - job_started_receipt_signature: str | None = None + job_started_receipt_payload: JobStartedReceiptPayload + job_started_receipt_signature: str @model_validator(mode="after") def validate_volume_or_volume_type(self) -> Self: diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 775e980ea..be69f4017 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -7,7 +7,6 @@ from pydantic import Field, BaseModel from compute_horde.executor_class import ExecutorClass -from compute_horde.utils import empty_string_none class ReceiptType(enum.Enum): @@ -31,9 +30,8 @@ def blob_for_signing(self): class JobStartedReceiptPayload(BaseReceiptPayload): receipt_type: ReceiptType = ReceiptType.JobStartedReceipt executor_class: ExecutorClass - time_accepted: Annotated[datetime.datetime | None, empty_string_none] max_timeout: int # seconds - ttl: Annotated[int | None, empty_string_none] = None # seconds + ttl: int class JobStillRunningReceiptPayload(BaseReceiptPayload): From 707dc7fd6eabfed4cd198dc2742265220f726066 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:01:32 +0600 Subject: [PATCH 46/99] Rename JobStillRunningReceipt -> JobAcceptedReceipt --- .../compute_horde/mv_protocol/validator_requests.py | 11 +++++------ compute_horde/compute_horde/receipts/schemas.py | 12 +++++++----- compute_horde/compute_horde/receipts/transfer.py | 10 ++++------ compute_horde/tests/test_receipts.py | 1 - 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 5ed2e0a50..547dc5412 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -12,9 +12,9 @@ from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass from ..receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, ) from ..utils import MachineSpecs @@ -26,9 +26,8 @@ class RequestType(enum.Enum): V0InitialJobRequest = "V0InitialJobRequest" V0MachineSpecsRequest = "V0MachineSpecsRequest" V0JobRequest = "V0JobRequest" + V0JobAcceptedReceiptRequest = "V0JobAcceptedReceiptRequest" V0JobFinishedReceiptRequest = "V0JobFinishedReceiptRequest" - V0JobStartedReceiptRequest = "V0JobStartedReceiptRequest" - V0JobStillRunningReceiptRequest = "V0JobStillRunningReceiptRequest" GenericError = "GenericError" @@ -108,9 +107,9 @@ def validate_volume_or_volume_type(self) -> Self: return self -class V0JobStillRunningReceiptRequest(BaseValidatorRequest): - message_type: RequestType = RequestType.V0JobStillRunningReceiptRequest - payload: JobStillRunningReceiptPayload +class V0JobAcceptedReceiptRequest(BaseValidatorRequest): + message_type: RequestType = RequestType.V0JobAcceptedReceiptRequest + payload: JobAcceptedReceiptPayload signature: str def blob_for_signing(self): diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index be69f4017..b01115400 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -4,14 +4,14 @@ from typing import Annotated import bittensor -from pydantic import Field, BaseModel +from pydantic import BaseModel, Field from compute_horde.executor_class import ExecutorClass class ReceiptType(enum.Enum): JobStartedReceipt = "JobStartedReceipt" - JobStillRunningReceipt = "JobStillRunningReceipt" + JobAcceptedReceipt = "JobAcceptedReceipt" JobFinishedReceipt = "JobFinishedReceipt" @@ -34,8 +34,10 @@ class JobStartedReceiptPayload(BaseReceiptPayload): ttl: int -class JobStillRunningReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobStillRunningReceipt +class JobAcceptedReceiptPayload(BaseReceiptPayload): + receipt_type: ReceiptType = ReceiptType.JobAcceptedReceipt + time_accepted: datetime.datetime + ttl: int class JobFinishedReceiptPayload(BaseReceiptPayload): @@ -54,7 +56,7 @@ def score(self): ReceiptPayload = Annotated[ - JobStartedReceiptPayload | JobFinishedReceiptPayload | JobStillRunningReceiptPayload, + JobStartedReceiptPayload | JobAcceptedReceiptPayload | JobFinishedReceiptPayload, Field(discriminator="receipt_type"), ] diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index a2d3df727..6f8671c61 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -11,9 +11,9 @@ from compute_horde.executor_class import ExecutorClass from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, Receipt, ReceiptType, ) @@ -46,9 +46,7 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: try: receipt_type = ReceiptType(raw_receipt["type"]) receipt_payload: ( - JobStartedReceiptPayload - | JobFinishedReceiptPayload - | JobStillRunningReceiptPayload + JobStartedReceiptPayload | JobFinishedReceiptPayload | JobAcceptedReceiptPayload ) match receipt_type: @@ -76,8 +74,8 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: score_str=raw_receipt["score_str"], ) - case ReceiptType.JobStillRunningReceipt: - receipt_payload = JobStillRunningReceiptPayload( + case ReceiptType.JobAcceptedReceipt: + receipt_payload = JobAcceptedReceiptPayload( job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index ef412e4fd..baa8619b5 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -6,7 +6,6 @@ from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, - JobStillRunningReceiptPayload, Receipt, ReceiptType, ) From 8630ce3d774035d0cd4e106a37683ae44abaa530 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:03:12 +0600 Subject: [PATCH 47/99] Undo move of V0InitialJobRequest --- .../mv_protocol/validator_requests.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 547dc5412..326422c15 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -54,6 +54,23 @@ def blob_for_signing(self): return self.payload.blob_for_signing() +class V0InitialJobRequest(BaseValidatorRequest, JobMixin): + message_type: RequestType = RequestType.V0InitialJobRequest + executor_class: ExecutorClass | None = None + base_docker_image_name: str | None = None + timeout_seconds: int | None = None + volume: Volume | None = None + volume_type: VolumeType | None = None + job_started_receipt_payload: JobStartedReceiptPayload + job_started_receipt_signature: str + + @model_validator(mode="after") + def validate_volume_or_volume_type(self) -> Self: + if bool(self.volume) and bool(self.volume_type): + raise ValueError("Expected either `volume` or `volume_type`, got both") + return self + + class V0JobRequest(BaseValidatorRequest, JobMixin): message_type: RequestType = RequestType.V0JobRequest executor_class: ExecutorClass | None = None @@ -90,23 +107,6 @@ def blob_for_signing(self): return self.payload.blob_for_signing() -class V0InitialJobRequest(BaseValidatorRequest, JobMixin): - message_type: RequestType = RequestType.V0InitialJobRequest - executor_class: ExecutorClass | None = None - base_docker_image_name: str | None = None - timeout_seconds: int | None = None - volume: Volume | None = None - volume_type: VolumeType | None = None - job_started_receipt_payload: JobStartedReceiptPayload - job_started_receipt_signature: str - - @model_validator(mode="after") - def validate_volume_or_volume_type(self) -> Self: - if bool(self.volume) and bool(self.volume_type): - raise ValueError("Expected either `volume` or `volume_type`, got both") - return self - - class V0JobAcceptedReceiptRequest(BaseValidatorRequest): message_type: RequestType = RequestType.V0JobAcceptedReceiptRequest payload: JobAcceptedReceiptPayload From 53c08873de3fd5c9a94d97e64cdadfda67d68163 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 03:15:03 +0600 Subject: [PATCH 48/99] Fix receipt payload discriminator --- compute_horde/compute_horde/receipts/schemas.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index b01115400..6d56930d8 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -1,7 +1,7 @@ import datetime import enum import json -from typing import Annotated +from typing import Annotated, Literal import bittensor from pydantic import BaseModel, Field @@ -16,7 +16,6 @@ class ReceiptType(enum.Enum): class BaseReceiptPayload(BaseModel): - receipt_type: ReceiptType job_uuid: str miner_hotkey: str validator_hotkey: str @@ -28,20 +27,20 @@ def blob_for_signing(self): class JobStartedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobStartedReceipt + receipt_type: Literal[ReceiptType.JobStartedReceipt] = ReceiptType.JobStartedReceipt executor_class: ExecutorClass max_timeout: int # seconds ttl: int class JobAcceptedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobAcceptedReceipt + receipt_type: Literal[ReceiptType.JobAcceptedReceipt] = ReceiptType.JobAcceptedReceipt time_accepted: datetime.datetime ttl: int class JobFinishedReceiptPayload(BaseReceiptPayload): - receipt_type: ReceiptType = ReceiptType.JobFinishedReceipt + receipt_type: Literal[ReceiptType.JobFinishedReceipt] = ReceiptType.JobFinishedReceipt time_started: datetime.datetime time_took_us: int # micro-seconds score_str: str From 7761bcc8714dd14c555837357379394cf32b816b Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 04:14:16 +0600 Subject: [PATCH 49/99] Add receipt model fields and fix mypy errors of lib --- .../compute_horde/miner_client/organic.py | 73 +++++++++++++------ .../compute_horde/receipts/models.py | 33 ++++++++- .../compute_horde/receipts/transfer.py | 20 +++-- 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index a0855dd4c..f66b4d7bb 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -13,7 +13,7 @@ from compute_horde.base.output_upload import OutputUpload from compute_horde.base.volume import Volume from compute_horde.base_requests import BaseRequest -from compute_horde.executor_class import ExecutorClass +from compute_horde.executor_class import EXECUTOR_CLASS, ExecutorClass from compute_horde.miner_client.base import ( AbstractMinerClient, ErrorCallback, @@ -36,16 +36,22 @@ AuthenticationPayload, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) -from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) from compute_horde.transport import AbstractTransport, TransportConnectionError, WSTransport from compute_horde.utils import MachineSpecs, Timer logger = logging.getLogger(__name__) +JOB_STARTED_RECEIPT_MIN_TTL = 30 + class OrganicMinerClient(AbstractMinerClient): """ @@ -216,39 +222,54 @@ def generate_authentication_message(self) -> V0AuthenticateRequest: def generate_job_started_receipt_message( self, executor_class: ExecutorClass, - accepted_timestamp: float, max_timeout: int, - ) -> V0JobStartedReceiptRequest: - time_accepted = datetime.datetime.fromtimestamp(accepted_timestamp, datetime.UTC) + ttl: int, + ) -> tuple[JobStartedReceiptPayload, str]: receipt_payload = JobStartedReceiptPayload( job_uuid=self.job_uuid, miner_hotkey=self.miner_hotkey, validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), executor_class=executor_class, - time_accepted=time_accepted, max_timeout=max_timeout, + ttl=ttl, + ) + signature = f"0x{self.my_keypair.sign(receipt_payload.blob_for_signing()).hex()}" + return receipt_payload, signature + + def generate_job_accepted_receipt_message( + self, + accepted_timestamp: float, + ttl: int, + ) -> V0JobAcceptedReceiptRequest: + time_accepted = datetime.datetime.fromtimestamp(accepted_timestamp, datetime.UTC) + receipt_payload = JobAcceptedReceiptPayload( + job_uuid=self.job_uuid, + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), + time_accepted=time_accepted, + ttl=ttl, ) - return V0JobStartedReceiptRequest( + return V0JobAcceptedReceiptRequest( payload=receipt_payload, signature=f"0x{self.my_keypair.sign(receipt_payload.blob_for_signing()).hex()}", ) - async def send_job_started_receipt_message( + async def send_job_accepted_receipt_message( self, - executor_class: ExecutorClass, accepted_timestamp: float, - max_timeout: int, + ttl: int, ) -> None: try: - receipt_message = self.generate_job_started_receipt_message( - executor_class, + receipt_message = self.generate_job_accepted_receipt_message( accepted_timestamp, - max_timeout, + ttl, ) await self.send_model(receipt_message) - logger.debug(f"Sent job started receipt for {self.job_uuid}") + logger.debug(f"Sent job accepted receipt for {self.job_uuid}") except Exception as e: - comment = f"Failed to send job started receipt to miner {self.miner_name} for job {self.job_uuid}: {e}" + comment = f"Failed to send job accepted receipt to miner {self.miner_name} for job {self.job_uuid}: {e}" logger.warning(comment) await self.notify_receipt_failure(comment) @@ -263,6 +284,7 @@ def generate_job_finished_receipt_message( job_uuid=self.job_uuid, miner_hotkey=self.miner_hotkey, validator_hotkey=self.my_hotkey, + timestamp=datetime.datetime.now(datetime.UTC), time_started=time_started, time_took_us=int(time_took_seconds * 1_000_000), score_str=f"{score:.6f}", @@ -318,9 +340,9 @@ def __init__(self, reason: FailureReason, received: BaseRequest | None = None): self.received = received def __str__(self): - s = f"Organic job failed, received: {self.received_str()}" + s = f"Organic job failed, {self.reason=}" if self.received: - s += f", {self.received=}" + s += f", received: {self.received_str()}" return s def __repr__(self): @@ -372,6 +394,14 @@ async def run_organic_job( job_timer = Timer(timeout=job_details.total_job_timeout) + receipt_payload, receipt_signature = client.generate_job_started_receipt_message( + executor_class=job_details.executor_class, + max_timeout=int(job_timer.time_left()), + ttl=max( + JOB_STARTED_RECEIPT_MIN_TTL, + EXECUTOR_CLASS[job_details.executor_class].spin_up_time or 0, + ), + ) await client.send_model( V0InitialJobRequest( job_uuid=job_details.job_uuid, @@ -379,6 +409,8 @@ async def run_organic_job( base_docker_image_name=job_details.docker_image, timeout_seconds=job_details.total_job_timeout, volume_type=job_details.volume.volume_type if job_details.volume else None, + job_started_receipt_payload=receipt_payload, + job_started_receipt_signature=receipt_signature, ), ) @@ -406,10 +438,9 @@ async def run_organic_job( await client.notify_executor_ready(executor_readiness_response) - await client.send_job_started_receipt_message( - executor_class=job_details.executor_class, + await client.send_job_accepted_receipt_message( accepted_timestamp=time.time(), - max_timeout=int(job_timer.time_left()), + ttl=int(job_timer.time_left()), ) await client.send_model( diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 642f7f37b..9298f408b 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -4,6 +4,7 @@ from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, @@ -20,6 +21,10 @@ class AbstractReceipt(models.Model): miner_hotkey = models.CharField(max_length=256) validator_signature = models.CharField(max_length=256) miner_signature = models.CharField(max_length=256, null=True, blank=True) + timestamp = models.DateTimeField() + + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobStartedReceipt"] class Meta: abstract = True @@ -33,8 +38,8 @@ def __str__(self): class JobStartedReceipt(AbstractReceipt): executor_class = models.CharField(max_length=255, default=DEFAULT_EXECUTOR_CLASS) - time_accepted = models.DateTimeField() max_timeout = models.IntegerField() + ttl = models.IntegerField() # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 objects: models.Manager["JobStartedReceipt"] @@ -48,9 +53,32 @@ def to_receipt(self) -> Receipt: job_uuid=str(self.job_uuid), miner_hotkey=self.miner_hotkey, validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, executor_class=ExecutorClass(self.executor_class), - time_accepted=self.time_accepted, max_timeout=self.max_timeout, + ttl=self.ttl, + ), + validator_signature=self.validator_signature, + miner_signature=self.miner_signature, + ) + + +class JobAcceptedReceipt(AbstractReceipt): + time_accepted = models.DateTimeField() + ttl = models.IntegerField() + + def to_receipt(self) -> Receipt: + if self.miner_signature is None: + raise ReceiptNotSigned("Miner signature is required") + + return Receipt( + payload=JobAcceptedReceiptPayload( + job_uuid=str(self.job_uuid), + miner_hotkey=self.miner_hotkey, + validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, + time_accepted=self.time_accepted, + ttl=self.ttl, ), validator_signature=self.validator_signature, miner_signature=self.miner_signature, @@ -80,6 +108,7 @@ def to_receipt(self) -> Receipt: job_uuid=str(self.job_uuid), miner_hotkey=self.miner_hotkey, validator_hotkey=self.validator_hotkey, + timestamp=self.timestamp, time_started=self.time_started, time_took_us=self.time_took_us, score_str=self.score_str, diff --git a/compute_horde/compute_horde/receipts/transfer.py b/compute_horde/compute_horde/receipts/transfer.py index 6f8671c61..adcc737a4 100644 --- a/compute_horde/compute_horde/receipts/transfer.py +++ b/compute_horde/compute_horde/receipts/transfer.py @@ -1,6 +1,5 @@ import contextlib import csv -import datetime import io import logging import shutil @@ -15,6 +14,7 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, + ReceiptPayload, ReceiptType, ) @@ -45,9 +45,7 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: for raw_receipt in csv_reader: try: receipt_type = ReceiptType(raw_receipt["type"]) - receipt_payload: ( - JobStartedReceiptPayload | JobFinishedReceiptPayload | JobAcceptedReceiptPayload - ) + receipt_payload: ReceiptPayload match receipt_type: case ReceiptType.JobStartedReceipt: @@ -55,11 +53,10 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] executor_class=ExecutorClass(raw_receipt["executor_class"]), - time_accepted=datetime.datetime.fromisoformat( - raw_receipt["time_accepted"] - ), max_timeout=int(raw_receipt["max_timeout"]), + ttl=int(raw_receipt["ttl"]), ) case ReceiptType.JobFinishedReceipt: @@ -67,9 +64,8 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], - time_started=datetime.datetime.fromisoformat( - raw_receipt["time_started"] - ), + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] + time_started=raw_receipt["time_started"], # type: ignore[arg-type] time_took_us=int(raw_receipt["time_took_us"]), score_str=raw_receipt["score_str"], ) @@ -79,7 +75,9 @@ def get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=raw_receipt["job_uuid"], miner_hotkey=raw_receipt["miner_hotkey"], validator_hotkey=raw_receipt["validator_hotkey"], - timestamp=datetime.datetime.fromisoformat(raw_receipt["timestamp"]), + timestamp=raw_receipt["timestamp"], # type: ignore[arg-type] + time_accepted=raw_receipt["time_accepted"], # type: ignore[arg-type] + ttl=int(raw_receipt["ttl"]), ) receipt = Receipt( From 81f337f0575d9cad097cbabb4f73c312ae42e841 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 18 Oct 2024 14:46:40 +0600 Subject: [PATCH 50/99] Send finished receipt when organic job fails --- .../compute_horde/miner_client/organic.py | 103 ++++++++++-------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index f66b4d7bb..32fd456ae 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -415,60 +415,69 @@ async def run_organic_job( ) try: - initial_response = await asyncio.wait_for( - client.miner_accepting_or_declining_future, - timeout=min(job_timer.time_left(), wait_timeout), - ) - except TimeoutError as exc: - raise OrganicJobError(FailureReason.INITIAL_RESPONSE_TIMED_OUT) from exc - if isinstance(initial_response, V0DeclineJobRequest): - raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) + try: + initial_response = await asyncio.wait_for( + client.miner_accepting_or_declining_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError as exc: + raise OrganicJobError(FailureReason.INITIAL_RESPONSE_TIMED_OUT) from exc + if isinstance(initial_response, V0DeclineJobRequest): + raise OrganicJobError(FailureReason.JOB_DECLINED, initial_response) - await client.notify_job_accepted(initial_response) + await client.notify_job_accepted(initial_response) - try: - executor_readiness_response = await asyncio.wait_for( - client.executor_ready_or_failed_future, - timeout=min(job_timer.time_left(), wait_timeout), + try: + executor_readiness_response = await asyncio.wait_for( + client.executor_ready_or_failed_future, + timeout=min(job_timer.time_left(), wait_timeout), + ) + except TimeoutError as exc: + raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc + if isinstance(executor_readiness_response, V0ExecutorFailedRequest): + raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) + + await client.notify_executor_ready(executor_readiness_response) + + await client.send_job_accepted_receipt_message( + accepted_timestamp=time.time(), + ttl=int(job_timer.time_left()), ) - except TimeoutError as exc: - raise OrganicJobError(FailureReason.EXECUTOR_READINESS_RESPONSE_TIMED_OUT) from exc - if isinstance(executor_readiness_response, V0ExecutorFailedRequest): - raise OrganicJobError(FailureReason.EXECUTOR_FAILED, executor_readiness_response) - - await client.notify_executor_ready(executor_readiness_response) - - await client.send_job_accepted_receipt_message( - accepted_timestamp=time.time(), - ttl=int(job_timer.time_left()), - ) - await client.send_model( - V0JobRequest( - job_uuid=job_details.job_uuid, - executor_class=job_details.executor_class, - docker_image_name=job_details.docker_image, - raw_script=job_details.raw_script, - docker_run_options_preset=job_details.docker_run_options_preset, - docker_run_cmd=job_details.docker_run_cmd, - volume=job_details.volume, - output_upload=job_details.output, + await client.send_model( + V0JobRequest( + job_uuid=job_details.job_uuid, + executor_class=job_details.executor_class, + docker_image_name=job_details.docker_image, + raw_script=job_details.raw_script, + docker_run_options_preset=job_details.docker_run_options_preset, + docker_run_cmd=job_details.docker_run_cmd, + volume=job_details.volume, + output_upload=job_details.output, + ) ) - ) - try: - final_response = await asyncio.wait_for( - client.miner_finished_or_failed_future, - timeout=job_timer.time_left(), - ) - if isinstance(final_response, V0JobFailedRequest): - raise OrganicJobError(FailureReason.JOB_FAILED, final_response) - return final_response.docker_process_stdout, final_response.docker_process_stderr - except TimeoutError as exc: - raise OrganicJobError(FailureReason.FINAL_RESPONSE_TIMED_OUT) from exc - finally: + try: + final_response = await asyncio.wait_for( + client.miner_finished_or_failed_future, + timeout=job_timer.time_left(), + ) + if isinstance(final_response, V0JobFailedRequest): + raise OrganicJobError(FailureReason.JOB_FAILED, final_response) + + await client.send_job_finished_receipt_message( + started_timestamp=job_timer.start_time.timestamp(), + time_took_seconds=job_timer.passed_time(), + score=0, # no score for organic jobs (at least right now) + ) + + return final_response.docker_process_stdout, final_response.docker_process_stderr + except TimeoutError as exc: + raise OrganicJobError(FailureReason.FINAL_RESPONSE_TIMED_OUT) from exc + except Exception: await client.send_job_finished_receipt_message( started_timestamp=job_timer.start_time.timestamp(), time_took_seconds=job_timer.passed_time(), - score=0, # no score for organic jobs (at least right now) + score=0, ) + raise From 7141448fcd9bd8aada30e4b3d1829dd043dc3657 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 00:53:27 +0600 Subject: [PATCH 51/99] Fix surface level errors in miner --- .../miner/liveness_check.py | 15 ++++ .../miner_consumer/validator_interface.py | 81 +++++++++++------ .../miner/receipt_store/local.py | 13 ++- .../src/compute_horde_miner/miner/tasks.py | 88 +++++++++++-------- 4 files changed, 124 insertions(+), 73 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/liveness_check.py b/miner/app/src/compute_horde_miner/miner/liveness_check.py index 486d1267d..c400a76a2 100644 --- a/miner/app/src/compute_horde_miner/miner/liveness_check.py +++ b/miner/app/src/compute_horde_miner/miner/liveness_check.py @@ -1,5 +1,6 @@ import asyncio import base64 +import datetime import io import json import os @@ -14,6 +15,7 @@ from compute_horde.base.volume import InlineVolume, VolumeType from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.mv_protocol import validator_requests +from compute_horde.receipts.schemas import JobStartedReceiptPayload from django.conf import settings from compute_horde_miner.channel_layer.channel_layer import ECRedisChannelLayer @@ -148,12 +150,25 @@ async def drive_executor() -> float: job_uuid = str(uuid.uuid4()) executor_token = f"{job_uuid}-{uuid.uuid4()}" + keypair = settings.BITTENSOR_WALLET().get_hotkey() + receipt_payload = JobStartedReceiptPayload( + job_uuid=job_uuid, + miner_hotkey=keypair.ss58_address, + validator_hotkey=keypair.ss58_address, + timestamp=datetime.datetime.now(datetime.UTC), + executor_class=DEFAULT_EXECUTOR_CLASS, + max_timeout=JOB_TIMEOUT_SECONDS, + ttl=30, + ) + receipt_signature = f"0x{keypair.sign(receipt_payload.blob_for_signing()).hex()}" initial_job_request = validator_requests.V0InitialJobRequest( job_uuid=job_uuid, executor_class=DEFAULT_EXECUTOR_CLASS, base_docker_image_name=JOB_IMAGE_NAME, timeout_seconds=JOB_TIMEOUT_SECONDS, volume_type=VolumeType.inline, + job_started_receipt_payload=receipt_payload, + job_started_receipt_signature=receipt_signature, ) job_request = validator_requests.V0JobRequest( job_uuid=job_uuid, diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index ee74158cd..5fb98ae82 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -10,7 +10,8 @@ from compute_horde.mv_protocol.validator_requests import ( BaseValidatorRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobStartedReceiptPayload, ReceiptPayload from django.conf import settings from django.utils import timezone @@ -156,32 +157,26 @@ def verify_auth_msg(self, msg: validator_requests.V0AuthenticateRequest) -> tupl return False, "Signature mismatches" - def verify_receipt_msg( - self, - msg: validator_requests.V0JobStartedReceiptRequest - | validator_requests.V0JobFinishedReceiptRequest, - ) -> bool: + def verify_receipt_payload(self, payload: ReceiptPayload, signature: str) -> bool: if settings.IS_LOCAL_MINER: return True - if self.my_hotkey != DONT_CHECK and msg.payload.miner_hotkey != self.my_hotkey: + if self.my_hotkey != DONT_CHECK and payload.miner_hotkey != self.my_hotkey: logger.warning( - f"Miner hotkey mismatch in receipt for job_uuid {msg.payload.job_uuid} ({msg.payload.miner_hotkey!r} != {self.my_hotkey!r})" + f"Miner hotkey mismatch in receipt for job_uuid {payload.job_uuid} ({payload.miner_hotkey!r} != {self.my_hotkey!r})" ) return False - if msg.payload.validator_hotkey != self.validator_key: + if payload.validator_hotkey != self.validator_key: logger.warning( - f"Validator hotkey mismatch in receipt for job_uuid {msg.payload.job_uuid} ({msg.payload.validator_hotkey!r} != {self.validator_key!r})" + f"Validator hotkey mismatch in receipt for job_uuid {payload.job_uuid} ({payload.validator_hotkey!r} != {self.validator_key!r})" ) return False keypair = bittensor.Keypair(ss58_address=self.validator_key) - if keypair.verify(msg.blob_for_signing(), msg.signature): + if keypair.verify(payload.blob_for_signing(), signature): return True - logger.warning( - f"Validator signature mismatch in receipt for job_uuid {msg.payload.job_uuid}" - ) + logger.warning(f"Validator signature mismatch in receipt for job_uuid {payload.job_uuid}") return False async def handle_authentication(self, msg: validator_requests.V0AuthenticateRequest): @@ -299,13 +294,13 @@ async def handle(self, msg: BaseValidatorRequest): await self.handle_job_request(msg) if isinstance( - msg, validator_requests.V0JobStartedReceiptRequest - ) and self.verify_receipt_msg(msg): - await self.handle_job_started_receipt(msg) + msg, validator_requests.V0JobAcceptedReceiptRequest + ) and self.verify_receipt_payload(msg.payload, msg.signature): + await self.handle_job_accepted_receipt(msg) if isinstance( msg, validator_requests.V0JobFinishedReceiptRequest - ) and self.verify_receipt_msg(msg): + ) and self.verify_receipt_payload(msg.payload, msg.signature): await self.handle_job_finished_receipt(msg) async def handle_initial_job_request(self, msg: validator_requests.V0InitialJobRequest): @@ -320,6 +315,11 @@ async def handle_initial_job_request(self, msg: validator_requests.V0InitialJobR miner_requests.V0DeclineJobRequest(job_uuid=msg.job_uuid).model_dump_json() ) return + + await self.handle_job_started_receipt( + msg.job_started_receipt_payload, msg.job_started_receipt_signature + ) + # TODO add rate limiting per validator key here token = f"{msg.job_uuid}-{uuid.uuid4()}" await self.group_add(token) @@ -378,26 +378,52 @@ async def handle_job_request(self, msg: validator_requests.V0JobRequest): job.full_job_details = msg.model_dump() await job.asave() - async def handle_job_started_receipt(self, msg: validator_requests.V0JobStartedReceiptRequest): + async def handle_job_started_receipt(self, payload: JobStartedReceiptPayload, signature: str): logger.info( f"Received job started receipt for" - f" job_uuid={msg.payload.job_uuid} validator_hotkey={msg.payload.validator_hotkey}" - f" max_timeout={msg.payload.max_timeout}" + f" job_uuid={payload.job_uuid} validator_hotkey={payload.validator_hotkey}" + f" max_timeout={payload.max_timeout}" ) if settings.IS_LOCAL_MINER: return + if not self.verify_receipt_payload(payload, signature): + return + await JobStartedReceipt.objects.acreate( - job_uuid=msg.payload.job_uuid, - validator_hotkey=msg.payload.validator_hotkey, - miner_hotkey=msg.payload.miner_hotkey, + job_uuid=payload.job_uuid, + validator_hotkey=payload.validator_hotkey, + miner_hotkey=payload.miner_hotkey, + validator_signature=signature, + miner_signature=get_miner_signature(payload), + timestamp=payload.timestamp, + executor_class=payload.executor_class, + max_timeout=payload.max_timeout, + ttl=payload.ttl, + ) + + async def handle_job_accepted_receipt( + self, msg: validator_requests.V0JobAcceptedReceiptRequest + ): + logger.info( + f"Received job accepted receipt for" + f" job_uuid={msg.payload.job_uuid} validator_hotkey={msg.payload.validator_hotkey}" + ) + + if settings.IS_LOCAL_MINER: + return + + await JobAcceptedReceipt.objects.acreate( validator_signature=msg.signature, miner_signature=get_miner_signature(msg), - executor_class=msg.payload.executor_class, - time_accepted=msg.payload.time_accepted, - max_timeout=msg.payload.max_timeout, + job_uuid=msg.payload.job_uuid, + miner_hotkey=msg.payload.miner_hotkey, + validator_hotkey=msg.payload.validator_hotkey, + timestamp=msg.payload.timestamp, + ttl=msg.payload.ttl, ) + prepare_receipts.delay() async def handle_job_finished_receipt( self, msg: validator_requests.V0JobFinishedReceiptRequest @@ -421,6 +447,7 @@ async def handle_job_finished_receipt( job_uuid=msg.payload.job_uuid, miner_hotkey=msg.payload.miner_hotkey, validator_hotkey=msg.payload.validator_hotkey, + timestamp=msg.payload.timestamp, time_started=msg.payload.time_started, time_took_us=msg.payload.time_took_us, score_str=msg.payload.score_str, diff --git a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py index 2aad9143e..c819ccd2f 100644 --- a/miner/app/src/compute_horde_miner/miner/receipt_store/local.py +++ b/miner/app/src/compute_horde_miner/miner/receipt_store/local.py @@ -4,11 +4,12 @@ import shutil import tempfile -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.schemas import Receipt, ReceiptType from django.conf import settings from compute_horde_miner.miner.receipt_store.base import BaseReceiptStore @@ -23,6 +24,7 @@ def store(self, receipts: list[Receipt]) -> None: payload_fields = set() payload_fields |= set(JobStartedReceiptPayload.model_fields.keys()) + payload_fields |= set(JobAcceptedReceiptPayload.model_fields.keys()) payload_fields |= set(JobFinishedReceiptPayload.model_fields.keys()) buf = io.StringIO() @@ -37,14 +39,9 @@ def store(self, receipts: list[Receipt]) -> None: ) csv_writer.writeheader() for receipt in receipts: - match receipt.payload: - case JobStartedReceiptPayload(): - receipt_type = ReceiptType.JobStartedReceipt - case JobFinishedReceiptPayload(): - receipt_type = ReceiptType.JobFinishedReceipt row = ( dict( - type=receipt_type.value, + type=receipt.payload.receipt_type.value, validator_signature=receipt.validator_signature, miner_signature=receipt.miner_signature, ) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index 8a3d987cc..cc7edf44f 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -2,11 +2,12 @@ from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import get_validators from constance import config @@ -70,27 +71,15 @@ def fetch_validators(): def prepare_receipts(): receipts = [] - job_started_receipts = JobStartedReceipt.objects.order_by("time_accepted").filter( - time_accepted__gt=now() - RECEIPTS_MAX_SERVED_PERIOD - ) - for job_started_receipt in job_started_receipts: - try: - receipts.append(job_started_receipt.to_receipt()) - except Exception as e: - logger.error( - f"Skipping job started receipt for job {job_started_receipt.job_uuid}: {e}" - ) - - job_finished_receipts = JobFinishedReceipt.objects.order_by("time_started").filter( - time_started__gt=now() - RECEIPTS_MAX_SERVED_PERIOD - ) - for job_finished_receipt in job_finished_receipts: - try: - receipts.append(job_finished_receipt.to_receipt()) - except Exception as e: - logger.error( - f"Skipping job finished receipt for job {job_finished_receipt.job_uuid}: {e}" - ) + for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: + db_objects = model.objects.order_by("timestamp").filter( + timestamp__gt=now() - RECEIPTS_MAX_SERVED_PERIOD + ) + for db_object in db_objects: + try: + receipts.append(db_object.to_receipt()) + except Exception as e: + logger.error(f"Skipping job started receipt for job {db_object.job_uuid}: {e}") logger.info(f"Stored receipts: {len(receipts)}") @@ -99,12 +88,8 @@ def prepare_receipts(): @app.task def clear_old_receipts(): - JobFinishedReceipt.objects.filter( - time_started__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD - ).delete() - JobStartedReceipt.objects.filter( - time_accepted__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD - ).delete() + for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: + model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() @app.task @@ -124,43 +109,70 @@ def get_receipts_from_old_miner(): tolerance = datetime.timedelta(hours=1) latest_job_started_receipt = ( - JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_accepted").first() + JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_started_receipt_cutoff_time = ( - latest_job_started_receipt.time_accepted - tolerance if latest_job_started_receipt else None + latest_job_started_receipt.timestamp - tolerance if latest_job_started_receipt else None ) job_started_receipt_to_create = [ JobStartedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, - time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, + ttl=receipt.payload.ttl, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) and ( job_started_receipt_cutoff_time is None - or receipt.payload.time_accepted > job_started_receipt_cutoff_time + or receipt.payload.timestamp > job_started_receipt_cutoff_time ) ] if job_started_receipt_to_create: JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) + latest_job_accepted_receipt = ( + JobAcceptedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() + ) + job_accepted_receipt_cutoff_time = ( + latest_job_accepted_receipt.timestamp - tolerance if latest_job_accepted_receipt else None + ) + job_accepted_receipt_to_create = [ + JobAcceptedReceipt( + job_uuid=receipt.payload.job_uuid, + miner_hotkey=receipt.payload.miner_hotkey, + validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, + time_accepted=receipt.payload.time_accepted, + ttl=receipt.payload.ttl, + ) + for receipt in receipts + if isinstance(receipt.payload, JobAcceptedReceiptPayload) + and ( + job_accepted_receipt_cutoff_time is None + or receipt.payload.timestamp > job_accepted_receipt_cutoff_time + ) + ] + if job_accepted_receipt_to_create: + JobAcceptedReceipt.objects.bulk_create( + job_accepted_receipt_to_create, ignore_conflicts=True + ) + latest_job_finished_receipt = ( - JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_started").first() + JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_finished_receipt_cutoff_time = ( - latest_job_finished_receipt.time_started - tolerance - if latest_job_finished_receipt - else None + latest_job_finished_receipt.timestamp - tolerance if latest_job_finished_receipt else None ) job_finished_receipt_to_create = [ JobFinishedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, score_str=receipt.payload.score_str, @@ -169,7 +181,7 @@ def get_receipts_from_old_miner(): if isinstance(receipt.payload, JobFinishedReceiptPayload) and ( job_finished_receipt_cutoff_time is None - or receipt.payload.time_started > job_finished_receipt_cutoff_time + or receipt.payload.timestamp > job_finished_receipt_cutoff_time ) ] if job_finished_receipt_to_create: From 9ec2636ba852219f9d506de6802ce4ba1b07163c Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 01:00:28 +0600 Subject: [PATCH 52/99] Fix receipts admin classes --- compute_horde/compute_horde/receipts/admin.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/compute_horde/compute_horde/receipts/admin.py b/compute_horde/compute_horde/receipts/admin.py index 712a15f79..411ac4641 100644 --- a/compute_horde/compute_horde/receipts/admin.py +++ b/compute_horde/compute_horde/receipts/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin # noqa from compute_horde.base.admin import ReadOnlyAdminMixin -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt, JobAcceptedReceipt class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): @@ -9,11 +9,24 @@ class JobStartedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): "job_uuid", "miner_hotkey", "validator_hotkey", + "timestamp", "executor_class", - "time_accepted", "max_timeout", + "ttl", + ] + ordering = ["-timestamp"] + + +class JobAcceptedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): + list_display = [ + "job_uuid", + "miner_hotkey", + "validator_hotkey", + "timestamp", + "time_accepted", + "ttl", ] - ordering = ["-time_accepted"] + ordering = ["-timestamp"] class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): @@ -21,12 +34,14 @@ class JobFinishedReceiptsReadOnlyAdmin(admin.ModelAdmin, ReadOnlyAdminMixin): "job_uuid", "miner_hotkey", "validator_hotkey", + "timestamp", "score", "time_started", "time_took", ] - ordering = ["-time_started"] + ordering = ["-timestamp"] admin.site.register(JobStartedReceipt, admin_class=JobStartedReceiptsReadOnlyAdmin) +admin.site.register(JobAcceptedReceipt, admin_class=JobAcceptedReceiptsReadOnlyAdmin) admin.site.register(JobFinishedReceipt, admin_class=JobFinishedReceiptsReadOnlyAdmin) From fe318a04e01fb399b8623e64726640cd479080c2 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 02:19:12 +0600 Subject: [PATCH 53/99] Fix miner tests --- .../compute_horde/receipts/schemas.py | 2 +- .../test_mocked_executor_manager.py | 77 ++++++++++++++----- .../miner/tests/test_migration.py | 28 +++++-- 3 files changed, 81 insertions(+), 26 deletions(-) diff --git a/compute_horde/compute_horde/receipts/schemas.py b/compute_horde/compute_horde/receipts/schemas.py index 6d56930d8..3af0b6c42 100644 --- a/compute_horde/compute_horde/receipts/schemas.py +++ b/compute_horde/compute_horde/receipts/schemas.py @@ -9,7 +9,7 @@ from compute_horde.executor_class import ExecutorClass -class ReceiptType(enum.Enum): +class ReceiptType(enum.StrEnum): JobStartedReceipt = "JobStartedReceipt" JobAcceptedReceipt = "JobAcceptedReceipt" JobFinishedReceipt = "JobFinishedReceipt" diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index cd099ea5f..460e39ad2 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -1,10 +1,12 @@ import contextlib +import json import time import uuid -from unittest.mock import MagicMock +from unittest.mock import Mock import pytest import pytest_asyncio +from bittensor import Keypair from channels.testing import WebsocketCommunicator from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from pytest_mock import MockerFixture @@ -20,9 +22,11 @@ @pytest.fixture def mock_keypair(mocker: MockerFixture): - return mocker.patch( - "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair" + mock = Mock(wraps=Keypair) + mocker.patch( + "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair", mock ) + return mock # Somehow the regular dependency mechanism doesn't work with multiple test cases @@ -44,8 +48,15 @@ def job_uuid(): @pytest.fixture -def validator_hotkey(): - return "some_public_key" +def validator_keypair(): + return Keypair.create_from_mnemonic( + "slot excuse valid grief praise rifle spoil auction weasel glove pen share" + ) + + +@pytest.fixture +def validator_hotkey(validator_keypair): + return validator_keypair.ss58_address @pytest_asyncio.fixture @@ -64,17 +75,20 @@ async def make_communicator(validator_key: str): await communicator.disconnect() -async def run_regular_flow_test(validator_key: str, job_uuid: str): - async with make_communicator(validator_key) as communicator: +async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, job_uuid: str): + async with make_communicator(validator_keypair.ss58_address) as communicator: + auth_payload = { + "validator_hotkey": validator_keypair.ss58_address, + "miner_hotkey": "some key", + "timestamp": int(time.time()), + } + auth_payload_blob = json.dumps(auth_payload, sort_keys=True) + auth_signature = f"0x{validator_keypair.sign(auth_payload_blob).hex()}" await communicator.send_json_to( { "message_type": "V0AuthenticateRequest", - "payload": { - "validator_hotkey": validator_key, - "miner_hotkey": "some key", - "timestamp": int(time.time()), - }, - "signature": "gibberish", + "payload": auth_payload, + "signature": auth_signature, } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -84,6 +98,18 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "executor_classes": [{"count": 1, "executor_class": DEFAULT_EXECUTOR_CLASS}] }, } + receipt = { + "receipt_type": "JobStartedReceipt", + "job_uuid": job_uuid, + "miner_hotkey": miner_hotkey, + "validator_hotkey": validator_keypair.ss58_address, + "timestamp": "2020-01-01T00:00Z", + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 5, + } + receipt_blob = json.dumps(receipt, sort_keys=True) + receipt_signature = f"0x{validator_keypair.sign(receipt_blob).hex()}" await communicator.send_json_to( { "message_type": "V0InitialJobRequest", @@ -92,6 +118,8 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "base_docker_image_name": "it's teeeeests", "timeout_seconds": 60, "volume_type": "inline", + "job_started_receipt_payload": receipt, + "job_started_receipt_signature": receipt_signature, } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -125,21 +153,32 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): } -async def test_main_loop(validator: Validator, job_uuid: str): - await run_regular_flow_test(validator.public_key, job_uuid) +async def test_main_loop(validator: Validator, validator_keypair: Keypair, job_uuid: str, settings): + await run_regular_flow_test( + validator_keypair, + settings.BITTENSOR_WALLET().hotkey.ss58_address, + job_uuid, + ) -async def test_local_miner(validator: Validator, job_uuid: str, mock_keypair: MagicMock, settings): +async def test_local_miner( + validator: Validator, + validator_keypair: Keypair, + job_uuid: str, + mock_keypair: Mock, + settings, +): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False - await run_regular_flow_test(validator.public_key, job_uuid) + await run_regular_flow_test( + validator_keypair, settings.BITTENSOR_WALLET().hotkey.ss58_address, job_uuid + ) mock_keypair.assert_called_once_with(ss58_address=validator.public_key) - mock_keypair.return_value.verify.assert_called_once() -async def test_local_miner_unknown_validator(mock_keypair: MagicMock, settings): +async def test_local_miner_unknown_validator(mock_keypair: Mock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 556195e33..224355579 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -1,4 +1,5 @@ import uuid +from datetime import UTC, datetime import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS @@ -6,8 +7,8 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import Receipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobAcceptedReceiptPayload, Receipt from django.utils.timezone import now from pytest_mock import MockerFixture @@ -47,9 +48,22 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): job_uuid=str(uuid.uuid4()), miner_hotkey="m1", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=now(), max_timeout=30, + ttl=5, + ), + validator_signature="0xv1", + miner_signature="0xm1", + ), + Receipt( + payload=JobAcceptedReceiptPayload( + job_uuid=str(uuid.uuid4()), + miner_hotkey="m1", + validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), + time_accepted=datetime(2020, 1, 1, tzinfo=UTC), + ttl=5, ), validator_signature="0xv1", miner_signature="0xm1", @@ -58,13 +72,14 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): payload=JobFinishedReceiptPayload( job_uuid=str(uuid.uuid4()), miner_hotkey="m1", - validator_hotkey="v2", + validator_hotkey="v3", + timestamp=datetime(2020, 1, 1, tzinfo=UTC), time_started=now(), time_took_us=35_000_000, score_str="103.45", ), - validator_signature="0xv2", - miner_signature="0xm2", + validator_signature="0xv3", + miner_signature="0xm3", ), ] mocker.patch("compute_horde_miner.miner.tasks.get_miner_receipts", return_value=receipts) @@ -73,4 +88,5 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): get_receipts_from_old_miner() assert JobStartedReceipt.objects.count() == 1 + assert JobAcceptedReceipt.objects.count() == 1 assert JobFinishedReceipt.objects.count() == 1 From 3c979116225ba535338dcbebb7fa515afe3166c7 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Sat, 19 Oct 2024 02:26:12 +0600 Subject: [PATCH 54/99] Add receipts migration --- ...obstartedreceipt_time_accepted_and_more.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py diff --git a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py new file mode 100644 index 000000000..232c72793 --- /dev/null +++ b/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 5.1.1 on 2024-10-18 19:07 + +import datetime + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("receipts", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="jobstartedreceipt", + name="time_accepted", + ), + migrations.AddField( + model_name="jobfinishedreceipt", + name="timestamp", + field=models.DateTimeField( + default=datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.UTC) + ), + preserve_default=False, + ), + migrations.AddField( + model_name="jobstartedreceipt", + name="timestamp", + field=models.DateTimeField( + default=datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.UTC) + ), + preserve_default=False, + ), + migrations.AddField( + model_name="jobstartedreceipt", + name="ttl", + field=models.IntegerField(default=0), + preserve_default=False, + ), + migrations.CreateModel( + name="JobAcceptedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), + ("timestamp", models.DateTimeField()), + ("time_accepted", models.DateTimeField()), + ("ttl", models.IntegerField()), + ], + options={ + "abstract": False, + "constraints": [ + models.UniqueConstraint( + fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" + ) + ], + }, + ), + ] From d8d1c9aaa2df98f718776c6331308d621f0b69ec Mon Sep 17 00:00:00 2001 From: Olzhas Arystanov Date: Mon, 7 Oct 2024 21:03:58 +0500 Subject: [PATCH 55/99] Configure vulnerability scanning workflow --- .github/workflows/vulnerability_scan.yml | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/vulnerability_scan.yml diff --git a/.github/workflows/vulnerability_scan.yml b/.github/workflows/vulnerability_scan.yml new file mode 100644 index 000000000..9a01bc5ff --- /dev/null +++ b/.github/workflows/vulnerability_scan.yml @@ -0,0 +1,43 @@ +name: "Vulnerability Scan" + +on: + schedule: + - cron: "0 0 * * *" + +env: + # temporary workaround for trivy db update issue + # https://github.com/aquasecurity/trivy/discussions/7538#discussioncomment-10741936 + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2 + +jobs: + trivy: + strategy: + matrix: + image_name: [ + "backenddevelopersltd/compute-horde-miner:v0-latest", + "backenddevelopersltd/compute-horde-miner-runner:v0-latest", + "backenddevelopersltd/compute-horde-miner-nginx:v0-latest", + "backenddevelopersltd/compute-horde-validator:v0-latest", + "backenddevelopersltd/compute-horde-validator-runner:v0-latest", + "backenddevelopersltd/compute-horde-validator-nginx:v0-latest", + "backenddevelopersltd/compute-horde-executor:v0-latest", + "backenddevelopersltd/compute-horde-job:v0-latest", + "backenddevelopersltd/compute-horde-job:v1-latest", + ] + runs-on: ubuntu-latest + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.24.0 + with: + image-ref: "${{ matrix.image_name }}" + scanners: "vuln" + severity: 'CRITICAL,HIGH,MEDIUM' + limit-severities-for-sarif: true + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' From 9941deeb3535630430bd74e9e735f20cc31ee5e8 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 16:52:39 +0200 Subject: [PATCH 56/99] save is_organic flag in receiver job started receipts --- .../miner/miner_consumer/validator_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index ee74158cd..2b2dfb0de 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -397,6 +397,7 @@ async def handle_job_started_receipt(self, msg: validator_requests.V0JobStartedR executor_class=msg.payload.executor_class, time_accepted=msg.payload.time_accepted, max_timeout=msg.payload.max_timeout, + is_organic=msg.payload.is_organic, ) async def handle_job_finished_receipt( From c58300fb75da6ad06d9430aa273574101dfdc067 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 16:52:54 +0200 Subject: [PATCH 57/99] test whether receipts get saved --- .../miner/tests/conftest.py | 15 +++ .../miner/tests/test_receipts.py | 92 +++++++++++++++++++ .../miner/tests/validator.py | 18 ++++ 3 files changed, 125 insertions(+) create mode 100644 miner/app/src/compute_horde_miner/miner/tests/test_receipts.py create mode 100644 miner/app/src/compute_horde_miner/miner/tests/validator.py diff --git a/miner/app/src/compute_horde_miner/miner/tests/conftest.py b/miner/app/src/compute_horde_miner/miner/tests/conftest.py index 7149037f2..15d42b46f 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/conftest.py +++ b/miner/app/src/compute_horde_miner/miner/tests/conftest.py @@ -1,8 +1,23 @@ from collections.abc import Generator +import bittensor import pytest +@pytest.fixture(scope="session") +def validator_wallet(): + wallet = bittensor.wallet(name="test_validator") + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + return wallet + + +@pytest.fixture(scope="function") +def miner_wallet(settings): + return settings.BITTENSOR_WALLET() + + @pytest.fixture def some() -> Generator[int, None, None]: # setup code diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py new file mode 100644 index 000000000..3b5c2f125 --- /dev/null +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -0,0 +1,92 @@ +from datetime import datetime +from datetime import datetime +from uuid import uuid4 + +import bittensor +import pytest +from compute_horde.executor_class import ExecutorClass +from compute_horde.mv_protocol.validator_requests import V0AuthenticateRequest, \ + AuthenticationPayload, V0JobStartedReceiptRequest, JobStartedReceiptPayload, JobFinishedReceiptPayload, \ + V0JobFinishedReceiptRequest +from compute_horde.receipts.models import JobStartedReceipt, JobFinishedReceipt +from django.utils import timezone +from pytest_mock import MockerFixture + +from compute_horde_miner.miner.models import Validator, AcceptedJob +from compute_horde_miner.miner.tests.validator import fake_validator + + +@pytest.mark.parametrize( + "organic_job", (True, False), +) +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_receipt_is_saved( + validator_wallet: bittensor.wallet, + miner_wallet: bittensor.wallet, + mocker: MockerFixture, + organic_job: bool, + settings, +) -> None: + mocker.patch("compute_horde_miner.miner.miner_consumer.validator_interface.prepare_receipts") + settings.DEBUG_TURN_AUTHENTICATION_OFF = True + job_uuid = str(uuid4()) + validator = await Validator.objects.acreate( + public_key=validator_wallet.hotkey.ss58_address, + active=True, + ) + + # 1. Authenticate to miner as test validator + # 2. Send JobStarted and JobFinished receiptss + async with fake_validator(validator_wallet) as fake_validator_channel: + # Authenticate, otherwise the consumer will refuse to talk to us + auth_payload = AuthenticationPayload( + validator_hotkey=validator_wallet.hotkey.ss58_address, + miner_hotkey=miner_wallet.hotkey.ss58_address, + timestamp=int(datetime.now().timestamp()), + ) + await fake_validator_channel.send_to(V0AuthenticateRequest( + payload=auth_payload, + signature=validator_wallet.hotkey.sign(auth_payload.blob_for_signing()).hex(), + ).model_dump_json()) + manifest = await fake_validator_channel.receive_json_from() + + # Skip doing the job + await AcceptedJob.objects.acreate( + job_uuid=job_uuid, + validator=validator, + initial_job_details={}, + ) + + # Send the receipts + job_started_receipt_payload = JobStartedReceiptPayload( + job_uuid=job_uuid, + is_organic=organic_job, + miner_hotkey=miner_wallet.hotkey.ss58_address, + validator_hotkey=validator_wallet.hotkey.ss58_address, + executor_class=ExecutorClass.spin_up_4min__gpu_24gb, + time_accepted=timezone.now(), + max_timeout=123, + ) + await fake_validator_channel.send_to(V0JobStartedReceiptRequest( + job_uuid=job_uuid, + payload=job_started_receipt_payload, + signature=f"0x{validator_wallet.hotkey.sign(job_started_receipt_payload.blob_for_signing()).hex()}", + ).model_dump_json()) + + job_finished_receipt_payload = JobFinishedReceiptPayload( + job_uuid=job_uuid, + miner_hotkey=miner_wallet.hotkey.ss58_address, + validator_hotkey=validator_wallet.hotkey.ss58_address, + time_started=timezone.now(), + time_took_us=123, + score_str="123.45", + ) + await fake_validator_channel.send_to(V0JobFinishedReceiptRequest( + job_uuid=job_uuid, + payload=job_finished_receipt_payload, + signature=f"0x{validator_wallet.hotkey.sign(job_finished_receipt_payload.blob_for_signing()).hex()}", + ).model_dump_json()) + + assert await JobStartedReceipt.objects.filter(job_uuid=job_uuid, is_organic=organic_job).aexists() + assert await JobFinishedReceipt.objects.filter(job_uuid=job_uuid).aexists() diff --git a/miner/app/src/compute_horde_miner/miner/tests/validator.py b/miner/app/src/compute_horde_miner/miner/tests/validator.py new file mode 100644 index 000000000..343c9bac9 --- /dev/null +++ b/miner/app/src/compute_horde_miner/miner/tests/validator.py @@ -0,0 +1,18 @@ +from contextlib import asynccontextmanager + +import bittensor +from channels.testing import WebsocketCommunicator + +from compute_horde_miner import asgi + + +@asynccontextmanager +async def fake_validator(validator_wallet: bittensor.wallet) -> WebsocketCommunicator: + communicator = WebsocketCommunicator( + application=asgi.application, + path=f"v0.1/validator_interface/{validator_wallet.hotkey.ss58_address}", + ) + connected, _ = await communicator.connect() + assert connected + yield communicator + await communicator.disconnect() From 00167a69965fe99db41e46c562fb3ffbe1013f56 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 16:58:02 +0200 Subject: [PATCH 58/99] fix miner wallet fixture --- miner/app/src/compute_horde_miner/miner/tests/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/conftest.py b/miner/app/src/compute_horde_miner/miner/tests/conftest.py index 15d42b46f..90ad532c2 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/conftest.py +++ b/miner/app/src/compute_horde_miner/miner/tests/conftest.py @@ -15,7 +15,11 @@ def validator_wallet(): @pytest.fixture(scope="function") def miner_wallet(settings): - return settings.BITTENSOR_WALLET() + wallet = bittensor.wallet(name="test_validator") + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="2" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="3" * 64, use_password=False, overwrite=True) + return wallet @pytest.fixture From 84e50b75b85e73a29356c3be703994b5ea870e2b Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 16:58:13 +0200 Subject: [PATCH 59/99] lint --- .../miner/tests/test_receipts.py | 75 ++++++++++++------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 3b5c2f125..49b5ef6ac 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -1,32 +1,37 @@ from datetime import datetime -from datetime import datetime from uuid import uuid4 import bittensor import pytest from compute_horde.executor_class import ExecutorClass -from compute_horde.mv_protocol.validator_requests import V0AuthenticateRequest, \ - AuthenticationPayload, V0JobStartedReceiptRequest, JobStartedReceiptPayload, JobFinishedReceiptPayload, \ - V0JobFinishedReceiptRequest -from compute_horde.receipts.models import JobStartedReceipt, JobFinishedReceipt +from compute_horde.mv_protocol.validator_requests import ( + AuthenticationPayload, + JobFinishedReceiptPayload, + JobStartedReceiptPayload, + V0AuthenticateRequest, + V0JobFinishedReceiptRequest, + V0JobStartedReceiptRequest, +) +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from django.utils import timezone from pytest_mock import MockerFixture -from compute_horde_miner.miner.models import Validator, AcceptedJob +from compute_horde_miner.miner.models import AcceptedJob, Validator from compute_horde_miner.miner.tests.validator import fake_validator @pytest.mark.parametrize( - "organic_job", (True, False), + "organic_job", + (True, False), ) @pytest.mark.django_db(transaction=True) @pytest.mark.asyncio async def test_receipt_is_saved( - validator_wallet: bittensor.wallet, - miner_wallet: bittensor.wallet, - mocker: MockerFixture, - organic_job: bool, - settings, + validator_wallet: bittensor.wallet, + miner_wallet: bittensor.wallet, + mocker: MockerFixture, + organic_job: bool, + settings, ) -> None: mocker.patch("compute_horde_miner.miner.miner_consumer.validator_interface.prepare_receipts") settings.DEBUG_TURN_AUTHENTICATION_OFF = True @@ -45,11 +50,19 @@ async def test_receipt_is_saved( miner_hotkey=miner_wallet.hotkey.ss58_address, timestamp=int(datetime.now().timestamp()), ) - await fake_validator_channel.send_to(V0AuthenticateRequest( - payload=auth_payload, - signature=validator_wallet.hotkey.sign(auth_payload.blob_for_signing()).hex(), - ).model_dump_json()) - manifest = await fake_validator_channel.receive_json_from() + await fake_validator_channel.send_to( + V0AuthenticateRequest( + payload=auth_payload, + signature=validator_wallet.hotkey.sign(auth_payload.blob_for_signing()).hex(), + ).model_dump_json() + ) + response = await fake_validator_channel.receive_json_from() + assert response == { + "message_type": "V0ExecutorManifestRequest", + "manifest": { + "executor_classes": [{"executor_class": "spin_up-4min.gpu-24gb", "count": 1}], + }, + } # Skip doing the job await AcceptedJob.objects.acreate( @@ -68,11 +81,13 @@ async def test_receipt_is_saved( time_accepted=timezone.now(), max_timeout=123, ) - await fake_validator_channel.send_to(V0JobStartedReceiptRequest( - job_uuid=job_uuid, - payload=job_started_receipt_payload, - signature=f"0x{validator_wallet.hotkey.sign(job_started_receipt_payload.blob_for_signing()).hex()}", - ).model_dump_json()) + await fake_validator_channel.send_to( + V0JobStartedReceiptRequest( + job_uuid=job_uuid, + payload=job_started_receipt_payload, + signature=f"0x{validator_wallet.hotkey.sign(job_started_receipt_payload.blob_for_signing()).hex()}", + ).model_dump_json() + ) job_finished_receipt_payload = JobFinishedReceiptPayload( job_uuid=job_uuid, @@ -82,11 +97,15 @@ async def test_receipt_is_saved( time_took_us=123, score_str="123.45", ) - await fake_validator_channel.send_to(V0JobFinishedReceiptRequest( - job_uuid=job_uuid, - payload=job_finished_receipt_payload, - signature=f"0x{validator_wallet.hotkey.sign(job_finished_receipt_payload.blob_for_signing()).hex()}", - ).model_dump_json()) + await fake_validator_channel.send_to( + V0JobFinishedReceiptRequest( + job_uuid=job_uuid, + payload=job_finished_receipt_payload, + signature=f"0x{validator_wallet.hotkey.sign(job_finished_receipt_payload.blob_for_signing()).hex()}", + ).model_dump_json() + ) - assert await JobStartedReceipt.objects.filter(job_uuid=job_uuid, is_organic=organic_job).aexists() + assert await JobStartedReceipt.objects.filter( + job_uuid=job_uuid, is_organic=organic_job + ).aexists() assert await JobFinishedReceipt.objects.filter(job_uuid=job_uuid).aexists() From b3a9abf35060028d2b20ae9e31e0cf435aa64140 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 17:01:21 +0200 Subject: [PATCH 60/99] fix wallet name --- miner/app/src/compute_horde_miner/miner/tests/conftest.py | 2 +- miner/app/src/compute_horde_miner/miner/tests/test_receipts.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/conftest.py b/miner/app/src/compute_horde_miner/miner/tests/conftest.py index 90ad532c2..6e21e1b1a 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/conftest.py +++ b/miner/app/src/compute_horde_miner/miner/tests/conftest.py @@ -15,7 +15,7 @@ def validator_wallet(): @pytest.fixture(scope="function") def miner_wallet(settings): - wallet = bittensor.wallet(name="test_validator") + wallet = bittensor.wallet(name="test_miner") # workaround the overwrite flag wallet.regenerate_coldkey(seed="2" * 64, use_password=False, overwrite=True) wallet.regenerate_hotkey(seed="3" * 64, use_password=False, overwrite=True) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 49b5ef6ac..5d315d7d6 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -35,6 +35,7 @@ async def test_receipt_is_saved( ) -> None: mocker.patch("compute_horde_miner.miner.miner_consumer.validator_interface.prepare_receipts") settings.DEBUG_TURN_AUTHENTICATION_OFF = True + settings.BITTENSOR_WALLET = lambda: miner_wallet job_uuid = str(uuid4()) validator = await Validator.objects.acreate( public_key=validator_wallet.hotkey.ss58_address, From 371977303155892500380233e0049438a0196a90 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Tue, 22 Oct 2024 17:30:46 +0200 Subject: [PATCH 61/99] cleanup --- .../compute_horde_miner/miner/tests/test_receipts.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 5d315d7d6..6376b6fce 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -20,6 +20,10 @@ from compute_horde_miner.miner.tests.validator import fake_validator +def _sign(msg, key: bittensor.Keypair): + return f"0x{key.sign(msg.blob_for_signing()).hex()}" + + @pytest.mark.parametrize( "organic_job", (True, False), @@ -43,7 +47,7 @@ async def test_receipt_is_saved( ) # 1. Authenticate to miner as test validator - # 2. Send JobStarted and JobFinished receiptss + # 2. Send JobStarted and JobFinished receipts async with fake_validator(validator_wallet) as fake_validator_channel: # Authenticate, otherwise the consumer will refuse to talk to us auth_payload = AuthenticationPayload( @@ -54,7 +58,7 @@ async def test_receipt_is_saved( await fake_validator_channel.send_to( V0AuthenticateRequest( payload=auth_payload, - signature=validator_wallet.hotkey.sign(auth_payload.blob_for_signing()).hex(), + signature=_sign(auth_payload, validator_wallet.hotkey), ).model_dump_json() ) response = await fake_validator_channel.receive_json_from() @@ -86,7 +90,7 @@ async def test_receipt_is_saved( V0JobStartedReceiptRequest( job_uuid=job_uuid, payload=job_started_receipt_payload, - signature=f"0x{validator_wallet.hotkey.sign(job_started_receipt_payload.blob_for_signing()).hex()}", + signature=_sign(job_started_receipt_payload, validator_wallet.hotkey), ).model_dump_json() ) @@ -102,7 +106,7 @@ async def test_receipt_is_saved( V0JobFinishedReceiptRequest( job_uuid=job_uuid, payload=job_finished_receipt_payload, - signature=f"0x{validator_wallet.hotkey.sign(job_finished_receipt_payload.blob_for_signing()).hex()}", + signature=_sign(job_finished_receipt_payload, validator_wallet.hotkey), ).model_dump_json() ) From bf2e8977c1cb6d4c0acfa458015967fe666373a7 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 01:42:28 +0600 Subject: [PATCH 62/99] Fix surface level errors in validator --- .../validator/synthetic_jobs/batch_run.py | 54 ++++++++----------- .../validator/tasks.py | 25 +++++---- .../validator/tests/test_miner_driver.py | 6 +-- .../validator/tests/test_receipts.py | 11 ++-- 4 files changed, 45 insertions(+), 51 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index ecda12a22..450bd5c1a 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -37,15 +37,13 @@ ) from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, - JobFinishedReceiptPayload, - JobStartedReceiptPayload, V0AuthenticateRequest, V0InitialJobRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload from compute_horde.transport import AbstractTransport, WSTransport from compute_horde.transport.base import TransportConnectionError from django.conf import settings @@ -268,7 +266,8 @@ class Job: machine_specs: V0MachineSpecsRequest | None = None # receipts - job_started_receipt: V0JobStartedReceiptRequest | None = None + job_started_receipt_payload: JobStartedReceiptPayload | None = None + job_started_receipt_signature: str | None = None job_finished_receipt: V0JobFinishedReceiptRequest | None = None # scoring @@ -685,7 +684,8 @@ def _get_total_executor_count(ctx: BatchContext) -> int: def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: - assert job.job_started_receipt is None + assert job.job_started_receipt_payload is None + assert job.job_started_receipt_signature is None assert job.executor_response_time is not None @@ -694,14 +694,14 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: job_uuid=job.uuid, miner_hotkey=job.miner_hotkey, validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), executor_class=ExecutorClass(job.executor_class), - time_accepted=job.executor_response_time, max_timeout=max_timeout, + ttl=30, # FIXME: use spin up time ) - job.job_started_receipt = V0JobStartedReceiptRequest( - payload=payload, - signature=f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}", - ) + signature = f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}" + job.job_started_receipt_payload = payload + job.job_started_receipt_signature = signature def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: @@ -720,6 +720,7 @@ def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: job_uuid=job.uuid, miner_hotkey=job.miner_hotkey, validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), time_started=job.job_before_sent_time, time_took_us=int(time_took_sec * 1_000_000), score_str=f"{job.score:.6g}", @@ -877,6 +878,8 @@ async def _send_initial_job_request( job = ctx.jobs[job_uuid] job.accept_barrier_time = barrier_time client = ctx.clients[job.miner_hotkey] + assert job.job_started_receipt_payload is not None + assert job.job_started_receipt_signature is not None spin_up_time = EXECUTOR_CLASS[job.executor_class].spin_up_time assert spin_up_time is not None @@ -890,6 +893,8 @@ async def _send_initial_job_request( base_docker_image_name=job.job_generator.base_docker_image_name(), timeout_seconds=job.job_generator.timeout_seconds(), volume=job.volume if job.job_generator.volume_in_initial_req() else None, + job_started_receipt_payload=job.job_started_receipt_payload, + job_started_receipt_signature=job.job_started_receipt_signature, ) request_json = request.model_dump_json() @@ -907,23 +912,6 @@ async def _send_initial_job_request( if isinstance(job.accept_response, V0AcceptJobRequest): await job.executor_response_event.wait() - # send the receipt from outside the timeout - if isinstance(job.executor_response, V0ExecutorReadyRequest): - _generate_job_started_receipt(ctx, job) - assert job.job_started_receipt is not None - try: - receipt_json = job.job_started_receipt.model_dump_json() - async with asyncio.timeout(_SEND_RECEIPT_TIMEOUT): - await client.send_check(receipt_json) - except (Exception, asyncio.CancelledError) as exc: - logger.warning("%s failed to send job started receipt: %r", job.name, exc) - job.system_event( - type=SystemEvent.EventType.RECEIPT_FAILURE, - subtype=SystemEvent.EventSubType.RECEIPT_SEND_ERROR, - description=repr(exc), - func="_send_initial_job_request", - ) - async def _send_job_request( ctx: BatchContext, start_barrier: asyncio.Barrier, job_uuid: str @@ -1505,16 +1493,19 @@ def _db_persist(ctx: BatchContext) -> None: job_started_receipts: list[JobStartedReceipt] = [] for job in ctx.jobs.values(): - if job.job_started_receipt is not None: - started_payload = job.job_started_receipt.payload + if ( + job.job_started_receipt_payload is not None + and job.job_started_receipt_signature is not None + ): + started_payload = job.job_started_receipt_payload job_started_receipts.append( JobStartedReceipt( job_uuid=started_payload.job_uuid, miner_hotkey=started_payload.miner_hotkey, validator_hotkey=started_payload.validator_hotkey, - validator_signature=job.job_started_receipt.signature, + validator_signature=job.job_started_receipt_signature, + timestamp=started_payload.timestamp, executor_class=started_payload.executor_class, - time_accepted=started_payload.time_accepted, max_timeout=started_payload.max_timeout, ) ) @@ -1530,6 +1521,7 @@ def _db_persist(ctx: BatchContext) -> None: miner_hotkey=finished_payload.miner_hotkey, validator_hotkey=finished_payload.validator_hotkey, validator_signature=job.job_finished_receipt.signature, + timestamp=finished_payload.timestamp, time_started=finished_payload.time_started, time_took_us=finished_payload.time_took_us, score_str=finished_payload.score_str, diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index f19bdcd14..83529965d 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,11 +19,11 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.transfer import get_miner_receipts from compute_horde.utils import ValidatorListError, get_validators from constance import config @@ -1023,43 +1023,42 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): tolerance = timedelta(hours=1) latest_job_started_receipt = ( - JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_accepted").first() + JobStartedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_started_receipt_cutoff_time = ( - latest_job_started_receipt.time_accepted - tolerance if latest_job_started_receipt else None + latest_job_started_receipt.timestamp - tolerance if latest_job_started_receipt else None ) job_started_receipt_to_create = [ JobStartedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, - time_accepted=receipt.payload.time_accepted, max_timeout=receipt.payload.max_timeout, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) and ( job_started_receipt_cutoff_time is None - or receipt.payload.time_accepted > job_started_receipt_cutoff_time + or receipt.payload.timestamp > job_started_receipt_cutoff_time ) ] logger.debug(f"Creating {len(job_started_receipt_to_create)} JobStartedReceipt. {hotkey=}") JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) latest_job_finished_receipt = ( - JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-time_started").first() + JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) job_finished_receipt_cutoff_time = ( - latest_job_finished_receipt.time_started - tolerance - if latest_job_finished_receipt - else None + latest_job_finished_receipt.timestamp - tolerance if latest_job_finished_receipt else None ) job_finished_receipt_to_create = [ JobFinishedReceipt( job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, score_str=receipt.payload.score_str, @@ -1068,7 +1067,7 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): if isinstance(receipt.payload, JobFinishedReceiptPayload) and ( job_finished_receipt_cutoff_time is None - or receipt.payload.time_started > job_finished_receipt_cutoff_time + or receipt.payload.timestamp > job_finished_receipt_cutoff_time ) ] logger.debug(f"Creating {len(job_finished_receipt_to_create)} JobFinishedReceipt. {hotkey=}") @@ -1079,8 +1078,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): def fetch_receipts(): """Fetch job receipts from the miners.""" # Delete old receipts before fetching new ones - JobStartedReceipt.objects.filter(time_accepted__lt=now() - timedelta(days=7)).delete() - JobFinishedReceipt.objects.filter(time_started__lt=now() - timedelta(days=7)).delete() + JobStartedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() + JobFinishedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() metagraph = bittensor.metagraph( netuid=settings.BITTENSOR_NETUID, network=settings.BITTENSOR_NETWORK diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index 6120d4d95..fa17550a3 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -12,7 +12,6 @@ ) from compute_horde.mv_protocol.validator_requests import ( V0JobFinishedReceiptRequest, - V0JobStartedReceiptRequest, ) from compute_horde_validator.validator.models import Miner @@ -163,7 +162,8 @@ async def track_job_status_updates(x): def condition(_): return True - if expected_job_started_receipt: - assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) + # FIXME + # if expected_job_started_receipt: + # assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) if expected_job_finished_receipt: assert miner_client._query_sent_models(condition, V0JobFinishedReceiptRequest) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index aa32769f0..38ad8dcc3 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,14 +1,15 @@ import uuid +from datetime import datetime from typing import NamedTuple import pytest from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS -from compute_horde.mv_protocol.validator_requests import ( +from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( JobFinishedReceiptPayload, JobStartedReceiptPayload, + Receipt, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import Receipt from django.utils.timezone import now from compute_horde_validator.validator.models import ( @@ -46,9 +47,10 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5G9qWBzLPVVu2fCPPvg3QgPPK5JaJmJKaJha95TPHH9NZWuL", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, 0, 0), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=now(), max_timeout=30, + ttl=5, ), validator_signature="0xv1", miner_signature="0xm1", @@ -61,6 +63,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5CPhGRp4cdEG4KSui7VQixHhvN5eBUSnMYeUF5thdxm4sKtz", validator_hotkey="v1", + timestamp=datetime(2020, 1, 1, 1, 0), time_started=now(), time_took_us=30_000_000, score_str="123.45", From bb4dd47bd4e048fa327093bf5bc3a2003a6eb255 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 02:18:34 +0600 Subject: [PATCH 63/99] Add job started receipts in validator --- .../validator/synthetic_jobs/batch_run.py | 60 ++++++++++++++++++- .../validator/tasks.py | 28 ++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 450bd5c1a..fb58946b0 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -39,11 +39,16 @@ AuthenticationPayload, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt -from compute_horde.receipts.schemas import JobFinishedReceiptPayload, JobStartedReceiptPayload +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + JobStartedReceiptPayload, +) from compute_horde.transport import AbstractTransport, WSTransport from compute_horde.transport.base import TransportConnectionError from django.conf import settings @@ -268,6 +273,7 @@ class Job: # receipts job_started_receipt_payload: JobStartedReceiptPayload | None = None job_started_receipt_signature: str | None = None + job_accepted_receipt: V0JobAcceptedReceiptRequest | None = None job_finished_receipt: V0JobFinishedReceiptRequest | None = None # scoring @@ -704,6 +710,24 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: job.job_started_receipt_signature = signature +def _generate_job_accepted_receipt(ctx: BatchContext, job: Job) -> None: + assert job.job_accepted_receipt is None + assert job.accept_response_time is not None + + payload = JobAcceptedReceiptPayload( + job_uuid=job.uuid, + miner_hotkey=job.miner_hotkey, + validator_hotkey=ctx.own_keypair.ss58_address, + timestamp=datetime.now(tz=UTC), + time_accepted=job.accept_response_time, + ttl=300, # FIXME: max time allowed to run the job + ) + job.job_accepted_receipt = V0JobAcceptedReceiptRequest( + payload=payload, + signature=f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}", + ) + + def _generate_job_finished_receipt(ctx: BatchContext, job: Job) -> None: assert job.job_finished_receipt is None assert job.job_before_sent_time is not None @@ -910,6 +934,21 @@ async def _send_initial_job_request( await job.accept_response_event.wait() if isinstance(job.accept_response, V0AcceptJobRequest): + _generate_job_accepted_receipt(ctx, job) + assert job.job_accepted_receipt is not None + try: + receipt_json = job.job_accepted_receipt.model_dump_json() + async with asyncio.timeout(_SEND_RECEIPT_TIMEOUT): + await client.send_check(receipt_json) + except (Exception, asyncio.CancelledError) as exc: + logger.warning("%s failed to send job accepted receipt: %r", job.name, exc) + job.system_event( + type=SystemEvent.EventType.RECEIPT_FAILURE, + subtype=SystemEvent.EventSubType.RECEIPT_SEND_ERROR, + description=repr(exc), + func="_send_initial_job_request", + ) + await job.executor_response_event.wait() @@ -1511,6 +1550,23 @@ def _db_persist(ctx: BatchContext) -> None: ) JobStartedReceipt.objects.bulk_create(job_started_receipts) + job_accepted_receipts: list[JobAcceptedReceipt] = [] + for job in ctx.jobs.values(): + if job.job_accepted_receipt is not None: + accepted_payload = job.job_accepted_receipt.payload + job_accepted_receipts.append( + JobAcceptedReceipt( + job_uuid=accepted_payload.job_uuid, + miner_hotkey=accepted_payload.miner_hotkey, + validator_hotkey=accepted_payload.validator_hotkey, + validator_signature=job.job_accepted_receipt.signature, + timestamp=accepted_payload.timestamp, + time_accepted=accepted_payload.time_accepted, + ttl=accepted_payload.ttl, + ) + ) + JobAcceptedReceipt.objects.bulk_create(job_accepted_receipts) + job_finished_receipts: list[JobFinishedReceipt] = [] for job in ctx.jobs.values(): if job.job_finished_receipt is not None: diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index 83529965d..e5090d703 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -19,8 +19,9 @@ from celery.result import allow_join_result from celery.utils.log import get_task_logger from compute_horde.dynamic_config import sync_dynamic_config -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, ) @@ -1047,6 +1048,30 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): logger.debug(f"Creating {len(job_started_receipt_to_create)} JobStartedReceipt. {hotkey=}") JobStartedReceipt.objects.bulk_create(job_started_receipt_to_create, ignore_conflicts=True) + latest_job_accepted_receipt = ( + JobAcceptedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() + ) + job_accepted_receipt_cutoff_time = ( + latest_job_accepted_receipt.timestamp - tolerance if latest_job_accepted_receipt else None + ) + job_accepted_receipt_to_create = [ + JobAcceptedReceipt( + job_uuid=receipt.payload.job_uuid, + miner_hotkey=receipt.payload.miner_hotkey, + validator_hotkey=receipt.payload.validator_hotkey, + timestamp=receipt.payload.timestamp, + ttl=receipt.payload.ttl, + ) + for receipt in receipts + if isinstance(receipt.payload, JobAcceptedReceiptPayload) + and ( + job_accepted_receipt_cutoff_time is None + or receipt.payload.timestamp > job_accepted_receipt_cutoff_time + ) + ] + logger.debug(f"Creating {len(job_accepted_receipt_to_create)} JobAcceptedReceipt. {hotkey=}") + JobAcceptedReceipt.objects.bulk_create(job_accepted_receipt_to_create, ignore_conflicts=True) + latest_job_finished_receipt = ( JobFinishedReceipt.objects.filter(miner_hotkey=hotkey).order_by("-timestamp").first() ) @@ -1079,6 +1104,7 @@ def fetch_receipts(): """Fetch job receipts from the miners.""" # Delete old receipts before fetching new ones JobStartedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() + JobAcceptedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() JobFinishedReceipt.objects.filter(timestamp__lt=now() - timedelta(days=7)).delete() metagraph = bittensor.metagraph( From 8d2b3139ae62490610de49e5c1c175a83e7627ab Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 02:55:15 +0600 Subject: [PATCH 64/99] Fix stupid mistakes! -_- --- .../validator/synthetic_jobs/batch_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index fb58946b0..362010963 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -693,8 +693,6 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: assert job.job_started_receipt_payload is None assert job.job_started_receipt_signature is None - assert job.executor_response_time is not None - max_timeout = job.job_generator.timeout_seconds() payload = JobStartedReceiptPayload( job_uuid=job.uuid, @@ -902,6 +900,8 @@ async def _send_initial_job_request( job = ctx.jobs[job_uuid] job.accept_barrier_time = barrier_time client = ctx.clients[job.miner_hotkey] + + _generate_job_started_receipt(ctx, job) assert job.job_started_receipt_payload is not None assert job.job_started_receipt_signature is not None @@ -1546,6 +1546,7 @@ def _db_persist(ctx: BatchContext) -> None: timestamp=started_payload.timestamp, executor_class=started_payload.executor_class, max_timeout=started_payload.max_timeout, + ttl=started_payload.ttl, ) ) JobStartedReceipt.objects.bulk_create(job_started_receipts) From dd0611986cfe5f29786343fb7779ba5f06186c82 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:19:49 +0600 Subject: [PATCH 65/99] Fix lib tests --- .../compute_horde/receipts/models.py | 6 +-- compute_horde/tests/conftest.py | 29 ++++++++++++--- compute_horde/tests/test_receipts.py | 37 +++++++++++-------- compute_horde/tests/test_run_organic_job.py | 4 +- .../validator/tests/test_receipts.py | 6 +-- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 9298f408b..1353a9885 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -23,9 +23,6 @@ class AbstractReceipt(models.Model): miner_signature = models.CharField(max_length=256, null=True, blank=True) timestamp = models.DateTimeField() - # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 - objects: models.Manager["JobStartedReceipt"] - class Meta: abstract = True constraints = [ @@ -67,6 +64,9 @@ class JobAcceptedReceipt(AbstractReceipt): time_accepted = models.DateTimeField() ttl = models.IntegerField() + # https://github.com/typeddjango/django-stubs/issues/1684#issuecomment-1706446344 + objects: models.Manager["JobAcceptedReceipt"] + def to_receipt(self) -> Receipt: if self.miner_signature is None: raise ReceiptNotSigned("Miner signature is required") diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index cc4fd69b7..02946164a 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -6,6 +6,7 @@ from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, @@ -39,9 +40,10 @@ def receipts(validator_keypair, miner_keypair): job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", miner_hotkey=miner_keypair.ss58_address, validator_hotkey=validator_keypair.ss58_address, + timestamp=datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=datetime.UTC), executor_class=DEFAULT_EXECUTOR_CLASS, - time_accepted=datetime.datetime(2024, 1, 2, 1, 55, 0, tzinfo=datetime.UTC), max_timeout=30, + ttl=30, ) receipt1 = Receipt( payload=payload1, @@ -49,13 +51,13 @@ def receipts(validator_keypair, miner_keypair): miner_signature=f"0x{miner_keypair.sign(payload1.blob_for_signing()).hex()}", ) - payload2 = JobFinishedReceiptPayload( + payload2 = JobAcceptedReceiptPayload( job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", miner_hotkey=miner_keypair.ss58_address, validator_hotkey=validator_keypair.ss58_address, - time_started=datetime.datetime(2024, 1, 2, 1, 57, 0, tzinfo=datetime.UTC), - time_took_us=2_000_000, - score_str="2.00", + timestamp=datetime.datetime(2020, 1, 1, 0, 5, 0, tzinfo=datetime.UTC), + time_accepted=datetime.datetime(2020, 1, 1, 0, 4, 0, tzinfo=datetime.UTC), + ttl=300, ) receipt2 = Receipt( payload=payload2, @@ -63,7 +65,22 @@ def receipts(validator_keypair, miner_keypair): miner_signature=f"0x{miner_keypair.sign(payload2.blob_for_signing()).hex()}", ) - return [receipt1, receipt2] + payload3 = JobFinishedReceiptPayload( + job_uuid="3342460e-4a99-438b-8757-795f4cb348dd", + miner_hotkey=miner_keypair.ss58_address, + validator_hotkey=validator_keypair.ss58_address, + timestamp=datetime.datetime(2020, 1, 1, 0, 10, 0, tzinfo=datetime.UTC), + time_started=datetime.datetime(2020, 1, 1, 0, 9, 0, tzinfo=datetime.UTC), + time_took_us=60_000_000, + score_str="2.00", + ) + receipt3 = Receipt( + payload=payload3, + validator_signature=f"0x{validator_keypair.sign(payload3.blob_for_signing()).hex()}", + miner_signature=f"0x{miner_keypair.sign(payload3.blob_for_signing()).hex()}", + ) + + return [receipt1, receipt2, receipt3] @pytest.fixture diff --git a/compute_horde/tests/test_receipts.py b/compute_horde/tests/test_receipts.py index baa8619b5..a4877a6bd 100644 --- a/compute_horde/tests/test_receipts.py +++ b/compute_horde/tests/test_receipts.py @@ -4,17 +4,21 @@ import pytest from compute_horde.receipts.schemas import ( + JobAcceptedReceiptPayload, JobFinishedReceiptPayload, JobStartedReceiptPayload, Receipt, - ReceiptType, ) from compute_horde.receipts.transfer import ReceiptFetchError, get_miner_receipts def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): payload_fields = set() - for payload_cls in [JobStartedReceiptPayload, JobFinishedReceiptPayload]: + for payload_cls in [ + JobStartedReceiptPayload, + JobAcceptedReceiptPayload, + JobFinishedReceiptPayload, + ]: payload_fields |= set(payload_cls.model_fields.keys()) buf = io.StringIO() @@ -29,11 +33,7 @@ def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): ) csv_writer.writeheader() for receipt in receipts: - match receipt.payload: - case JobStartedReceiptPayload(): - receipt_type = ReceiptType.JobStartedReceipt - case JobFinishedReceiptPayload(): - receipt_type = ReceiptType.JobFinishedReceipt + receipt_type = receipt.payload.receipt_type row = ( dict( type=receipt_type.value, @@ -51,13 +51,20 @@ def receipts_helper(mocked_responses, receipts: list[Receipt], miner_keypair): def receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair): got_receipts = receipts_helper(mocked_responses, receipts, miner_keypair) # only the valid receipt should be stored - assert len(got_receipts) == 1 - assert got_receipts[0] == receipts[0] + assert len(got_receipts) == len(receipts) - 1 + for receipt in receipts[1:]: + got_receipt = [ + x + for x in got_receipts + if x.payload.job_uuid == receipt.payload.job_uuid + and x.payload.__class__ is receipt.payload.__class__ + ][0] + assert got_receipt == receipt def test__get_miner_receipts__happy_path(mocked_responses, receipts, miner_keypair): got_receipts = receipts_helper(mocked_responses, receipts, miner_keypair) - assert len(got_receipts) == 2 + assert len(got_receipts) == len(receipts) for receipt in receipts: got_receipt = [ x @@ -74,29 +81,29 @@ def test__get_miner_receipts__invalid_receipt_skipped(mocked_responses, receipts Invalidate one receipt payload fields to make it invalid """ - receipts[1].payload.miner_hotkey = 0 - receipts[1].payload.validator_hotkey = None + receipts[0].payload.miner_hotkey = 0 + receipts[0].payload.validator_hotkey = None receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__miner_hotkey_mismatch_skipped( mocked_responses, receipts, miner_keypair, keypair ): - receipts[1].payload.miner_hotkey = keypair.ss58_address + receipts[0].payload.miner_hotkey = keypair.ss58_address receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__invalid_miner_signature_skipped( mocked_responses, receipts, miner_keypair ): - receipts[1].miner_signature = f"0x{miner_keypair.sign('bla').hex()}" + receipts[0].miner_signature = f"0x{miner_keypair.sign('bla').hex()}" receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) def test__get_miner_receipts__invalid_validator_signature_skipped( mocked_responses, receipts, miner_keypair ): - receipts[1].validator_signature = f"0x{miner_keypair.sign('bla').hex()}" + receipts[0].validator_signature = f"0x{miner_keypair.sign('bla').hex()}" receipts_one_skipped_helper(mocked_responses, receipts, miner_keypair) diff --git a/compute_horde/tests/test_run_organic_job.py b/compute_horde/tests/test_run_organic_job.py index ad43f8555..f28972590 100644 --- a/compute_horde/tests/test_run_organic_job.py +++ b/compute_horde/tests/test_run_organic_job.py @@ -14,9 +14,9 @@ BaseValidatorRequest, V0AuthenticateRequest, V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, V0JobRequest, - V0JobStartedReceiptRequest, ) from compute_horde.transport import StubTransport @@ -65,7 +65,7 @@ async def test_run_organic_job__success(keypair): assert sent_models_types == [ V0AuthenticateRequest, V0InitialJobRequest, - V0JobStartedReceiptRequest, + V0JobAcceptedReceiptRequest, V0JobRequest, V0JobFinishedReceiptRequest, ] diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index 38ad8dcc3..ff9179f82 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime +from datetime import datetime, UTC from typing import NamedTuple import pytest @@ -47,7 +47,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5G9qWBzLPVVu2fCPPvg3QgPPK5JaJmJKaJha95TPHH9NZWuL", validator_hotkey="v1", - timestamp=datetime(2020, 1, 1, 0, 0), + timestamp=datetime(2020, 1, 1, 0, 0, tzinfo=UTC), executor_class=DEFAULT_EXECUTOR_CLASS, max_timeout=30, ttl=5, @@ -63,7 +63,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: job_uuid=str(uuid.uuid4()), miner_hotkey="5CPhGRp4cdEG4KSui7VQixHhvN5eBUSnMYeUF5thdxm4sKtz", validator_hotkey="v1", - timestamp=datetime(2020, 1, 1, 1, 0), + timestamp=datetime(2020, 1, 1, 1, 0, tzinfo=UTC), time_started=now(), time_took_us=30_000_000, score_str="123.45", From 2b8e95d34319e7625e4693c1db1b45a8fb717476 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:20:20 +0600 Subject: [PATCH 66/99] Fix when accepted receipt is sent in organic job --- compute_horde/compute_horde/miner_client/organic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compute_horde/compute_horde/miner_client/organic.py b/compute_horde/compute_horde/miner_client/organic.py index 32fd456ae..561ce76e5 100644 --- a/compute_horde/compute_horde/miner_client/organic.py +++ b/compute_horde/compute_horde/miner_client/organic.py @@ -427,6 +427,11 @@ async def run_organic_job( await client.notify_job_accepted(initial_response) + await client.send_job_accepted_receipt_message( + accepted_timestamp=time.time(), + ttl=int(job_timer.time_left()), + ) + try: executor_readiness_response = await asyncio.wait_for( client.executor_ready_or_failed_future, @@ -439,11 +444,6 @@ async def run_organic_job( await client.notify_executor_ready(executor_readiness_response) - await client.send_job_accepted_receipt_message( - accepted_timestamp=time.time(), - ttl=int(job_timer.time_left()), - ) - await client.send_model( V0JobRequest( job_uuid=job_details.job_uuid, From f343613a2a44c119930cbc47fa175b31d3a60c3a Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:40:49 +0600 Subject: [PATCH 67/99] Fix receipts missing fields in validator db --- .../app/src/compute_horde_validator/validator/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index e5090d703..2c76795bc 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1034,9 +1034,12 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, max_timeout=receipt.payload.max_timeout, + ttl=receipt.payload.ttl, ) for receipt in receipts if isinstance(receipt.payload, JobStartedReceiptPayload) @@ -1059,6 +1062,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, ttl=receipt.payload.ttl, ) @@ -1083,6 +1088,8 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + miner_signature=receipt.miner_signature, + validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, From 234870de0790535f6a3007426615947aa98b9f5e Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 03:46:17 +0600 Subject: [PATCH 68/99] rufFFFFFF --- .../compute_horde_validator/validator/tests/test_receipts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index ff9179f82..4f816ad55 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, UTC +from datetime import UTC, datetime from typing import NamedTuple import pytest From 55eb66c20f1dddac76bb0a6bdba8e5328ccea895 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 15:12:36 +0600 Subject: [PATCH 69/99] Ignore `AbstractReceipt.objects` type hints --- miner/app/src/compute_horde_miner/miner/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index cc7edf44f..3b0544499 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -72,7 +72,7 @@ def prepare_receipts(): receipts = [] for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: - db_objects = model.objects.order_by("timestamp").filter( + db_objects = model.objects.order_by("timestamp").filter( # type: ignore[attr-defined] timestamp__gt=now() - RECEIPTS_MAX_SERVED_PERIOD ) for db_object in db_objects: @@ -89,7 +89,7 @@ def prepare_receipts(): @app.task def clear_old_receipts(): for model in [JobStartedReceipt, JobAcceptedReceipt, JobFinishedReceipt]: - model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() + model.objects.filter(timestamp__lt=now() - RECEIPTS_MAX_RETENTION_PERIOD).delete() # type: ignore[attr-defined] @app.task From 153987231e2526824325776b2756fd254a8aea47 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 16:04:29 +0600 Subject: [PATCH 70/99] Fix remaining miner test --- .../test_mocked_executor_manager.py | 88 +++++++------------ 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index 460e39ad2..311da15ab 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -1,12 +1,10 @@ import contextlib -import json import time import uuid -from unittest.mock import Mock +from unittest.mock import MagicMock import pytest import pytest_asyncio -from bittensor import Keypair from channels.testing import WebsocketCommunicator from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from pytest_mock import MockerFixture @@ -22,11 +20,9 @@ @pytest.fixture def mock_keypair(mocker: MockerFixture): - mock = Mock(wraps=Keypair) - mocker.patch( - "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair", mock + return mocker.patch( + "compute_horde_miner.miner.miner_consumer.validator_interface.bittensor.Keypair" ) - return mock # Somehow the regular dependency mechanism doesn't work with multiple test cases @@ -48,15 +44,8 @@ def job_uuid(): @pytest.fixture -def validator_keypair(): - return Keypair.create_from_mnemonic( - "slot excuse valid grief praise rifle spoil auction weasel glove pen share" - ) - - -@pytest.fixture -def validator_hotkey(validator_keypair): - return validator_keypair.ss58_address +def validator_hotkey(): + return "some_public_key" @pytest_asyncio.fixture @@ -75,20 +64,17 @@ async def make_communicator(validator_key: str): await communicator.disconnect() -async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, job_uuid: str): - async with make_communicator(validator_keypair.ss58_address) as communicator: - auth_payload = { - "validator_hotkey": validator_keypair.ss58_address, - "miner_hotkey": "some key", - "timestamp": int(time.time()), - } - auth_payload_blob = json.dumps(auth_payload, sort_keys=True) - auth_signature = f"0x{validator_keypair.sign(auth_payload_blob).hex()}" +async def run_regular_flow_test(validator_key: str, job_uuid: str): + async with make_communicator(validator_key) as communicator: await communicator.send_json_to( { "message_type": "V0AuthenticateRequest", - "payload": auth_payload, - "signature": auth_signature, + "payload": { + "validator_hotkey": validator_key, + "miner_hotkey": "some key", + "timestamp": int(time.time()), + }, + "signature": "gibberish", } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -98,18 +84,6 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j "executor_classes": [{"count": 1, "executor_class": DEFAULT_EXECUTOR_CLASS}] }, } - receipt = { - "receipt_type": "JobStartedReceipt", - "job_uuid": job_uuid, - "miner_hotkey": miner_hotkey, - "validator_hotkey": validator_keypair.ss58_address, - "timestamp": "2020-01-01T00:00Z", - "executor_class": DEFAULT_EXECUTOR_CLASS, - "max_timeout": 60, - "ttl": 5, - } - receipt_blob = json.dumps(receipt, sort_keys=True) - receipt_signature = f"0x{validator_keypair.sign(receipt_blob).hex()}" await communicator.send_json_to( { "message_type": "V0InitialJobRequest", @@ -118,8 +92,17 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j "base_docker_image_name": "it's teeeeests", "timeout_seconds": 60, "volume_type": "inline", - "job_started_receipt_payload": receipt, - "job_started_receipt_signature": receipt_signature, + "job_started_receipt_payload": { + "receipt_type": "JobStartedReceipt", + "job_uuid": job_uuid, + "miner_hotkey": "miner_hotkey", + "validator_hotkey": validator_key, + "timestamp": "2020-01-01T00:00Z", + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 5, + }, + "job_started_receipt_signature": "gibberish", } ) response = await communicator.receive_json_from(timeout=WEBSOCKET_TIMEOUT) @@ -153,32 +136,21 @@ async def run_regular_flow_test(validator_keypair: Keypair, miner_hotkey: str, j } -async def test_main_loop(validator: Validator, validator_keypair: Keypair, job_uuid: str, settings): - await run_regular_flow_test( - validator_keypair, - settings.BITTENSOR_WALLET().hotkey.ss58_address, - job_uuid, - ) +async def test_main_loop(validator: Validator, job_uuid: str, mock_keypair: MagicMock): + await run_regular_flow_test(validator.public_key, job_uuid) -async def test_local_miner( - validator: Validator, - validator_keypair: Keypair, - job_uuid: str, - mock_keypair: Mock, - settings, -): +async def test_local_miner(validator: Validator, job_uuid: str, mock_keypair: MagicMock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False - await run_regular_flow_test( - validator_keypair, settings.BITTENSOR_WALLET().hotkey.ss58_address, job_uuid - ) + await run_regular_flow_test(validator.public_key, job_uuid) mock_keypair.assert_called_once_with(ss58_address=validator.public_key) + mock_keypair.return_value.verify.assert_called_once() -async def test_local_miner_unknown_validator(mock_keypair: Mock, settings): +async def test_local_miner_unknown_validator(mock_keypair: MagicMock, settings): settings.IS_LOCAL_MINER = True settings.DEBUG_TURN_AUTHENTICATION_OFF = False From 208013cb0407393ee424ecbb647e3c820dc9efe9 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 16:43:11 +0600 Subject: [PATCH 71/99] Add fixture to create miner test wallet --- .../miner/tests/conftest.py | 15 +++++++++++++++ .../miner/tests/settings.py | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/miner/app/src/compute_horde_miner/miner/tests/conftest.py b/miner/app/src/compute_horde_miner/miner/tests/conftest.py index 7149037f2..63a8facca 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/conftest.py +++ b/miner/app/src/compute_horde_miner/miner/tests/conftest.py @@ -1,10 +1,25 @@ +import logging from collections.abc import Generator +import bittensor import pytest +logger = logging.getLogger(__name__) + @pytest.fixture def some() -> Generator[int, None, None]: # setup code yield 1 # teardown code + + +@pytest.fixture(scope="session", autouse=True) +def wallet(): + wallet = bittensor.wallet(name="test_miner") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") diff --git a/miner/app/src/compute_horde_miner/miner/tests/settings.py b/miner/app/src/compute_horde_miner/miner/tests/settings.py index c3f29f844..e52a0333e 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/settings.py +++ b/miner/app/src/compute_horde_miner/miner/tests/settings.py @@ -1,4 +1,7 @@ import os +import pathlib + +import bittensor os.environ.update( { @@ -13,3 +16,19 @@ EXECUTOR_MANAGER_CLASS_PATH = "compute_horde_miner.miner.tests.executor_manager:StubExecutorManager" DEBUG_TURN_AUTHENTICATION_OFF = True + +BITTENSOR_WALLET_DIRECTORY = pathlib.Path("~").expanduser() / ".bittensor" / "wallets" +BITTENSOR_WALLET_NAME = "test_miner" +BITTENSOR_WALLET_HOTKEY_NAME = "default" + + +def BITTENSOR_WALLET() -> bittensor.wallet: # type: ignore + if not BITTENSOR_WALLET_NAME or not BITTENSOR_WALLET_HOTKEY_NAME: + raise RuntimeError("Wallet not configured") + wallet = bittensor.wallet( + name=BITTENSOR_WALLET_NAME, + hotkey=BITTENSOR_WALLET_HOTKEY_NAME, + path=str(BITTENSOR_WALLET_DIRECTORY), + ) + wallet.hotkey_file.get_keypair() # this raises errors if the keys are inaccessible + return wallet From 16d4f16a776ef00d2f39a1a2fa61ffe27f56f117 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:27:55 +0600 Subject: [PATCH 72/99] Fix integration test --- .../test_miner_on_dev_executor_manager.py | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index 591970e7d..ec8c1d852 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -1,3 +1,4 @@ +from datetime import datetime, UTC import asyncio import base64 import io @@ -11,6 +12,8 @@ import uuid import zipfile from unittest import mock +import bittensor +import logging import pytest import requests @@ -21,7 +24,29 @@ MINER_PORT = 8045 WEBSOCKET_TIMEOUT = 10 -validator_key = str(uuid.uuid4()) +logger = logging.getLogger(__name__) + + +def get_miner_wallet(): + wallet = bittensor.wallet(name="test_miner") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") + return wallet + + +def get_validator_wallet(): + wallet = bittensor.wallet(name="test_validator") + try: + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + except Exception as e: + logger.error(f"Failed to create wallet: {e}") + return wallet class Test(ActiveSubnetworkBaseTest): @@ -43,6 +68,7 @@ def miner_path_and_args(cls) -> list[str]: @classmethod def miner_preparation_tasks(cls): + validator_key = get_validator_wallet().get_hotkey().ss58_address db_shell_cmd = f"{sys.executable} miner/app/src/manage.py dbshell" for cmd in [ f'echo "DROP DATABASE IF EXISTS compute_horde_miner_integration_test" | {db_shell_cmd}', @@ -69,6 +95,7 @@ def miner_environ(cls) -> dict[str, str]: "PORT_FOR_EXECUTORS": str(MINER_PORT), "DATABASE_SUFFIX": "_integration_test", "DEBUG_TURN_AUTHENTICATION_OFF": "1", + "BITTENSOR_WALLET_NAME": "test_miner", } @classmethod @@ -77,11 +104,16 @@ def validator_path_and_args(cls) -> list[str]: @classmethod def validator_environ(cls) -> dict[str, str]: - return {} + return { + "BITTENSOR_WALLET_NAME": "test_validator", + } @pytest.mark.asyncio async def test_echo_image(self): job_uuid = str(uuid.uuid4()) + miner_key = get_miner_wallet().get_hotkey().ss58_address + validator_wallet = get_validator_wallet() + validator_key = validator_wallet.get_hotkey().ss58_address payload = "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(32) @@ -121,6 +153,18 @@ async def test_echo_image(self): ] }, } + + receipt_payload = { + "job_uuid": job_uuid, + "miner_hotkey": miner_key, + "validator_hotkey": validator_key, + "timestamp": datetime.now(tz=UTC).isoformat(), + "executor_class": DEFAULT_EXECUTOR_CLASS, + "max_timeout": 60, + "ttl": 30, + } + blob = json.dumps(receipt_payload, sort_keys=True) + signature = "0x" + validator_wallet.get_hotkey().sign(blob).hex() await ws.send( json.dumps( { @@ -130,6 +174,8 @@ async def test_echo_image(self): "base_docker_image_name": "backenddevelopersltd/compute-horde-job-echo:v0-latest", "timeout_seconds": 60, "volume_type": "inline", + "job_started_receipt_payload": receipt_payload, + "job_started_receipt_signature": signature, } ) ) From aa75e6c1fd7eb01fb20b8dd68362f1e21193a36d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:51:45 +0600 Subject: [PATCH 73/99] Fix job started receipt ttl --- .../validator/synthetic_jobs/batch_run.py | 15 +++++++++------ .../validator/tests/test_miner_driver.py | 12 ++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index 362010963..498d8f6ac 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -391,6 +391,12 @@ def emit_telemetry_event(self) -> SystemEvent | None: data=data, ) + def get_spin_up_time(self) -> int: + spin_up_time = EXECUTOR_CLASS[self.executor_class].spin_up_time + assert spin_up_time is not None + spin_up_time = max(spin_up_time, _MIN_SPIN_UP_TIME) + return spin_up_time + @dataclass class BatchContext: @@ -701,7 +707,7 @@ def _generate_job_started_receipt(ctx: BatchContext, job: Job) -> None: timestamp=datetime.now(tz=UTC), executor_class=ExecutorClass(job.executor_class), max_timeout=max_timeout, - ttl=30, # FIXME: use spin up time + ttl=job.get_spin_up_time(), ) signature = f"0x{ctx.own_keypair.sign(payload.blob_for_signing()).hex()}" job.job_started_receipt_payload = payload @@ -718,7 +724,7 @@ def _generate_job_accepted_receipt(ctx: BatchContext, job: Job) -> None: validator_hotkey=ctx.own_keypair.ss58_address, timestamp=datetime.now(tz=UTC), time_accepted=job.accept_response_time, - ttl=300, # FIXME: max time allowed to run the job + ttl=6 * 60, # FIXME: max time allowed to run the job ) job.job_accepted_receipt = V0JobAcceptedReceiptRequest( payload=payload, @@ -905,10 +911,7 @@ async def _send_initial_job_request( assert job.job_started_receipt_payload is not None assert job.job_started_receipt_signature is not None - spin_up_time = EXECUTOR_CLASS[job.executor_class].spin_up_time - assert spin_up_time is not None - spin_up_time = max(spin_up_time, _MIN_SPIN_UP_TIME) - stagger_wait_interval = max_spin_up_time - spin_up_time + stagger_wait_interval = max_spin_up_time - job.get_spin_up_time() assert stagger_wait_interval >= 0 request = V0InitialJobRequest( diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py index fa17550a3..f92c7cb47 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_miner_driver.py @@ -11,6 +11,7 @@ V0JobFinishedRequest, ) from compute_horde.mv_protocol.validator_requests import ( + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, ) @@ -36,7 +37,7 @@ "expected_job_status_updates", "organic_job_status", "dummy_job_factory", - "expected_job_started_receipt", + "expected_job_accepted_receipt", "expected_job_finished_receipt", ), [ @@ -61,7 +62,7 @@ ["accepted", "failed"], OrganicJob.Status.FAILED, get_dummy_job_request_v0, - False, + True, False, ), ( @@ -103,7 +104,7 @@ async def test_miner_driver( expected_job_status_updates, organic_job_status, dummy_job_factory, - expected_job_started_receipt, + expected_job_accepted_receipt, expected_job_finished_receipt, ): miner, _ = await Miner.objects.aget_or_create(hotkey="miner_client") @@ -162,8 +163,7 @@ async def track_job_status_updates(x): def condition(_): return True - # FIXME - # if expected_job_started_receipt: - # assert miner_client._query_sent_models(condition, V0JobStartedReceiptRequest) + if expected_job_accepted_receipt: + assert miner_client._query_sent_models(condition, V0JobAcceptedReceiptRequest) if expected_job_finished_receipt: assert miner_client._query_sent_models(condition, V0JobFinishedReceiptRequest) From 697a7f66a8d210e08dddfe0c6b310af4b8f5e92f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 17:56:56 +0600 Subject: [PATCH 74/99] Removed unused code --- compute_horde/compute_horde/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 1be8f2a8d..84e00fa9e 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -3,7 +3,6 @@ import bittensor import pydantic -from pydantic import BeforeValidator from substrateinterface.exceptions import SubstrateRequestException if TYPE_CHECKING: @@ -65,16 +64,3 @@ def time_left(self): if self.timeout is None: raise ValueError("timeout was not specified") return self.timeout - self.passed_time() - - -def _empty_string_none(value: Any) -> Any: - """ - Converts value to None if it is empty-string, otherwise returns the same value. - Intended to be used with pydantic validators. - """ - if value == "": - return None - return value - - -empty_string_none = BeforeValidator(_empty_string_none) From 632825f5b59001a8b01f85c27acf1783f7bebe16 Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Wed, 23 Oct 2024 14:31:54 +0200 Subject: [PATCH 75/99] default value on is_organic is not needed --- compute_horde/compute_horde/mv_protocol/validator_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index d811c91a6..129da6568 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -133,7 +133,7 @@ class JobStartedReceiptPayload(ReceiptPayload): executor_class: ExecutorClass time_accepted: datetime.datetime max_timeout: int # seconds - is_organic: bool = False + is_organic: bool @field_serializer("time_accepted") def serialize_dt(self, dt: datetime.datetime, _info): From c26c042abe8be96780735b8fe847800903b188ab Mon Sep 17 00:00:00 2001 From: Kordian Kowalski Date: Wed, 23 Oct 2024 14:41:18 +0200 Subject: [PATCH 76/99] fix tests --- compute_horde/tests/conftest.py | 1 + miner/app/src/compute_horde_miner/miner/tests/test_migration.py | 1 + .../src/compute_horde_validator/validator/tests/test_receipts.py | 1 + 3 files changed, 3 insertions(+) diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index bde902a63..844d5aaf0 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -42,6 +42,7 @@ def receipts(validator_keypair, miner_keypair): executor_class=DEFAULT_EXECUTOR_CLASS, time_accepted=datetime.datetime(2024, 1, 2, 1, 55, 0, tzinfo=datetime.UTC), max_timeout=30, + is_organic=False, ) receipt1 = Receipt( payload=payload1, diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py index 556195e33..b478fb433 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_migration.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_migration.py @@ -50,6 +50,7 @@ def test_get_receipts_from_old_miner(mocker: MockerFixture): executor_class=DEFAULT_EXECUTOR_CLASS, time_accepted=now(), max_timeout=30, + is_organic=False, ), validator_signature="0xv1", miner_signature="0xm1", diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py index aa32769f0..c1d6bb53d 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_receipts.py @@ -49,6 +49,7 @@ def mocked_get_miner_receipts(hotkey: str, ip: str, port: int) -> list[Receipt]: executor_class=DEFAULT_EXECUTOR_CLASS, time_accepted=now(), max_timeout=30, + is_organic=False, ), validator_signature="0xv1", miner_signature="0xm1", From a30f3196eeadc0339b60434e68b6d9a87c44a38d Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Wed, 23 Oct 2024 20:12:08 +0600 Subject: [PATCH 77/99] Fix migration order of lib --- ...> 0003_remove_jobstartedreceipt_time_accepted_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename compute_horde/compute_horde/receipts/migrations/{0002_remove_jobstartedreceipt_time_accepted_and_more.py => 0003_remove_jobstartedreceipt_time_accepted_and_more.py} (95%) diff --git a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py similarity index 95% rename from compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py rename to compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py index 232c72793..903d6ebbf 100644 --- a/compute_horde/compute_horde/receipts/migrations/0002_remove_jobstartedreceipt_time_accepted_and_more.py +++ b/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-18 19:07 +# Generated by Django 5.1.1 on 2024-10-23 14:11 import datetime @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("receipts", "0001_initial"), + ("receipts", "0002_jobstartedreceipt_is_organic"), ] operations = [ From 02b48c29da3e673bcd1385b0ae803467000afeaf Mon Sep 17 00:00:00 2001 From: Andreea Popescu Date: Thu, 24 Oct 2024 06:33:46 +0100 Subject: [PATCH 78/99] add huggingface volume type --- compute_horde/compute_horde/base/volume.py | 16 ++++- .../management/commands/run_executor.py | 29 ++++++++ .../tests/integration/test_main_loop.py | 67 +++++++++++++++++++ .../src/compute_horde_executor/settings.py | 2 + executor/pyproject.toml | 1 + .../executor_manager/_internal/docker.py | 6 ++ miner/app/src/compute_horde_miner/settings.py | 2 + 7 files changed, 121 insertions(+), 2 deletions(-) diff --git a/compute_horde/compute_horde/base/volume.py b/compute_horde/compute_horde/base/volume.py index b4ec87e8d..2d23a708d 100644 --- a/compute_horde/compute_horde/base/volume.py +++ b/compute_horde/compute_horde/base/volume.py @@ -14,11 +14,22 @@ class VolumeType(str, enum.Enum): zip_url = "zip_url" single_file = "single_file" multi_volume = "multi_volume" + huggingface_volume = "huggingface_volume" def __str__(self): return str.__str__(self) +class HuggingfaceVolume(pydantic.BaseModel): + volume_type: Literal[VolumeType.huggingface_volume] = VolumeType.huggingface_volume + repo_id: str + revision: str | None = None # Git revision id: branch name / tag / commit hash + relative_path: str | None = None + + def is_safe(self) -> bool: + return True + + class InlineVolume(pydantic.BaseModel): volume_type: Literal[VolumeType.inline] = VolumeType.inline contents: str @@ -56,7 +67,8 @@ class MultiVolume(pydantic.BaseModel): volume_type: Literal[VolumeType.multi_volume] = VolumeType.multi_volume volumes: list[ Annotated[ - InlineVolume | ZipUrlVolume | SingleFileVolume, Field(discriminator="volume_type") + InlineVolume | ZipUrlVolume | SingleFileVolume | HuggingfaceVolume, + Field(discriminator="volume_type"), ] ] @@ -65,6 +77,6 @@ def is_safe(self) -> bool: Volume = Annotated[ - InlineVolume | ZipUrlVolume | SingleFileVolume | MultiVolume, + InlineVolume | ZipUrlVolume | SingleFileVolume | MultiVolume | HuggingfaceVolume, Field(discriminator="volume_type"), ] diff --git a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py index 1209d2083..27aafed2a 100644 --- a/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py +++ b/executor/app/src/compute_horde_executor/executor/management/commands/run_executor.py @@ -16,8 +16,10 @@ import httpx import pydantic +from asgiref.sync import sync_to_async from compute_horde.base.docker import DockerRunOptionsPreset from compute_horde.base.volume import ( + HuggingfaceVolume, InlineVolume, MultiVolume, SingleFileVolume, @@ -47,6 +49,7 @@ from compute_horde.utils import MachineSpecs from django.conf import settings from django.core.management.base import BaseCommand +from huggingface_hub import snapshot_download from compute_horde_executor.executor.output_uploader import OutputUploader, OutputUploadFailed @@ -316,6 +319,19 @@ def __init__(self, concurrency=3, max_retries=3): self.semaphore = asyncio.Semaphore(concurrency) self.max_retries = max_retries + def download_from_huggingface( + self, relative_path: pathlib.Path, repo_id: str, revision: str | None + ): + try: + snapshot_download( + repo_id=repo_id, + revision=revision, + token=settings.HF_ACCESS_TOKEN, + local_dir=relative_path, + ) + except Exception as e: + logger.error(f"Failed to download model from Hugging Face: {e}") + async def download(self, fp, url): async with self.semaphore: retries = 0 @@ -581,6 +597,8 @@ async def _unpack_volume(self, volume: Volume | None): await self._unpack_single_file_volume(volume) elif isinstance(volume, MultiVolume): await self._unpack_multi_volume(volume) + elif isinstance(volume, HuggingfaceVolume): + await self._unpack_huggingface_volume(volume) else: raise NotImplementedError(f"Unsupported volume_type: {volume.volume_type}") @@ -608,6 +626,15 @@ async def _unpack_zip_url_volume(self, volume: ZipUrlVolume): extraction_path /= volume.relative_path zip_file.extractall(extraction_path.as_posix()) + async def _unpack_huggingface_volume(self, volume: HuggingfaceVolume): + with tempfile.NamedTemporaryFile(): + extraction_path = self.volume_mount_dir + if volume.relative_path: + extraction_path /= volume.relative_path + await sync_to_async(self.download_manager.download_from_huggingface)( + extraction_path, volume.repo_id, volume.revision + ) + async def _unpack_single_file_volume(self, volume: SingleFileVolume): file_path = self.volume_mount_dir / volume.relative_path file_path.parent.mkdir(parents=True, exist_ok=True) @@ -622,6 +649,8 @@ async def _unpack_multi_volume(self, volume: MultiVolume): await self._unpack_zip_url_volume(sub_volume) elif isinstance(sub_volume, SingleFileVolume): await self._unpack_single_file_volume(sub_volume) + elif isinstance(sub_volume, HuggingfaceVolume): + await self._unpack_huggingface_volume(sub_volume) else: raise NotImplementedError(f"Unsupported sub-volume type: {type(sub_volume)}") diff --git a/executor/app/src/compute_horde_executor/executor/tests/integration/test_main_loop.py b/executor/app/src/compute_horde_executor/executor/tests/integration/test_main_loop.py index 324273e10..1a0761ae3 100644 --- a/executor/app/src/compute_horde_executor/executor/tests/integration/test_main_loop.py +++ b/executor/app/src/compute_horde_executor/executor/tests/integration/test_main_loop.py @@ -8,6 +8,7 @@ import zipfile from functools import partial from unittest import mock +from unittest.mock import patch import httpx from compute_horde.transport import StubTransport @@ -110,6 +111,72 @@ def test_main_loop(): ] +def test_huggingface_volume(): + # Arrange + repo_id = "huggingface/model" + revision = "main" + + def mock_download(local_dir, **kwargs): + with open(local_dir / "payload.txt", "w") as file: + file.write(payload) + + with patch( + "compute_horde_executor.executor.management.commands.run_executor.snapshot_download", + mock_download, + ): + command = CommandTested( + iter( + [ + json.dumps( + { + "message_type": "V0PrepareJobRequest", + "base_docker_image_name": "backenddevelopersltd/compute-horde-job-echo:v0-latest", + "timeout_seconds": None, + "volume_type": "huggingface_volume", + "job_uuid": job_uuid, + } + ), + json.dumps( + { + "message_type": "V0RunJobRequest", + "docker_image_name": "backenddevelopersltd/compute-horde-job-echo:v0-latest", + "docker_run_cmd": [], + "docker_run_options_preset": "none", + "volume": { + "volume_type": "huggingface_volume", + "repo_id": repo_id, + "revision": revision, + }, + "job_uuid": job_uuid, + } + ), + ] + ) + ) + + # Act + command.handle() + + # Assert + assert [json.loads(msg) for msg in command.miner_client_for_tests.transport.sent_messages] == [ + { + "message_type": "V0ReadyRequest", + "job_uuid": job_uuid, + }, + { + "message_type": "V0MachineSpecsRequest", + "specs": mock.ANY, + "job_uuid": job_uuid, + }, + { + "message_type": "V0FinishedRequest", + "docker_process_stdout": payload, + "docker_process_stderr": mock.ANY, + "job_uuid": job_uuid, + }, + ] + + def test_zip_url_volume(httpx_mock: HTTPXMock): zip_url = "https://localhost/payload.txt" httpx_mock.add_response(url=zip_url, content=zip_contents) diff --git a/executor/app/src/compute_horde_executor/settings.py b/executor/app/src/compute_horde_executor/settings.py index 004b5bcde..ac884cd45 100644 --- a/executor/app/src/compute_horde_executor/settings.py +++ b/executor/app/src/compute_horde_executor/settings.py @@ -43,6 +43,8 @@ def wrapped(*args, **kwargs): ENV = env("ENV", default="prod") +HF_ACCESS_TOKEN = env("HF_ACCESS_TOKEN", default=None) + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = env("SECRET_KEY", default="dummy") diff --git a/executor/pyproject.toml b/executor/pyproject.toml index 84c34d330..afbe040b6 100644 --- a/executor/pyproject.toml +++ b/executor/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "prometheus-client~=0.17.0", "django-prometheus==2.3.1", "django-business-metrics @ git+https://github.com/reef-technologies/django-business-metrics.git@9d08ddb3a9d26e8a7e478110d7c8c34c3aa03a01", + "huggingface-hub==0.24.6", ] [build-system] diff --git a/miner/app/src/compute_horde_miner/miner/executor_manager/_internal/docker.py b/miner/app/src/compute_horde_miner/miner/executor_manager/_internal/docker.py index f715579c0..056ee7a49 100644 --- a/miner/app/src/compute_horde_miner/miner/executor_manager/_internal/docker.py +++ b/miner/app/src/compute_horde_miner/miner/executor_manager/_internal/docker.py @@ -65,6 +65,11 @@ async def start_new_executor(self, token, executor_class, timeout): "Pulling executor container timed out, pulling it from shell might provide more details" ) raise ExecutorUnavailable("Failed to pull executor image") + hf_args = ( + [] + if settings.HF_ACCESS_TOKEN is None + else ["-e", f"HF_ACCESS_TOKEN={settings.HF_ACCESS_TOKEN}"] + ) process_executor = await asyncio.create_subprocess_exec( # noqa: S607 "docker", "run", @@ -73,6 +78,7 @@ async def start_new_executor(self, token, executor_class, timeout): f"MINER_ADDRESS=ws://{address}:{settings.PORT_FOR_EXECUTORS}", "-e", f"EXECUTOR_TOKEN={token}", + *hf_args, "--name", token, # the executor must be able to spawn images on host diff --git a/miner/app/src/compute_horde_miner/settings.py b/miner/app/src/compute_horde_miner/settings.py index d097bf58c..5b570e6d2 100644 --- a/miner/app/src/compute_horde_miner/settings.py +++ b/miner/app/src/compute_horde_miner/settings.py @@ -49,6 +49,8 @@ def wrapped(*args, **kwargs): ENV = env("ENV", default="prod") +HF_ACCESS_TOKEN = env("HF_ACCESS_TOKEN", default=None) + DEFAULT_ADMIN_PASSWORD = env("DEFAULT_ADMIN_PASSWORD", default=None) DEFAULT_ADMIN_USERNAME = env("DEFAULT_ADMIN_USERNAME", default="admin") DEFAULT_ADMIN_EMAIL = env("DEFAULT_ADMIN_EMAIL", default="admin@admin.com") From 7dd5c7e264ed50590b5f9ae56790e5b39effc6c4 Mon Sep 17 00:00:00 2001 From: Andreea Popescu Date: Thu, 24 Oct 2024 06:39:05 +0100 Subject: [PATCH 79/99] add hf package --- executor/pdm.lock | 34 +++++++++++++++++++++++++++++++++- executor/pyproject.toml | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/executor/pdm.lock b/executor/pdm.lock index e4b49ff01..36a03b70c 100644 --- a/executor/pdm.lock +++ b/executor/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "format", "lint", "security_check", "test", "type_check"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:ec4b903d2f9da2c6a3336aef9d4dcc090dde39ecb53eb692936b08374ffa32ec" +content_hash = "sha256:642086c0127d76c727836a5476f5b45022ddae3a18d28b4e9ef7f556547b3224" [[metadata.targets]] requires_python = "==3.11.*" @@ -611,6 +611,7 @@ path = "../compute_horde" summary = "" groups = ["default"] dependencies = [ + "Django~=4.2.4", "bittensor<8.0.0,>=7.3.1", "more-itertools>=10.2.0", "pydantic<3,>=2.3", @@ -1115,6 +1116,17 @@ files = [ {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, ] +[[package]] +name = "fsspec" +version = "2024.10.0" +requires_python = ">=3.8" +summary = "File-system specification" +groups = ["default"] +files = [ + {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, + {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, +] + [[package]] name = "fuzzywuzzy" version = "0.18.0" @@ -1186,6 +1198,26 @@ files = [ {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] +[[package]] +name = "huggingface-hub" +version = "0.26.1" +requires_python = ">=3.8.0" +summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +groups = ["default"] +dependencies = [ + "filelock", + "fsspec>=2023.5.0", + "packaging>=20.9", + "pyyaml>=5.1", + "requests", + "tqdm>=4.42.1", + "typing-extensions>=3.7.4.3", +] +files = [ + {file = "huggingface_hub-0.26.1-py3-none-any.whl", hash = "sha256:5927a8fc64ae68859cd954b7cc29d1c8390a5e15caba6d3d349c973be8fdacf3"}, + {file = "huggingface_hub-0.26.1.tar.gz", hash = "sha256:414c0d9b769eecc86c70f9d939d0f48bb28e8461dd1130021542eff0212db890"}, +] + [[package]] name = "humanize" version = "4.10.0" diff --git a/executor/pyproject.toml b/executor/pyproject.toml index afbe040b6..8c9150e8d 100644 --- a/executor/pyproject.toml +++ b/executor/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "prometheus-client~=0.17.0", "django-prometheus==2.3.1", "django-business-metrics @ git+https://github.com/reef-technologies/django-business-metrics.git@9d08ddb3a9d26e8a7e478110d7c8c34c3aa03a01", - "huggingface-hub==0.24.6", + "huggingface-hub>=0.26.1", ] [build-system] From 4c52ff910cecb8374ec6a9f098949d6d65fe3e2b Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 18:01:14 +0300 Subject: [PATCH 80/99] copied signature module from facilitator-sdk --- compute_horde/compute_horde/signature.py | 294 +++++++++++++++++ compute_horde/pdm.lock | 390 +++++++++++++---------- compute_horde/pyproject.toml | 1 + compute_horde/tests/conftest.py | 10 + compute_horde/tests/test_signature.py | 124 +++++++ pdm.lock | 260 ++++++++------- 6 files changed, 797 insertions(+), 282 deletions(-) create mode 100644 compute_horde/compute_horde/signature.py create mode 100644 compute_horde/tests/test_signature.py diff --git a/compute_horde/compute_horde/signature.py b/compute_horde/compute_horde/signature.py new file mode 100644 index 000000000..f3d7134d9 --- /dev/null +++ b/compute_horde/compute_horde/signature.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import abc +import base64 +import dataclasses +import datetime +import hashlib +import json +import re +import time +import typing +from typing import ClassVar, Protocol + +from class_registry import ClassRegistry, RegistryKeyError + +if typing.TYPE_CHECKING: + import bittensor + +JSONValue = str | int | float | bool | None +JSONDict = dict[str, "JSONType"] +JSONArray = list["JSONType"] +JSONType = JSONValue | JSONDict | JSONArray + +SIGNERS_REGISTRY: ClassRegistry[Signer] = ClassRegistry("signature_type") +VERIFIERS_REGISTRY: ClassRegistry[Verifier] = ClassRegistry("signature_type") + + +@dataclasses.dataclass +class Signature: + signature_type: str + signatory: str # identity of the signer (e.g. sa58 address if signature_type == "bittensor") + timestamp_ns: int # UNIX timestamp in nanoseconds + signature: bytes + + +def verify_signature( + payload: JSONType | bytes, + signature: Signature, + *, + newer_than: datetime.datetime | None = None, +): + """ + Verifies the signature of the payload + + :param payload: payload to be verified + :param signature: signature object + :param newer_than: if provided, checks if the signature is newer than the provided timestamp + :return: None + :raises SignatureInvalidException: if the signature is invalid + """ + try: + verifier = VERIFIERS_REGISTRY.get(signature.signature_type) + except RegistryKeyError as e: + raise SignatureInvalidException( + f"Invalid signature type: {signature.signature_type!r}" + ) from e + verifier.verify(payload, signature, newer_than) + + +class SignatureExtractor(Protocol): + def __call__(self, headers: dict[str, str], prefix: str = "") -> Signature: ... + + +def signature_from_headers(headers: dict[str, str], prefix: str = "X-CH-") -> Signature: + """ + Extracts the signature from the headers + + :param headers: headers dict + :return: Signature object + """ + try: + return Signature( + signature_type=headers[f"{prefix}Signature-Type"], + signatory=headers[f"{prefix}Signatory"], + timestamp_ns=int(headers[f"{prefix}Timestamp-NS"]), + signature=base64.b64decode(headers[f"{prefix}Signature"]), + ) + except ( + KeyError, + ValueError, + TypeError, + ) as e: + raise SignatureNotFound("Signature not found in headers") from e + + +def verify_request( + method: str, + url: str, + headers: dict[str, str], + json: JSONType | None = None, + *, + newer_than: datetime.datetime | None = None, + signature_extractor: SignatureExtractor = signature_from_headers, +) -> Signature | None: + """ + Verifies the signature of the request + + :param method: HTTP method + :param url: request URL + :param headers: request headers + :param json: request JSON payload + :param newer_than: if provided, checks if the signature is newer than the provided timestamp + :param signature_extractor: function to extract the signature from the headers + :return: Signature object or None if no signature found + :raises SignatureInvalidException: if the signature is invalid + """ + try: + signature = signature_extractor(headers) + except SignatureNotFound: + return None + try: + verifier = VERIFIERS_REGISTRY.get(signature.signature_type) + except RegistryKeyError as e: + raise SignatureInvalidException( + f"Invalid signature type: {signature.signature_type!r}" + ) from e + payload = verifier.payload_from_request(method, url, headers=headers, json=json) + verifier.verify(payload, signature, newer_than) + return signature + + +def signature_to_headers(signature: Signature, prefix: str = "X-CH-") -> dict[str, str]: + """ + Converts the signature to headers + + :param signature: Signature object + :return: headers dict + """ + return { + f"{prefix}Signature-Type": signature.signature_type, + f"{prefix}Signatory": signature.signatory, + f"{prefix}Timestamp-NS": str(signature.timestamp_ns), + f"{prefix}Signature": base64.b64encode(signature.signature).decode("utf-8"), + } + + +class SignatureException(Exception): + pass + + +class SignatureNotFound(SignatureException): + pass + + +class SignatureInvalidException(SignatureException): + pass + + +class SignatureTimeoutException(SignatureInvalidException): + pass + + +def hash_message_signature(payload: bytes | JSONType, signature: Signature) -> bytes: + """ + Hashes the message to be signed with the signature parameters + + :param payload: payload to be signed + :param signature: incomplete signature object with Signature parameters + :return: + """ + if not isinstance(payload, bytes): + payload = json.dumps(payload, sort_keys=True).encode("utf-8") + + hasher = hashlib.blake2b() + hasher.update(signature.timestamp_ns.to_bytes(8, "big")) + hasher.update(payload) + return hasher.digest() + + +_REMOVE_URL_SCHEME_N_HOST_RE = re.compile(r"^\w+://[^/]+") + + +def signature_payload( + method: str, url: str, headers: dict[str, str], json: JSONType | None = None +) -> JSONType: + reduced_url = _REMOVE_URL_SCHEME_N_HOST_RE.sub("", url) + return { + "action": f"{method.upper()} {reduced_url}", + "json": json, + } + + +class SignatureScheme(abc.ABC): + signature_type: ClassVar[str] + + def payload_from_request( + self, + method: str, + url: str, + headers: dict[str, str], + json: JSONType | None = None, + ): + return signature_payload( + method=method, + url=url, + headers=headers, + json=json, + ) + + +class Signer(SignatureScheme): + def sign(self, payload: JSONType | bytes) -> Signature: + signature = Signature( + signature_type=self.signature_type, + signatory=self.get_signatory(), + timestamp_ns=time.time_ns(), + signature=b"", + ) + payload_hash = hash_message_signature(payload, signature) + signature.signature = self._sign(payload_hash) + return signature + + def signature_for_request( + self, method: str, url: str, headers: dict[str, str], json: JSONType | None = None + ) -> Signature: + return self.sign(self.payload_from_request(method, url, headers=headers, json=json)) + + @abc.abstractmethod + def _sign(self, payload: bytes) -> bytes: + raise NotImplementedError + + @abc.abstractmethod + def get_signatory(self) -> str: + raise NotImplementedError + + +class Verifier(SignatureScheme): + def verify( + self, + payload: JSONType | bytes, + signature: Signature, + newer_than: datetime.datetime | None = None, + ): + payload_hash = hash_message_signature(payload, signature) + self._verify(payload_hash, signature) + + if newer_than is not None: + if newer_than > datetime.datetime.fromtimestamp(signature.timestamp_ns / 1_000_000_000): + raise SignatureTimeoutException("Signature is too old") + + @abc.abstractmethod + def _verify(self, payload: bytes, signature: Signature) -> None: + raise NotImplementedError + + +def _require_bittensor(): + try: + import bittensor + except ImportError as e: + raise ImportError("bittensor package is required for BittensorWalletSigner") from e + return bittensor + + +class BittensorSignatureScheme: + signature_type = "bittensor" + + +@SIGNERS_REGISTRY.register +class BittensorWalletSigner(BittensorSignatureScheme, Signer): + def __init__(self, wallet: bittensor.wallet | bittensor.Keypair | None = None): + bittensor = _require_bittensor() + + if isinstance(wallet, bittensor.Keypair): + keypair = wallet + else: + keypair = (wallet or bittensor.wallet()).hotkey + self._keypair = keypair + + def _sign(self, payload: bytes) -> bytes: + signature: bytes = self._keypair.sign(payload) + return signature + + def get_signatory(self) -> str: + signatory: str = self._keypair.ss58_address + return signatory + + +@VERIFIERS_REGISTRY.register +class BittensorWalletVerifier(BittensorSignatureScheme, Verifier): + def __init__(self, *args, **kwargs): + self._bittensor = _require_bittensor() + + super().__init__(*args, **kwargs) + + def _verify(self, payload: bytes, signature: Signature) -> None: + try: + keypair = self._bittensor.Keypair(ss58_address=signature.signatory) + except ValueError: + raise SignatureInvalidException("Invalid signatory for BittensorWalletVerifier") + try: + if not keypair.verify(data=payload, signature=signature.signature): + raise SignatureInvalidException("Signature is invalid") + except (ValueError, TypeError) as e: + raise SignatureInvalidException("Signature is malformed") from e diff --git a/compute_horde/pdm.lock b/compute_horde/pdm.lock index 006f67895..ff8a74f95 100644 --- a/compute_horde/pdm.lock +++ b/compute_horde/pdm.lock @@ -5,25 +5,25 @@ groups = ["default", "format", "lint", "release", "test", "type_check"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:6c0e95ad053091a2938575210bf10a2efa03ab8148a955d0227d0a8f2ebcdea4" +content_hash = "sha256:4b26dffcfd6d1a1d819ff8bb949ebe47ec3cead08108de871630aa8bc7a0de8c" [[metadata.targets]] requires_python = "==3.11.*" [[package]] name = "aiohappyeyeballs" -version = "2.4.0" +version = "2.4.3" requires_python = ">=3.8" summary = "Happy Eyeballs for asyncio" groups = ["default"] files = [ - {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"}, - {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"}, + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, ] [[package]] name = "aiohttp" -version = "3.10.6" +version = "3.10.10" requires_python = ">=3.8" summary = "Async http client/server framework (asyncio)" groups = ["default"] @@ -37,22 +37,22 @@ dependencies = [ "yarl<2.0,>=1.12.0", ] files = [ - {file = "aiohttp-3.10.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f52e54fd776ad0da1006708762213b079b154644db54bcfc62f06eaa5b896402"}, - {file = "aiohttp-3.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:995ab1a238fd0d19dc65f2d222e5eb064e409665c6426a3e51d5101c1979ee84"}, - {file = "aiohttp-3.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0749c4d5a08a802dd66ecdf59b2df4d76b900004017468a7bb736c3b5a3dd902"}, - {file = "aiohttp-3.10.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e05b39158f2af0e2438cc2075cfc271f4ace0c3cc4a81ec95b27a0432e161951"}, - {file = "aiohttp-3.10.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9f196c970db2dcde4f24317e06615363349dc357cf4d7a3b0716c20ac6d7bcd"}, - {file = "aiohttp-3.10.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47647c8af04a70e07a2462931b0eba63146a13affa697afb4ecbab9d03a480ce"}, - {file = "aiohttp-3.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669c0efe7e99f6d94d63274c06344bd0e9c8daf184ce5602a29bc39e00a18720"}, - {file = "aiohttp-3.10.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9721cdd83a994225352ca84cd537760d41a9da3c0eacb3ff534747ab8fba6d0"}, - {file = "aiohttp-3.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0b82c8ebed66ce182893e7c0b6b60ba2ace45b1df104feb52380edae266a4850"}, - {file = "aiohttp-3.10.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b169f8e755e541b72e714b89a831b315bbe70db44e33fead28516c9e13d5f931"}, - {file = "aiohttp-3.10.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0be3115753baf8b4153e64f9aa7bf6c0c64af57979aa900c31f496301b374570"}, - {file = "aiohttp-3.10.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e1f80cd17d81a404b6e70ef22bfe1870bafc511728397634ad5f5efc8698df56"}, - {file = "aiohttp-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6419728b08fb6380c66a470d2319cafcec554c81780e2114b7e150329b9a9a7f"}, - {file = "aiohttp-3.10.6-cp311-cp311-win32.whl", hash = "sha256:bd294dcdc1afdc510bb51d35444003f14e327572877d016d576ac3b9a5888a27"}, - {file = "aiohttp-3.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:bf861da9a43d282d6dd9dcd64c23a0fccf2c5aa5cd7c32024513c8c79fb69de3"}, - {file = "aiohttp-3.10.6.tar.gz", hash = "sha256:d2578ef941be0c2ba58f6f421a703527d08427237ed45ecb091fed6f83305336"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c30a0eafc89d28e7f959281b58198a9fa5e99405f716c0289b7892ca345fe45f"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:258c5dd01afc10015866114e210fb7365f0d02d9d059c3c3415382ab633fcbcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:15ecd889a709b0080f02721255b3f80bb261c2293d3c748151274dfea93ac871"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3935f82f6f4a3820270842e90456ebad3af15810cf65932bd24da4463bc0a4c"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:413251f6fcf552a33c981c4709a6bba37b12710982fec8e558ae944bfb2abd38"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1720b4f14c78a3089562b8875b53e36b51c97c51adc53325a69b79b4b48ebcb"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:679abe5d3858b33c2cf74faec299fda60ea9de62916e8b67e625d65bf069a3b7"}, + {file = "aiohttp-3.10.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79019094f87c9fb44f8d769e41dbb664d6e8fcfd62f665ccce36762deaa0e911"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe2fb38c2ed905a2582948e2de560675e9dfbee94c6d5ccdb1301c6d0a5bf092"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a3f00003de6eba42d6e94fabb4125600d6e484846dbf90ea8e48a800430cc142"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1bbb122c557a16fafc10354b9d99ebf2f2808a660d78202f10ba9d50786384b9"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:30ca7c3b94708a9d7ae76ff281b2f47d8eaf2579cd05971b5dc681db8caac6e1"}, + {file = "aiohttp-3.10.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:df9270660711670e68803107d55c2b5949c2e0f2e4896da176e1ecfc068b974a"}, + {file = "aiohttp-3.10.10-cp311-cp311-win32.whl", hash = "sha256:aafc8ee9b742ce75044ae9a4d3e60e3d918d15a4c2e08a6c3c3e38fa59b92d94"}, + {file = "aiohttp-3.10.10-cp311-cp311-win_amd64.whl", hash = "sha256:362f641f9071e5f3ee6f8e7d37d5ed0d95aae656adf4ef578313ee585b585959"}, + {file = "aiohttp-3.10.10.tar.gz", hash = "sha256:0631dd7c9f0822cc61c88586ca76d5b5ada26538097d0f1df510b082bad3411a"}, ] [[package]] @@ -131,7 +131,7 @@ files = [ [[package]] name = "anyio" -version = "4.6.0" +version = "4.6.2.post1" requires_python = ">=3.9" summary = "High level compatibility layer for multiple asynchronous event loop implementations" groups = ["default"] @@ -142,8 +142,8 @@ dependencies = [ "typing-extensions>=4.1; python_version < \"3.11\"", ] files = [ - {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, - {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, + {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, + {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, ] [[package]] @@ -282,28 +282,28 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.3.2" +version = "3.4.0" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." groups = ["default", "test", "type_check"] files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] [[package]] @@ -397,8 +397,8 @@ files = [ [[package]] name = "cytoolz" -version = "0.12.3" -requires_python = ">=3.7" +version = "1.0.0" +requires_python = ">=3.8" summary = "Cython implementation of Toolz: High performance functional utilities" groups = ["default"] marker = "implementation_name == \"cpython\"" @@ -406,21 +406,21 @@ dependencies = [ "toolz>=0.8.0", ] files = [ - {file = "cytoolz-0.12.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ac4f2fb38bbc67ff1875b7d2f0f162a247f43bd28eb7c9d15e6175a982e558d"}, - {file = "cytoolz-0.12.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0cf1e1e96dd86829a0539baf514a9c8473a58fbb415f92401a68e8e52a34ecd5"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08a438701c6141dd34eaf92e9e9a1f66e23a22f7840ef8a371eba274477de85d"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6b6f11b0d7ed91be53166aeef2a23a799e636625675bb30818f47f41ad31821"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7fde09384d23048a7b4ac889063761e44b89a0b64015393e2d1d21d5c1f534a"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d3bfe45173cc8e6c76206be3a916d8bfd2214fb2965563e288088012f1dabfc"}, - {file = "cytoolz-0.12.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27513a5d5b6624372d63313574381d3217a66e7a2626b056c695179623a5cb1a"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d294e5e81ff094fe920fd545052ff30838ea49f9e91227a55ecd9f3ca19774a0"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:727b01a2004ddb513496507a695e19b5c0cfebcdfcc68349d3efd92a1c297bf4"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:fe1e1779a39dbe83f13886d2b4b02f8c4b10755e3c8d9a89b630395f49f4f406"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:de74ef266e2679c3bf8b5fc20cee4fc0271ba13ae0d9097b1491c7a9bcadb389"}, - {file = "cytoolz-0.12.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e04d22049233394e0b08193aca9737200b4a2afa28659d957327aa780ddddf2"}, - {file = "cytoolz-0.12.3-cp311-cp311-win32.whl", hash = "sha256:20d36430d8ac809186736fda735ee7d595b6242bdb35f69b598ef809ebfa5605"}, - {file = "cytoolz-0.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:780c06110f383344d537f48d9010d79fa4f75070d214fc47f389357dd4f010b6"}, - {file = "cytoolz-0.12.3.tar.gz", hash = "sha256:4503dc59f4ced53a54643272c61dc305d1dbbfbd7d6bdf296948de9f34c3a282"}, + {file = "cytoolz-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dffc22fd2c91be64dbdbc462d0786f8e8ac9a275cfa1869a1084d1867d4f67e0"}, + {file = "cytoolz-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a99e7e29274e293f4ffe20e07f76c2ac753a78f1b40c1828dfc54b2981b2f6c4"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c507a3e0a45c41d66b43f96797290d75d1e7a8549aa03a4a6b8854fdf3f7b8d8"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:643a593ec272ef7429099e1182a22f64ec2696c00d295d2a5be390db1b7ff176"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ce38e2e42cbae30446190c59b92a8a9029e1806fd79eaf88f48b0fe33003893"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810a6a168b8c5ecb412fbae3dd6f7ed6c6253a63caf4174ee9794ebd29b2224f"}, + {file = "cytoolz-1.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ce8a2a85c0741c1b19b16e6782c4a5abc54c3caecda66793447112ab2fa9884"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ea4ac72e6b830861035c4c7999af8e55813f57c6d1913a3d93cc4a6babc27bf7"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a09cdfb21dfb38aa04df43e7546a41f673377eb5485da88ceb784e327ec7603b"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:658dd85deb375ff7af990a674e5c9058cef1c9d1f5dc89bc87b77be499348144"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9715d1ff5576919d10b68f17241375f6a1eec8961c25b78a83e6ef1487053f39"}, + {file = "cytoolz-1.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f370a1f1f1afc5c1c8cc5edc1cfe0ba444263a0772af7ce094be8e734f41769d"}, + {file = "cytoolz-1.0.0-cp311-cp311-win32.whl", hash = "sha256:dbb2ec1177dca700f3db2127e572da20de280c214fc587b2a11c717fc421af56"}, + {file = "cytoolz-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0983eee73df86e54bb4a79fcc4996aa8b8368fdbf43897f02f9c3bf39c4dc4fb"}, + {file = "cytoolz-1.0.0.tar.gz", hash = "sha256:eb453b30182152f9917a5189b7d99046b6ce90cdf8aeb0feff4b2683e600defd"}, ] [[package]] @@ -576,7 +576,7 @@ files = [ [[package]] name = "eth-keys" -version = "0.5.1" +version = "0.6.0" requires_python = "<4,>=3.8" summary = "eth-keys: Common API for Ethereum key operations" groups = ["default"] @@ -585,13 +585,13 @@ dependencies = [ "eth-utils>=2", ] files = [ - {file = "eth_keys-0.5.1-py3-none-any.whl", hash = "sha256:ad13d920a2217a49bed3a1a7f54fb0980f53caf86d3bbab2139fd3330a17b97e"}, - {file = "eth_keys-0.5.1.tar.gz", hash = "sha256:2b587e4bbb9ac2195215a7ab0c0fb16042b17d4ec50240ed670bbb8f53da7a48"}, + {file = "eth_keys-0.6.0-py3-none-any.whl", hash = "sha256:b396fdfe048a5bba3ef3990739aec64901eb99901c03921caa774be668b1db6e"}, + {file = "eth_keys-0.6.0.tar.gz", hash = "sha256:ba33230f851d02c894e83989185b21d76152c49b37e35b61b1d8a6d9f1d20430"}, ] [[package]] name = "eth-typing" -version = "5.0.0" +version = "5.0.1" requires_python = "<4,>=3.8" summary = "eth-typing: Common type annotations for ethereum python packages" groups = ["default"] @@ -599,8 +599,8 @@ dependencies = [ "typing-extensions>=4.5.0", ] files = [ - {file = "eth_typing-5.0.0-py3-none-any.whl", hash = "sha256:c7ebc8595e7b65175bb4b4176c2b548ab21b13329f2058e84d4f8c289ba9f577"}, - {file = "eth_typing-5.0.0.tar.gz", hash = "sha256:87ce7cee75665c09d2dcff8de1b496609d5e32fcd2e2b1d8fc0370c29eedcdc0"}, + {file = "eth_typing-5.0.1-py3-none-any.whl", hash = "sha256:f30d1af16aac598f216748a952eeb64fbcb6e73efa691d2de31148138afe96de"}, + {file = "eth_typing-5.0.1.tar.gz", hash = "sha256:83debf88c9df286db43bb7374974681ebcc9f048fac81be2548dbc549a3203c0"}, ] [[package]] @@ -653,28 +653,28 @@ files = [ [[package]] name = "frozenlist" -version = "1.4.1" +version = "1.5.0" requires_python = ">=3.8" summary = "A list-like structure which implements collections.abc.MutableSequence" groups = ["default"] files = [ - {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, - {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, - {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, - {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, - {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, - {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, - {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf"}, + {file = "frozenlist-1.5.0-cp311-cp311-win32.whl", hash = "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942"}, + {file = "frozenlist-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, ] [[package]] @@ -795,22 +795,22 @@ files = [ [[package]] name = "markupsafe" -version = "2.1.5" -requires_python = ">=3.7" +version = "3.0.2" +requires_python = ">=3.9" summary = "Safely add untrusted strings to HTML/XML markup." groups = ["default", "release"] files = [ - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] @@ -1008,6 +1008,17 @@ files = [ {file = "password_strength-0.0.3.post2.tar.gz", hash = "sha256:bf4df10a58fcd3abfa182367307b4fd7b1cec518121dd83bf80c1c42ba796762"}, ] +[[package]] +name = "phx-class-registry" +version = "5.1.1" +requires_python = "<4.0,>=3.11" +summary = "Factory+Registry pattern for Python classes" +groups = ["default"] +files = [ + {file = "phx_class_registry-5.1.1-py3-none-any.whl", hash = "sha256:b093ecc1dad34c5dc6eda2530046d956f2303a5cfaa543bf7fba35ce3c7b1672"}, + {file = "phx_class_registry-5.1.1.tar.gz", hash = "sha256:06c9af198b846a7530406314f63f8d83441daf42d29ee25d8c0b19a9dbc37939"}, +] + [[package]] name = "pluggy" version = "1.5.0" @@ -1019,6 +1030,33 @@ files = [ {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] +[[package]] +name = "propcache" +version = "0.2.0" +requires_python = ">=3.8" +summary = "Accelerated property cache" +groups = ["default"] +files = [ + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de"}, + {file = "propcache-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4"}, + {file = "propcache-0.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b"}, + {file = "propcache-0.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b"}, + {file = "propcache-0.2.0-cp311-cp311-win32.whl", hash = "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1"}, + {file = "propcache-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71"}, + {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, + {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, +] + [[package]] name = "py" version = "1.11.0" @@ -1097,22 +1135,22 @@ files = [ [[package]] name = "pycryptodome" -version = "3.20.0" -requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "3.21.0" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" summary = "Cryptographic library for Python" groups = ["default"] files = [ - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, - {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:2480ec2c72438430da9f601ebc12c518c093c13111a5c1644c82cdfc2e50b1e4"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:de18954104667f565e2fbb4783b56667f30fb49c4d79b346f52a29cb198d5b6b"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de4b7263a33947ff440412339cb72b28a5a4c769b5c1ca19e33dd6cd1dcec6e"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0714206d467fc911042d01ea3a1847c847bc10884cf674c82e12915cfe1649f8"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d85c1b613121ed3dbaa5a97369b3b757909531a959d229406a75b912dd51dd1"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8898a66425a57bcf15e25fc19c12490b87bd939800f39a03ea2de2aea5e3611a"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:932c905b71a56474bff8a9c014030bc3c882cee696b448af920399f730a650c2"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18caa8cfbc676eaaf28613637a89980ad2fd96e00c564135bf90bc3f0b34dd93"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win32.whl", hash = "sha256:280b67d20e33bb63171d55b1067f61fbd932e0b1ad976b3a184303a3dad22764"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b7aa25fc0baa5b1d95b7633af4f5f1838467f1815442b22487426f94e0d66c53"}, + {file = "pycryptodome-3.21.0.tar.gz", hash = "sha256:f7787e0d469bdae763b876174cf2e6c0f7be79808af26b1da96f1a64bcf47297"}, ] [[package]] @@ -1365,45 +1403,45 @@ files = [ [[package]] name = "rich" -version = "13.8.1" -requires_python = ">=3.7.0" +version = "13.9.3" +requires_python = ">=3.8.0" summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" groups = ["default"] dependencies = [ "markdown-it-py>=2.2.0", "pygments<3.0.0,>=2.13.0", - "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"", + "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", ] files = [ - {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"}, - {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"}, + {file = "rich-13.9.3-py3-none-any.whl", hash = "sha256:9836f5096eb2172c9e77df411c1b009bace4193d6a481d534fea75ebba758283"}, + {file = "rich-13.9.3.tar.gz", hash = "sha256:bc1e01b899537598cf02579d2b9f4a415104d3fc439313a7a2c165d76557a08e"}, ] [[package]] name = "ruff" -version = "0.6.7" +version = "0.7.0" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." groups = ["format", "lint"] files = [ - {file = "ruff-0.6.7-py3-none-linux_armv6l.whl", hash = "sha256:08277b217534bfdcc2e1377f7f933e1c7957453e8a79764d004e44c40db923f2"}, - {file = "ruff-0.6.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c6707a32e03b791f4448dc0dce24b636cbcdee4dd5607adc24e5ee73fd86c00a"}, - {file = "ruff-0.6.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:533d66b7774ef224e7cf91506a7dafcc9e8ec7c059263ec46629e54e7b1f90ab"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17a86aac6f915932d259f7bec79173e356165518859f94649d8c50b81ff087e9"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3f8822defd260ae2460ea3832b24d37d203c3577f48b055590a426a722d50ef"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba4efe5c6dbbb58be58dd83feedb83b5e95c00091bf09987b4baf510fee5c99"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:525201b77f94d2b54868f0cbe5edc018e64c22563da6c5c2e5c107a4e85c1c0d"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8854450839f339e1049fdbe15d875384242b8e85d5c6947bb2faad33c651020b"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f0b62056246234d59cbf2ea66e84812dc9ec4540518e37553513392c171cb18"}, - {file = "ruff-0.6.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b1462fa56c832dc0cea5b4041cfc9c97813505d11cce74ebc6d1aae068de36b"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:02b083770e4cdb1495ed313f5694c62808e71764ec6ee5db84eedd82fd32d8f5"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0c05fd37013de36dfa883a3854fae57b3113aaa8abf5dea79202675991d48624"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f49c9caa28d9bbfac4a637ae10327b3db00f47d038f3fbb2195c4d682e925b14"}, - {file = "ruff-0.6.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a0e1655868164e114ba43a908fd2d64a271a23660195017c17691fb6355d59bb"}, - {file = "ruff-0.6.7-py3-none-win32.whl", hash = "sha256:a939ca435b49f6966a7dd64b765c9df16f1faed0ca3b6f16acdf7731969deb35"}, - {file = "ruff-0.6.7-py3-none-win_amd64.whl", hash = "sha256:590445eec5653f36248584579c06252ad2e110a5d1f32db5420de35fb0e1c977"}, - {file = "ruff-0.6.7-py3-none-win_arm64.whl", hash = "sha256:b28f0d5e2f771c1fe3c7a45d3f53916fc74a480698c4b5731f0bea61e52137c8"}, - {file = "ruff-0.6.7.tar.gz", hash = "sha256:44e52129d82266fa59b587e2cd74def5637b730a69c4542525dfdecfaae38bd5"}, + {file = "ruff-0.7.0-py3-none-linux_armv6l.whl", hash = "sha256:0cdf20c2b6ff98e37df47b2b0bd3a34aaa155f59a11182c1303cce79be715628"}, + {file = "ruff-0.7.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:496494d350c7fdeb36ca4ef1c9f21d80d182423718782222c29b3e72b3512737"}, + {file = "ruff-0.7.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:214b88498684e20b6b2b8852c01d50f0651f3cc6118dfa113b4def9f14faaf06"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630fce3fefe9844e91ea5bbf7ceadab4f9981f42b704fae011bb8efcaf5d84be"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:211d877674e9373d4bb0f1c80f97a0201c61bcd1e9d045b6e9726adc42c156aa"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:194d6c46c98c73949a106425ed40a576f52291c12bc21399eb8f13a0f7073495"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:82c2579b82b9973a110fab281860403b397c08c403de92de19568f32f7178598"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9af971fe85dcd5eaed8f585ddbc6bdbe8c217fb8fcf510ea6bca5bdfff56040e"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b641c7f16939b7d24b7bfc0be4102c56562a18281f84f635604e8a6989948914"}, + {file = "ruff-0.7.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d71672336e46b34e0c90a790afeac8a31954fd42872c1f6adaea1dff76fd44f9"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ab7d98c7eed355166f367597e513a6c82408df4181a937628dbec79abb2a1fe4"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1eb54986f770f49edb14f71d33312d79e00e629a57387382200b1ef12d6a4ef9"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dc452ba6f2bb9cf8726a84aa877061a2462afe9ae0ea1d411c53d226661c601d"}, + {file = "ruff-0.7.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4b406c2dce5be9bad59f2de26139a86017a517e6bcd2688da515481c05a2cb11"}, + {file = "ruff-0.7.0-py3-none-win32.whl", hash = "sha256:f6c968509f767776f524a8430426539587d5ec5c662f6addb6aa25bc2e8195ec"}, + {file = "ruff-0.7.0-py3-none-win_amd64.whl", hash = "sha256:ff4aabfbaaba880e85d394603b9e75d32b0693152e16fa659a3064a85df7fce2"}, + {file = "ruff-0.7.0-py3-none-win_arm64.whl", hash = "sha256:10842f69c245e78d6adec7e1db0a7d9ddc2fff0621d730e61657b64fa36f207e"}, + {file = "ruff-0.7.0.tar.gz", hash = "sha256:47a86360cf62d9cd53ebfb0b5eb0e882193fc191c6d717e8bef4462bc3b9ea2b"}, ] [[package]] @@ -1494,7 +1532,7 @@ files = [ [[package]] name = "substrate-interface" -version = "1.7.10" +version = "1.7.11" requires_python = "<4,>=3.7" summary = "Library for interfacing with a Substrate node" groups = ["default"] @@ -1504,7 +1542,7 @@ dependencies = [ "certifi>=2019.3.9", "ecdsa<1,>=0.17.0", "eth-keys<1,>=0.2.1", - "eth-utils<3,>=1.3.0", + "eth-utils<6,>=1.3.0", "idna<4,>=2.1.0", "py-bip39-bindings<1,>=0.1.9", "py-ed25519-zebra-bindings<2,>=1.0", @@ -1516,31 +1554,31 @@ dependencies = [ "xxhash<4,>=1.3.0", ] files = [ - {file = "substrate-interface-1.7.10.tar.gz", hash = "sha256:0dec0104abc16d01c3d22700253a84e67430b5e56c46efea71fea47063a8eaa4"}, - {file = "substrate_interface-1.7.10-py3-none-any.whl", hash = "sha256:4873e9f1b75375ed9fcdd12d7bca66c47ab0e9fbd532ec4f9538ceac0f0ab2f5"}, + {file = "substrate-interface-1.7.11.tar.gz", hash = "sha256:4caa5eacb9996edbe76ad12249521b3542bbd8d9d69b96734087201db1fef8f6"}, + {file = "substrate_interface-1.7.11-py3-none-any.whl", hash = "sha256:ce19bc97481769238ed23c752db985a3058637918693f2db6aeed2fab3756075"}, ] [[package]] name = "termcolor" -version = "2.4.0" -requires_python = ">=3.8" +version = "2.5.0" +requires_python = ">=3.9" summary = "ANSI color formatting for output in terminal" groups = ["default"] files = [ - {file = "termcolor-2.4.0-py3-none-any.whl", hash = "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63"}, - {file = "termcolor-2.4.0.tar.gz", hash = "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"}, + {file = "termcolor-2.5.0-py3-none-any.whl", hash = "sha256:37b17b5fc1e604945c2642c872a3764b5d547a48009871aea3edd3afa180afb8"}, + {file = "termcolor-2.5.0.tar.gz", hash = "sha256:998d8d27da6d48442e8e1f016119076b690d962507531df4890fcd2db2ef8a6f"}, ] [[package]] name = "toolz" -version = "0.12.1" -requires_python = ">=3.7" +version = "1.0.0" +requires_python = ">=3.8" summary = "List processing tools and functional utilities" groups = ["default"] marker = "implementation_name == \"pypy\" or implementation_name == \"cpython\"" files = [ - {file = "toolz-0.12.1-py3-none-any.whl", hash = "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85"}, - {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"}, + {file = "toolz-1.0.0-py3-none-any.whl", hash = "sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236"}, + {file = "toolz-1.0.0.tar.gz", hash = "sha256:2c86e3d9a04798ac556793bced838816296a2f085017664e4995cb40a1047a02"}, ] [[package]] @@ -1587,13 +1625,13 @@ files = [ [[package]] name = "types-python-dateutil" -version = "2.9.0.20240906" +version = "2.9.0.20241003" requires_python = ">=3.8" summary = "Typing stubs for python-dateutil" groups = ["type_check"] files = [ - {file = "types-python-dateutil-2.9.0.20240906.tar.gz", hash = "sha256:9706c3b68284c25adffc47319ecc7947e5bb86b3773f843c73906fd598bc176e"}, - {file = "types_python_dateutil-2.9.0.20240906-py3-none-any.whl", hash = "sha256:27c8cc2d058ccb14946eebcaaa503088f4f6dbc4fb6093d3d456a49aef2753f6"}, + {file = "types-python-dateutil-2.9.0.20241003.tar.gz", hash = "sha256:58cb85449b2a56d6684e41aeefb4c4280631246a0da1a719bdbe6f3fb0317446"}, + {file = "types_python_dateutil-2.9.0.20241003-py3-none-any.whl", hash = "sha256:250e1d8e80e7bbc3a6c99b907762711d1a1cdd00e978ad39cb5940f6f0a87f3d"}, ] [[package]] @@ -1609,7 +1647,7 @@ files = [ [[package]] name = "types-requests" -version = "2.32.0.20240914" +version = "2.32.0.20241016" requires_python = ">=3.8" summary = "Typing stubs for requests" groups = ["type_check"] @@ -1617,8 +1655,8 @@ dependencies = [ "urllib3>=2", ] files = [ - {file = "types-requests-2.32.0.20240914.tar.gz", hash = "sha256:2850e178db3919d9bf809e434eef65ba49d0e7e33ac92d588f4a5e295fffd405"}, - {file = "types_requests-2.32.0.20240914-py3-none-any.whl", hash = "sha256:59c2f673eb55f32a99b2894faf6020e1a9f4a402ad0f192bfee0b64469054310"}, + {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, + {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, ] [[package]] @@ -1657,7 +1695,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.30.6" +version = "0.32.0" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["default"] @@ -1667,8 +1705,8 @@ dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.30.6-py3-none-any.whl", hash = "sha256:65fd46fe3fda5bdc1b03b94eb634923ff18cd35b2f084813ea79d1f103f711b5"}, - {file = "uvicorn-0.30.6.tar.gz", hash = "sha256:4b15decdda1e72be08209e860a1e10e92439ad5b97cf44cc945fcbee66fc5788"}, + {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, + {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, ] [[package]] @@ -1742,30 +1780,32 @@ files = [ [[package]] name = "yarl" -version = "1.12.1" -requires_python = ">=3.8" +version = "1.16.0" +requires_python = ">=3.9" summary = "Yet another URL library" groups = ["default"] dependencies = [ "idna>=2.0", "multidict>=4.0", -] -files = [ - {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:28389a68981676bf74e2e199fe42f35d1aa27a9c98e3a03e6f58d2d3d054afe1"}, - {file = "yarl-1.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f736f54565f8dd7e3ab664fef2bc461d7593a389a7f28d4904af8d55a91bd55f"}, - {file = "yarl-1.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dee0496d5f1a8f57f0f28a16f81a2033fc057a2cf9cd710742d11828f8c80e2"}, - {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8981a94a27ac520a398302afb74ae2c0be1c3d2d215c75c582186a006c9e7b0"}, - {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff54340fc1129e8e181827e2234af3ff659b4f17d9bbe77f43bc19e6577fadec"}, - {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54c8cee662b5f8c30ad7eedfc26123f845f007798e4ff1001d9528fe959fd23c"}, - {file = "yarl-1.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e97a29b37830ba1262d8dfd48ddb5b28ad4d3ebecc5d93a9c7591d98641ec737"}, - {file = "yarl-1.12.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c89894cc6f6ddd993813e79244b36b215c14f65f9e4f1660b1f2ba9e5594b95"}, - {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:712ba8722c0699daf186de089ddc4677651eb9875ed7447b2ad50697522cbdd9"}, - {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6e9a9f50892153bad5046c2a6df153224aa6f0573a5a8ab44fc54a1e886f6e21"}, - {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1d4017e78fb22bc797c089b746230ad78ecd3cdb215bc0bd61cb72b5867da57e"}, - {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f494c01b28645c431239863cb17af8b8d15b93b0d697a0320d5dd34cd9d7c2fa"}, - {file = "yarl-1.12.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de4544b1fb29cf14870c4e2b8a897c0242449f5dcebd3e0366aa0aa3cf58a23a"}, - {file = "yarl-1.12.1-cp311-cp311-win32.whl", hash = "sha256:7564525a4673fde53dee7d4c307a961c0951918f0b8c7f09b2c9e02067cf6504"}, - {file = "yarl-1.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:f23bb1a7a6e8e8b612a164fdd08e683bcc16c76f928d6dbb7bdbee2374fbfee6"}, - {file = "yarl-1.12.1-py3-none-any.whl", hash = "sha256:dc3192a81ecd5ff954cecd690327badd5a84d00b877e1573f7c9097ce13e5bfb"}, - {file = "yarl-1.12.1.tar.gz", hash = "sha256:5b860055199aec8d6fe4dcee3c5196ce506ca198a50aab0059ffd26e8e815828"}, + "propcache>=0.2.0", +] +files = [ + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d8643975a0080f361639787415a038bfc32d29208a4bf6b783ab3075a20b1ef3"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:676d96bafc8c2d0039cea0cd3fd44cee7aa88b8185551a2bb93354668e8315c2"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d9525f03269e64310416dbe6c68d3b23e5d34aaa8f47193a1c45ac568cecbc49"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b37d5ec034e668b22cf0ce1074d6c21fd2a08b90d11b1b73139b750a8b0dd97"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f32c4cb7386b41936894685f6e093c8dfaf0960124d91fe0ec29fe439e201d0"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b8e265a0545637492a7e12fd7038370d66c9375a61d88c5567d0e044ded9202"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:789a3423f28a5fff46fbd04e339863c169ece97c827b44de16e1a7a42bc915d2"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1d1f45e3e8d37c804dca99ab3cf4ab3ed2e7a62cd82542924b14c0a4f46d243"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:621280719c4c5dad4c1391160a9b88925bb8b0ff6a7d5af3224643024871675f"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ed097b26f18a1f5ff05f661dc36528c5f6735ba4ce8c9645e83b064665131349"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f1fe2b2e3ee418862f5ebc0c0083c97f6f6625781382f828f6d4e9b614eba9b"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:87dd10bc0618991c66cee0cc65fa74a45f4ecb13bceec3c62d78ad2e42b27a16"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4199db024b58a8abb2cfcedac7b1292c3ad421684571aeb622a02f242280e8d6"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:99a9dcd4b71dd5f5f949737ab3f356cfc058c709b4f49833aeffedc2652dac56"}, + {file = "yarl-1.16.0-cp311-cp311-win32.whl", hash = "sha256:a9394c65ae0ed95679717d391c862dece9afacd8fa311683fc8b4362ce8a410c"}, + {file = "yarl-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b9101f528ae0f8f65ac9d64dda2bb0627de8a50344b2f582779f32fda747c1d"}, + {file = "yarl-1.16.0-py3-none-any.whl", hash = "sha256:e6980a558d8461230c457218bd6c92dfc1d10205548215c2c21d79dc8d0a96f3"}, + {file = "yarl-1.16.0.tar.gz", hash = "sha256:b6f687ced5510a9a2474bbae96a4352e5ace5fa34dc44a217b0537fec1db00b4"}, ] diff --git a/compute_horde/pyproject.toml b/compute_horde/pyproject.toml index 641efa7cc..2f1905f48 100644 --- a/compute_horde/pyproject.toml +++ b/compute_horde/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "bittensor<8.0.0,>=7.3.1", "websockets>=11.0", "more-itertools>=10.2.0", + "phx-class-registry>=4.1.0", "requests>=2.32.2", "Django~=4.2.4", ] diff --git a/compute_horde/tests/conftest.py b/compute_horde/tests/conftest.py index 844d5aaf0..ffd8c3c8b 100644 --- a/compute_horde/tests/conftest.py +++ b/compute_horde/tests/conftest.py @@ -1,5 +1,6 @@ import datetime +import bittensor import pytest import responses from bittensor import Keypair @@ -33,6 +34,15 @@ def miner_keypair(): ) +@pytest.fixture +def signature_wallet(): + wallet = bittensor.wallet(name="test_signature") + # workaround the overwrite flag + wallet.regenerate_coldkey(seed="8" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="9" * 64, use_password=False, overwrite=True) + return wallet + + @pytest.fixture def receipts(validator_keypair, miner_keypair): payload1 = JobStartedReceiptPayload( diff --git a/compute_horde/tests/test_signature.py b/compute_horde/tests/test_signature.py new file mode 100644 index 000000000..62952b31a --- /dev/null +++ b/compute_horde/tests/test_signature.py @@ -0,0 +1,124 @@ +import base64 +import dataclasses +import datetime + +import freezegun +import pytest + +from compute_horde.signature import ( + SIGNERS_REGISTRY, + VERIFIERS_REGISTRY, + BittensorWalletSigner, + BittensorWalletVerifier, + Signature, + SignatureInvalidException, + SignatureNotFound, + hash_message_signature, + signature_from_headers, + signature_payload, +) + + +def test_available_signers(): + assert list(SIGNERS_REGISTRY) == ["bittensor"] + + +def test_available_verifiers(): + assert list(VERIFIERS_REGISTRY) == ["bittensor"] + + +@pytest.fixture +def sample_data(): + return { + "b": 2, + "array": [1, 2, 3], + "dict": {"subdict:": {"null": None}}, + } + + +@pytest.fixture +def sample_signature(): + return Signature( + signature_type="bittensor", + signatory="5FUJCuGtQPonu8B9JKH4BwsKzEdtyBTpyvbBk2beNZ4iX8sk", # hotkey address + timestamp_ns=1718845323456788992, + signature=base64.b85decode( + "1SaAcLt*GG`2RG*@xmapXZ14m*Y`@b1MP(hAfEnwXkO5Os<30drw{`X`15JFP4GWR96T7p>rUmYA#=8Z" + ), + ) + + +def test_hash_message_signature(sample_data, sample_signature): + assert hash_message_signature(sample_data, sample_signature) == base64.b85decode( + "Dk7mtj^WM^#_n_!-C@$EHF*O@>;mdK%XS?515>IhxRvgxdT=4.1; python_version < \"3.11\"", ] files = [ - {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, - {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, + {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, + {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, ] [[package]] @@ -364,23 +364,23 @@ files = [ [[package]] name = "boto3" -version = "1.35.38" +version = "1.35.47" requires_python = ">=3.8" summary = "The AWS SDK for Python" groups = ["dev"] dependencies = [ - "botocore<1.36.0,>=1.35.38", + "botocore<1.36.0,>=1.35.47", "jmespath<2.0.0,>=0.7.1", "s3transfer<0.11.0,>=0.10.0", ] files = [ - {file = "boto3-1.35.38-py3-none-any.whl", hash = "sha256:234a475fe56b65e99b4f5cfff50adaac6b23d39558d6b55137bbf1e50dd0ef08"}, - {file = "boto3-1.35.38.tar.gz", hash = "sha256:90c8cddc4a08c8040057ad44c7468ff82fea9fe8b6517db5ff01a9b2900299cc"}, + {file = "boto3-1.35.47-py3-none-any.whl", hash = "sha256:0b307f685875e9c7857ce21c0d3050d8d4f3778455a6852d5f98ac75194b400e"}, + {file = "boto3-1.35.47.tar.gz", hash = "sha256:65b808e4cf1af8c2f405382d53656a0d92eee8f85c7388c43d64c7a5571b065f"}, ] [[package]] name = "botocore" -version = "1.35.38" +version = "1.35.47" requires_python = ">=3.8" summary = "Low-level, data-driven core of boto 3." groups = ["dev"] @@ -391,8 +391,8 @@ dependencies = [ "urllib3<1.27,>=1.25.4; python_version < \"3.10\"", ] files = [ - {file = "botocore-1.35.38-py3-none-any.whl", hash = "sha256:2eb17d32fa2d3bb5d475132a83564d28e3acc2161534f24b75a54418a1d51359"}, - {file = "botocore-1.35.38.tar.gz", hash = "sha256:55d9305c44e5ba29476df456120fa4fb919f03f066afa82f2ae400485e7465f4"}, + {file = "botocore-1.35.47-py3-none-any.whl", hash = "sha256:05f4493119a96799ff84d43e78691efac3177e1aec8840cca99511de940e342a"}, + {file = "botocore-1.35.47.tar.gz", hash = "sha256:f8f703463d3cd8b6abe2bedc443a7ab29f0e2ff1588a2e83164b108748645547"}, ] [[package]] @@ -613,15 +613,17 @@ files = [ [[package]] name = "compute-horde" -version = "0.0.11.dev120" +version = "0.0.12.dev28" requires_python = "==3.11.*" editable = true path = "./compute_horde" summary = "" groups = ["dev"] dependencies = [ + "Django~=4.2.4", "bittensor<8.0.0,>=7.3.1", "more-itertools>=10.2.0", + "phx-class-registry>=4.1.0", "pydantic<3,>=2.3", "requests>=2.32.2", "websockets>=11.0", @@ -941,7 +943,7 @@ files = [ [[package]] name = "eth-keys" -version = "0.5.1" +version = "0.6.0" requires_python = "<4,>=3.8" summary = "eth-keys: Common API for Ethereum key operations" groups = ["dev"] @@ -950,13 +952,13 @@ dependencies = [ "eth-utils>=2", ] files = [ - {file = "eth_keys-0.5.1-py3-none-any.whl", hash = "sha256:ad13d920a2217a49bed3a1a7f54fb0980f53caf86d3bbab2139fd3330a17b97e"}, - {file = "eth_keys-0.5.1.tar.gz", hash = "sha256:2b587e4bbb9ac2195215a7ab0c0fb16042b17d4ec50240ed670bbb8f53da7a48"}, + {file = "eth_keys-0.6.0-py3-none-any.whl", hash = "sha256:b396fdfe048a5bba3ef3990739aec64901eb99901c03921caa774be668b1db6e"}, + {file = "eth_keys-0.6.0.tar.gz", hash = "sha256:ba33230f851d02c894e83989185b21d76152c49b37e35b61b1d8a6d9f1d20430"}, ] [[package]] name = "eth-typing" -version = "5.0.0" +version = "5.0.1" requires_python = "<4,>=3.8" summary = "eth-typing: Common type annotations for ethereum python packages" groups = ["dev"] @@ -964,8 +966,8 @@ dependencies = [ "typing-extensions>=4.5.0", ] files = [ - {file = "eth_typing-5.0.0-py3-none-any.whl", hash = "sha256:c7ebc8595e7b65175bb4b4176c2b548ab21b13329f2058e84d4f8c289ba9f577"}, - {file = "eth_typing-5.0.0.tar.gz", hash = "sha256:87ce7cee75665c09d2dcff8de1b496609d5e32fcd2e2b1d8fc0370c29eedcdc0"}, + {file = "eth_typing-5.0.1-py3-none-any.whl", hash = "sha256:f30d1af16aac598f216748a952eeb64fbcb6e73efa691d2de31148138afe96de"}, + {file = "eth_typing-5.0.1.tar.gz", hash = "sha256:83debf88c9df286db43bb7374974681ebcc9f048fac81be2548dbc549a3203c0"}, ] [[package]] @@ -1031,6 +1033,7 @@ dependencies = [ "flower~=2.0.0", "gunicorn==20.1.0", "httpx~=0.26.0", + "huggingface-hub>=0.26.1", "ipython~=8.14.0", "nox==2023.4.22", "prometheus-client~=0.17.0", @@ -1087,28 +1090,39 @@ files = [ [[package]] name = "frozenlist" -version = "1.4.1" +version = "1.5.0" requires_python = ">=3.8" summary = "A list-like structure which implements collections.abc.MutableSequence" groups = ["dev"] files = [ - {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, - {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, - {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, - {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, - {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, - {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, - {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, - {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, - {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5"}, + {file = "frozenlist-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45"}, + {file = "frozenlist-1.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2"}, + {file = "frozenlist-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf"}, + {file = "frozenlist-1.5.0-cp311-cp311-win32.whl", hash = "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942"}, + {file = "frozenlist-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + +[[package]] +name = "fsspec" +version = "2024.10.0" +requires_python = ">=3.8" +summary = "File-system specification" +groups = ["dev"] +files = [ + {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, + {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, ] [[package]] @@ -1182,6 +1196,26 @@ files = [ {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] +[[package]] +name = "huggingface-hub" +version = "0.26.1" +requires_python = ">=3.8.0" +summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +groups = ["dev"] +dependencies = [ + "filelock", + "fsspec>=2023.5.0", + "packaging>=20.9", + "pyyaml>=5.1", + "requests", + "tqdm>=4.42.1", + "typing-extensions>=3.7.4.3", +] +files = [ + {file = "huggingface_hub-0.26.1-py3-none-any.whl", hash = "sha256:5927a8fc64ae68859cd954b7cc29d1c8390a5e15caba6d3d349c973be8fdacf3"}, + {file = "huggingface_hub-0.26.1.tar.gz", hash = "sha256:414c0d9b769eecc86c70f9d939d0f48bb28e8461dd1130021542eff0212db890"}, +] + [[package]] name = "humanize" version = "4.11.0" @@ -1371,22 +1405,22 @@ files = [ [[package]] name = "markupsafe" -version = "3.0.1" +version = "3.0.2" requires_python = ">=3.9" summary = "Safely add untrusted strings to HTML/XML markup." groups = ["dev"] files = [ - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, - {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, - {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] [[package]] @@ -1642,6 +1676,17 @@ files = [ {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, ] +[[package]] +name = "phx-class-registry" +version = "5.1.1" +requires_python = "<4.0,>=3.11" +summary = "Factory+Registry pattern for Python classes" +groups = ["dev"] +files = [ + {file = "phx_class_registry-5.1.1-py3-none-any.whl", hash = "sha256:b093ecc1dad34c5dc6eda2530046d956f2303a5cfaa543bf7fba35ce3c7b1672"}, + {file = "phx_class_registry-5.1.1.tar.gz", hash = "sha256:06c9af198b846a7530406314f63f8d83441daf42d29ee25d8c0b19a9dbc37939"}, +] + [[package]] name = "pickleshare" version = "0.7.5" @@ -1731,24 +1776,24 @@ files = [ [[package]] name = "psycopg2-binary" -version = "2.9.9" -requires_python = ">=3.7" +version = "2.9.10" +requires_python = ">=3.8" summary = "psycopg2 - Python-PostgreSQL Database Adapter" groups = ["dev"] files = [ - {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, + {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, ] [[package]] @@ -2182,7 +2227,7 @@ files = [ [[package]] name = "rich" -version = "13.9.2" +version = "13.9.3" requires_python = ">=3.8.0" summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" groups = ["dev"] @@ -2192,8 +2237,8 @@ dependencies = [ "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", ] files = [ - {file = "rich-13.9.2-py3-none-any.whl", hash = "sha256:8c82a3d3f8dcfe9e734771313e606b39d8247bb6b826e196f4914b333b743cf1"}, - {file = "rich-13.9.2.tar.gz", hash = "sha256:51a2c62057461aaf7152b4d611168f93a9fc73068f8ded2790f29fe2b5366d0c"}, + {file = "rich-13.9.3-py3-none-any.whl", hash = "sha256:9836f5096eb2172c9e77df411c1b009bace4193d6a481d534fea75ebba758283"}, + {file = "rich-13.9.3.tar.gz", hash = "sha256:bc1e01b899537598cf02579d2b9f4a415104d3fc439313a7a2c165d76557a08e"}, ] [[package]] @@ -2535,7 +2580,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.31.1" +version = "0.32.0" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["dev"] @@ -2545,24 +2590,24 @@ dependencies = [ "typing-extensions>=4.0; python_version < \"3.11\"", ] files = [ - {file = "uvicorn-0.31.1-py3-none-any.whl", hash = "sha256:adc42d9cac80cf3e51af97c1851648066841e7cfb6993a4ca8de29ac1548ed41"}, - {file = "uvicorn-0.31.1.tar.gz", hash = "sha256:f5167919867b161b7bcaf32646c6a94cdbd4c3aa2eb5c17d36bb9aa5cfd8c493"}, + {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, + {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, ] [[package]] name = "uvloop" -version = "0.20.0" +version = "0.21.0" requires_python = ">=3.8.0" summary = "Fast implementation of asyncio event loop on top of libuv" groups = ["dev"] files = [ - {file = "uvloop-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e50289c101495e0d1bb0bfcb4a60adde56e32f4449a67216a1ab2750aa84f037"}, - {file = "uvloop-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e237f9c1e8a00e7d9ddaa288e535dc337a39bcbf679f290aee9d26df9e72bce9"}, - {file = "uvloop-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:746242cd703dc2b37f9d8b9f173749c15e9a918ddb021575a0205ec29a38d31e"}, - {file = "uvloop-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82edbfd3df39fb3d108fc079ebc461330f7c2e33dbd002d146bf7c445ba6e756"}, - {file = "uvloop-0.20.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80dc1b139516be2077b3e57ce1cb65bfed09149e1d175e0478e7a987863b68f0"}, - {file = "uvloop-0.20.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f44af67bf39af25db4c1ac27e82e9665717f9c26af2369c404be865c8818dcf"}, - {file = "uvloop-0.20.0.tar.gz", hash = "sha256:4603ca714a754fc8d9b197e325db25b2ea045385e8a3ad05d3463de725fdf469"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d"}, + {file = "uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3"}, ] [[package]] @@ -2616,8 +2661,8 @@ files = [ [[package]] name = "virtualenv" -version = "20.26.6" -requires_python = ">=3.7" +version = "20.27.0" +requires_python = ">=3.8" summary = "Virtual Python Environment builder" groups = ["dev"] dependencies = [ @@ -2627,8 +2672,8 @@ dependencies = [ "platformdirs<5,>=3.9.1", ] files = [ - {file = "virtualenv-20.26.6-py3-none-any.whl", hash = "sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2"}, - {file = "virtualenv-20.26.6.tar.gz", hash = "sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48"}, + {file = "virtualenv-20.27.0-py3-none-any.whl", hash = "sha256:44a72c29cceb0ee08f300b314848c86e57bf8d1f13107a5e671fb9274138d655"}, + {file = "virtualenv-20.27.0.tar.gz", hash = "sha256:2ca56a68ed615b8fe4326d11a0dca5dfbe8fd68510fb6c6349163bed3c15f2b2"}, ] [[package]] @@ -2715,8 +2760,8 @@ files = [ [[package]] name = "yarl" -version = "1.14.0" -requires_python = ">=3.8" +version = "1.16.0" +requires_python = ">=3.9" summary = "Yet another URL library" groups = ["dev"] dependencies = [ @@ -2725,28 +2770,29 @@ dependencies = [ "propcache>=0.2.0", ] files = [ - {file = "yarl-1.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:07f9eaf57719d6721ab15805d85f4b01a5b509a0868d7320134371bcb652152d"}, - {file = "yarl-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c14b504a74e58e2deb0378b3eca10f3d076635c100f45b113c18c770b4a47a50"}, - {file = "yarl-1.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:16a682a127930f3fc4e42583becca6049e1d7214bcad23520c590edd741d2114"}, - {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73bedd2be05f48af19f0f2e9e1353921ce0c83f4a1c9e8556ecdcf1f1eae4892"}, - {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3ab950f8814f3b7b5e3eebc117986f817ec933676f68f0a6c5b2137dd7c9c69"}, - {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b693c63e7e64b524f54aa4888403c680342d1ad0d97be1707c531584d6aeeb4f"}, - {file = "yarl-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85cb3e40eaa98489f1e2e8b29f5ad02ee1ee40d6ce6b88d50cf0f205de1d9d2c"}, - {file = "yarl-1.14.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f24f08b6c9b9818fd80612c97857d28f9779f0d1211653ece9844fc7b414df2"}, - {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29a84a46ec3ebae7a1c024c055612b11e9363a8a23238b3e905552d77a2bc51b"}, - {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5cd5dad8366e0168e0fd23d10705a603790484a6dbb9eb272b33673b8f2cce72"}, - {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a152751af7ef7b5d5fa6d215756e508dd05eb07d0cf2ba51f3e740076aa74373"}, - {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3d569f877ed9a708e4c71a2d13d2940cb0791da309f70bd970ac1a5c088a0a92"}, - {file = "yarl-1.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a615cad11ec3428020fb3c5a88d85ce1b5c69fd66e9fcb91a7daa5e855325dd"}, - {file = "yarl-1.14.0-cp311-cp311-win32.whl", hash = "sha256:bab03192091681d54e8225c53f270b0517637915d9297028409a2a5114ff4634"}, - {file = "yarl-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:985623575e5c4ea763056ffe0e2d63836f771a8c294b3de06d09480538316b13"}, - {file = "yarl-1.14.0-py3-none-any.whl", hash = "sha256:c8ed4034f0765f8861620c1f2f2364d2e58520ea288497084dae880424fc0d9f"}, - {file = "yarl-1.14.0.tar.gz", hash = "sha256:88c7d9d58aab0724b979ab5617330acb1c7030b79379c8138c1c8c94e121d1b3"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d8643975a0080f361639787415a038bfc32d29208a4bf6b783ab3075a20b1ef3"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:676d96bafc8c2d0039cea0cd3fd44cee7aa88b8185551a2bb93354668e8315c2"}, + {file = "yarl-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d9525f03269e64310416dbe6c68d3b23e5d34aaa8f47193a1c45ac568cecbc49"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b37d5ec034e668b22cf0ce1074d6c21fd2a08b90d11b1b73139b750a8b0dd97"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f32c4cb7386b41936894685f6e093c8dfaf0960124d91fe0ec29fe439e201d0"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b8e265a0545637492a7e12fd7038370d66c9375a61d88c5567d0e044ded9202"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:789a3423f28a5fff46fbd04e339863c169ece97c827b44de16e1a7a42bc915d2"}, + {file = "yarl-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1d1f45e3e8d37c804dca99ab3cf4ab3ed2e7a62cd82542924b14c0a4f46d243"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:621280719c4c5dad4c1391160a9b88925bb8b0ff6a7d5af3224643024871675f"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ed097b26f18a1f5ff05f661dc36528c5f6735ba4ce8c9645e83b064665131349"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:2f1fe2b2e3ee418862f5ebc0c0083c97f6f6625781382f828f6d4e9b614eba9b"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:87dd10bc0618991c66cee0cc65fa74a45f4ecb13bceec3c62d78ad2e42b27a16"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4199db024b58a8abb2cfcedac7b1292c3ad421684571aeb622a02f242280e8d6"}, + {file = "yarl-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:99a9dcd4b71dd5f5f949737ab3f356cfc058c709b4f49833aeffedc2652dac56"}, + {file = "yarl-1.16.0-cp311-cp311-win32.whl", hash = "sha256:a9394c65ae0ed95679717d391c862dece9afacd8fa311683fc8b4362ce8a410c"}, + {file = "yarl-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b9101f528ae0f8f65ac9d64dda2bb0627de8a50344b2f582779f32fda747c1d"}, + {file = "yarl-1.16.0-py3-none-any.whl", hash = "sha256:e6980a558d8461230c457218bd6c92dfc1d10205548215c2c21d79dc8d0a96f3"}, + {file = "yarl-1.16.0.tar.gz", hash = "sha256:b6f687ced5510a9a2474bbae96a4352e5ace5fa34dc44a217b0537fec1db00b4"}, ] [[package]] name = "zope-interface" -version = "7.1.0" +version = "7.1.1" requires_python = ">=3.8" summary = "Interfaces for Python" groups = ["dev"] @@ -2754,11 +2800,11 @@ dependencies = [ "setuptools", ] files = [ - {file = "zope.interface-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ac20581fc6cd7c754f6dff0ae06fedb060fa0e9ea6309d8be8b2701d9ea51c4"}, - {file = "zope.interface-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:848b6fa92d7c8143646e64124ed46818a0049a24ecc517958c520081fd147685"}, - {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec1ef1fdb6f014d5886b97e52b16d0f852364f447d2ab0f0c6027765777b6667"}, - {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bcff5c09d0215f42ba64b49205a278e44413d9bf9fa688fd9e42bfe472b5f4f"}, - {file = "zope.interface-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07add15de0cc7e69917f7d286b64d54125c950aeb43efed7a5ea7172f000fbc1"}, - {file = "zope.interface-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:9940d5bc441f887c5f375ec62bcf7e7e495a2d5b1da97de1184a88fb567f06af"}, - {file = "zope_interface-7.1.0.tar.gz", hash = "sha256:3f005869a1a05e368965adb2075f97f8ee9a26c61898a9e52a9764d93774f237"}, + {file = "zope.interface-7.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3e59f175e868f856a77c0a77ba001385c377df2104fdbda6b9f99456a01e102a"}, + {file = "zope.interface-7.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0de23bcb93401994ea00bc5c677ef06d420340ac0a4e9c10d80e047b9ce5af3f"}, + {file = "zope.interface-7.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdb7e7e5524b76d3ec037c1d81a9e2c7457b240fd4cb0a2476b65c3a5a6c81f"}, + {file = "zope.interface-7.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3603ef82a9920bd0bfb505423cb7e937498ad971ad5a6141841e8f76d2fd5446"}, + {file = "zope.interface-7.1.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1d52d052355e0c5c89e0630dd2ff7c0b823fd5f56286a663e92444761b35e25"}, + {file = "zope.interface-7.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:179ad46ece518c9084cb272e4a69d266b659f7f8f48e51706746c2d8a426433e"}, + {file = "zope.interface-7.1.1.tar.gz", hash = "sha256:4284d664ef0ff7b709836d4de7b13d80873dc5faeffc073abdb280058bfac5e3"}, ] From 5d70f0aaca421ec8090a094a2d4ee80eb81603c7 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 22:10:17 +0300 Subject: [PATCH 81/99] added fv_protocol package --- .../compute_horde/fv_protocol/__init__.py | 0 .../fv_protocol/facilitator_requests.py | 96 +++++++++++++++++++ .../fv_protocol/validator_requests.py | 35 +++++++ 3 files changed, 131 insertions(+) create mode 100644 compute_horde/compute_horde/fv_protocol/__init__.py create mode 100644 compute_horde/compute_horde/fv_protocol/facilitator_requests.py create mode 100644 compute_horde/compute_horde/fv_protocol/validator_requests.py diff --git a/compute_horde/compute_horde/fv_protocol/__init__.py b/compute_horde/compute_horde/fv_protocol/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py new file mode 100644 index 000000000..c99508cf8 --- /dev/null +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -0,0 +1,96 @@ +from typing import Annotated, Literal, Self + +import pydantic +from pydantic import BaseModel, model_validator + +from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload +from compute_horde.base.volume import Volume, ZipUrlVolume +from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass + + +class Error(BaseModel, extra="allow"): + msg: str + type: str + help: str = "" + + +class Response(BaseModel, extra="forbid"): + """Message sent from facilitator to validator in response to AuthenticationRequest & JobStatusUpdate""" + + status: Literal["error", "success"] + errors: list[Error] = [] + + +class V0FacilitatorJobRequest(BaseModel, extra="forbid"): + """Message sent from facilitator to validator to request a job execution""" + + # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) + type: Literal["job.new"] = "job.new" + message_type: Literal["V0JobRequest"] = "V0JobRequest" + + uuid: str + miner_hotkey: str + # TODO: remove default after we add executor class support to facilitator + executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS + docker_image: str + raw_script: str + args: list[str] + env: dict[str, str] + use_gpu: bool + input_url: str + output_url: str + + def get_args(self): + return self.args + + @model_validator(mode="after") + def validate_at_least_docker_image_or_raw_script(self) -> Self: + if not (bool(self.docker_image) or bool(self.raw_script)): + raise ValueError("Expected at least one of `docker_image` or `raw_script`") + return self + + @property + def volume(self) -> Volume | None: + if self.input_url: + return ZipUrlVolume(contents=self.input_url) + return None + + @property + def output_upload(self) -> OutputUpload | None: + if self.output_url: + return ZipAndHttpPutUpload(url=self.output_url) + return None + + +class V1FacilitatorJobRequest(BaseModel, extra="forbid"): + """Message sent from facilitator to validator to request a job execution""" + + # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) + type: Literal["job.new"] = "job.new" + message_type: Literal["V1JobRequest"] = "V1JobRequest" + uuid: str + miner_hotkey: str + # TODO: remove default after we add executor class support to facilitator + executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS + docker_image: str + raw_script: str + args: list[str] + env: dict[str, str] + use_gpu: bool + volume: Volume | None = None + output_upload: OutputUpload | None = None + + def get_args(self): + return self.args + + @model_validator(mode="after") + def validate_at_least_docker_image_or_raw_script(self) -> Self: + if not (bool(self.docker_image) or bool(self.raw_script)): + raise ValueError("Expected at least one of `docker_image` or `raw_script`") + return self + + +JobRequest = Annotated[ + V0FacilitatorJobRequest | V1FacilitatorJobRequest, + pydantic.Field(discriminator="message_type"), +] diff --git a/compute_horde/compute_horde/fv_protocol/validator_requests.py b/compute_horde/compute_horde/fv_protocol/validator_requests.py new file mode 100644 index 000000000..5bad44219 --- /dev/null +++ b/compute_horde/compute_horde/fv_protocol/validator_requests.py @@ -0,0 +1,35 @@ +from typing import Any, Self + +import bittensor +from pydantic import BaseModel + + +class Heartbeat(BaseModel, extra="forbid"): + """Message sent from validator to facilitator to keep connection alive""" + + message_type: str = "V0Heartbeat" + + +class AuthenticationRequest(BaseModel, extra="forbid"): + """Message sent from validator to facilitator to authenticate itself""" + + message_type: str = "V0AuthenticationRequest" + public_key: str + signature: str + + @classmethod + def from_keypair(cls, keypair: bittensor.Keypair) -> Self: + return cls( + public_key=keypair.public_key.hex(), + signature=f"0x{keypair.sign(keypair.public_key).hex()}", + ) + + +class MachineSpecsUpdate(BaseModel, extra="forbid"): + """Message sent from validator to facilitator to update miner specs""" + + message_type: str = "V0MachineSpecsUpdate" + miner_hotkey: str + validator_hotkey: str + specs: dict[str, Any] + batch_id: str From f2d31f38db3cbbb81b772f45745eb4aa2d971026 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 22:10:53 +0300 Subject: [PATCH 82/99] replaced facilitator_api with fv_protocol --- .../validator/organic_jobs/facilitator_api.py | 123 ------------------ .../organic_jobs/facilitator_client.py | 14 +- .../validator/organic_jobs/miner_driver.py | 8 +- .../validator/tests/helpers.py | 8 +- .../tests/test_facilitator_client.py | 9 +- 5 files changed, 17 insertions(+), 145 deletions(-) delete mode 100644 validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py deleted file mode 100644 index fa2028e07..000000000 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_api.py +++ /dev/null @@ -1,123 +0,0 @@ -from typing import Annotated, Any, Literal, Self - -import bittensor -import pydantic -from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload -from compute_horde.base.volume import Volume, ZipUrlVolume -from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass -from pydantic import BaseModel, model_validator - - -class Error(BaseModel, extra="allow"): - msg: str - type: str - help: str = "" - - -class Response(BaseModel, extra="forbid"): - """Message sent from facilitator to validator in response to AuthenticationRequest & JobStatusUpdate""" - - status: Literal["error", "success"] - errors: list[Error] = [] - - -class AuthenticationRequest(BaseModel, extra="forbid"): - """Message sent from validator to facilitator to authenticate itself""" - - message_type: str = "V0AuthenticationRequest" - public_key: str - signature: str - - @classmethod - def from_keypair(cls, keypair: bittensor.Keypair) -> Self: - return cls( - public_key=keypair.public_key.hex(), - signature=f"0x{keypair.sign(keypair.public_key).hex()}", - ) - - -class V0FacilitatorJobRequest(BaseModel, extra="forbid"): - """Message sent from facilitator to validator to request a job execution""" - - # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) - type: Literal["job.new"] = "job.new" - message_type: Literal["V0JobRequest"] = "V0JobRequest" - - uuid: str - miner_hotkey: str - # TODO: remove default after we add executor class support to facilitator - executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS - docker_image: str - raw_script: str - args: list[str] - env: dict[str, str] - use_gpu: bool - input_url: str - output_url: str - - def get_args(self): - return self.args - - @model_validator(mode="after") - def validate_at_least_docker_image_or_raw_script(self) -> Self: - if not (bool(self.docker_image) or bool(self.raw_script)): - raise ValueError("Expected at least one of `docker_image` or `raw_script`") - return self - - @property - def volume(self) -> Volume | None: - if self.input_url: - return ZipUrlVolume(contents=self.input_url) - return None - - @property - def output_upload(self) -> OutputUpload | None: - if self.output_url: - return ZipAndHttpPutUpload(url=self.output_url) - return None - - -class V1FacilitatorJobRequest(BaseModel, extra="forbid"): - """Message sent from facilitator to validator to request a job execution""" - - # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) - type: Literal["job.new"] = "job.new" - message_type: Literal["V1JobRequest"] = "V1JobRequest" - uuid: str - miner_hotkey: str - # TODO: remove default after we add executor class support to facilitator - executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS - docker_image: str - raw_script: str - args: list[str] - env: dict[str, str] - use_gpu: bool - volume: Volume | None = None - output_upload: OutputUpload | None = None - - def get_args(self): - return self.args - - @model_validator(mode="after") - def validate_at_least_docker_image_or_raw_script(self) -> Self: - if not (bool(self.docker_image) or bool(self.raw_script)): - raise ValueError("Expected at least one of `docker_image` or `raw_script`") - return self - - -JobRequest = Annotated[ - V0FacilitatorJobRequest | V1FacilitatorJobRequest, - pydantic.Field(discriminator="message_type"), -] - - -class Heartbeat(BaseModel, extra="forbid"): - message_type: str = "V0Heartbeat" - - -class MachineSpecsUpdate(BaseModel, extra="forbid"): - message_type: str = "V0MachineSpecsUpdate" - miner_hotkey: str - validator_hotkey: str - specs: dict[str, Any] - batch_id: str diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py index d620ed3b3..15ff6755d 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py @@ -8,6 +8,12 @@ import tenacity import websockets from channels.layers import get_channel_layer +from compute_horde.fv_protocol.facilitator_requests import Error, JobRequest, Response +from compute_horde.fv_protocol.validator_requests import ( + AuthenticationRequest, + Heartbeat, + MachineSpecsUpdate, +) from django.conf import settings from pydantic import BaseModel @@ -16,14 +22,6 @@ get_miner_axon_info, ) from compute_horde_validator.validator.models import Miner, OrganicJob, SystemEvent -from compute_horde_validator.validator.organic_jobs.facilitator_api import ( - AuthenticationRequest, - Error, - Heartbeat, - JobRequest, - MachineSpecsUpdate, - Response, -) from compute_horde_validator.validator.organic_jobs.miner_client import MinerClient from compute_horde_validator.validator.organic_jobs.miner_driver import execute_organic_job from compute_horde_validator.validator.utils import MACHINE_SPEC_CHANNEL diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index a4d9057c0..475b61488 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -4,6 +4,10 @@ from typing import Literal from compute_horde.executor_class import ExecutorClass +from compute_horde.fv_protocol.facilitator_requests import ( + V0FacilitatorJobRequest, + V1FacilitatorJobRequest, +) from compute_horde.miner_client.organic import ( FailureReason, OrganicJobDetails, @@ -20,10 +24,6 @@ OrganicJob, SystemEvent, ) -from compute_horde_validator.validator.organic_jobs.facilitator_api import ( - V0FacilitatorJobRequest, - V1FacilitatorJobRequest, -) from compute_horde_validator.validator.organic_jobs.miner_client import MinerClient logger = logging.getLogger(__name__) diff --git a/validator/app/src/compute_horde_validator/validator/tests/helpers.py b/validator/app/src/compute_horde_validator/validator/tests/helpers.py index 6ec366dcd..179234d9f 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/helpers.py +++ b/validator/app/src/compute_horde_validator/validator/tests/helpers.py @@ -14,6 +14,10 @@ import numpy as np from bittensor import Balance from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS +from compute_horde.fv_protocol.facilitator_requests import ( + V0FacilitatorJobRequest, + V1FacilitatorJobRequest, +) from compute_horde.mv_protocol.miner_requests import ( V0AcceptJobRequest, V0ExecutorReadyRequest, @@ -24,10 +28,6 @@ from substrateinterface.exceptions import SubstrateRequestException from compute_horde_validator.validator.models import SystemEvent -from compute_horde_validator.validator.organic_jobs.facilitator_api import ( - V0FacilitatorJobRequest, - V1FacilitatorJobRequest, -) from compute_horde_validator.validator.organic_jobs.miner_client import MinerClient from compute_horde_validator.validator.synthetic_jobs import batch_run diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py index 9b7175837..4e6bfdde8 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py @@ -6,14 +6,11 @@ import pytest import websockets from channels.layers import get_channel_layer +from compute_horde.fv_protocol.facilitator_requests import Response +from compute_horde.fv_protocol.validator_requests import AuthenticationRequest, MachineSpecsUpdate from compute_horde_validator.validator.models import OrganicJob -from compute_horde_validator.validator.organic_jobs.facilitator_api import MachineSpecsUpdate -from compute_horde_validator.validator.organic_jobs.facilitator_client import ( - AuthenticationRequest, - FacilitatorClient, - Response, -) +from compute_horde_validator.validator.organic_jobs.facilitator_client import FacilitatorClient from compute_horde_validator.validator.organic_jobs.miner_driver import JobStatusUpdate from compute_horde_validator.validator.utils import MACHINE_SPEC_CHANNEL From 9cd26601de9c3ddfb122d11434709963432bc397 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 22:28:56 +0300 Subject: [PATCH 83/99] renamed facilitator protocol message classes --- .../fv_protocol/facilitator_requests.py | 6 +++--- .../fv_protocol/validator_requests.py | 6 +++--- docs/facilitator-protocol.md | 2 +- .../validator/organic_jobs/facilitator_client.py | 16 ++++++++-------- .../validator/organic_jobs/miner_driver.py | 6 +++--- .../validator/tests/helpers.py | 12 ++++++------ .../validator/tests/test_facilitator_client.py | 11 +++++++---- 7 files changed, 31 insertions(+), 28 deletions(-) diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py index c99508cf8..8e800a4db 100644 --- a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -21,7 +21,7 @@ class Response(BaseModel, extra="forbid"): errors: list[Error] = [] -class V0FacilitatorJobRequest(BaseModel, extra="forbid"): +class V0JobRequest(BaseModel, extra="forbid"): """Message sent from facilitator to validator to request a job execution""" # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) @@ -62,7 +62,7 @@ def output_upload(self) -> OutputUpload | None: return None -class V1FacilitatorJobRequest(BaseModel, extra="forbid"): +class V1JobRequest(BaseModel, extra="forbid"): """Message sent from facilitator to validator to request a job execution""" # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) @@ -91,6 +91,6 @@ def validate_at_least_docker_image_or_raw_script(self) -> Self: JobRequest = Annotated[ - V0FacilitatorJobRequest | V1FacilitatorJobRequest, + V0JobRequest | V1JobRequest, pydantic.Field(discriminator="message_type"), ] diff --git a/compute_horde/compute_horde/fv_protocol/validator_requests.py b/compute_horde/compute_horde/fv_protocol/validator_requests.py index 5bad44219..b5d4accda 100644 --- a/compute_horde/compute_horde/fv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/validator_requests.py @@ -4,13 +4,13 @@ from pydantic import BaseModel -class Heartbeat(BaseModel, extra="forbid"): +class V0Heartbeat(BaseModel, extra="forbid"): """Message sent from validator to facilitator to keep connection alive""" message_type: str = "V0Heartbeat" -class AuthenticationRequest(BaseModel, extra="forbid"): +class V0AuthenticationRequest(BaseModel, extra="forbid"): """Message sent from validator to facilitator to authenticate itself""" message_type: str = "V0AuthenticationRequest" @@ -25,7 +25,7 @@ def from_keypair(cls, keypair: bittensor.Keypair) -> Self: ) -class MachineSpecsUpdate(BaseModel, extra="forbid"): +class V0MachineSpecsUpdate(BaseModel, extra="forbid"): """Message sent from validator to facilitator to update miner specs""" message_type: str = "V0MachineSpecsUpdate" diff --git a/docs/facilitator-protocol.md b/docs/facilitator-protocol.md index ce0eea17b..ebf38a49d 100644 --- a/docs/facilitator-protocol.md +++ b/docs/facilitator-protocol.md @@ -43,7 +43,7 @@ sequenceDiagram end validator->>facilitator: V0Heartbeat - validator->>facilitator: MachineSpecsUpdate + validator->>facilitator: V0MachineSpecsUpdate ``` ## `V0AuthenticationRequest` message diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py index 15ff6755d..9b41e11c3 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py @@ -10,9 +10,9 @@ from channels.layers import get_channel_layer from compute_horde.fv_protocol.facilitator_requests import Error, JobRequest, Response from compute_horde.fv_protocol.validator_requests import ( - AuthenticationRequest, - Heartbeat, - MachineSpecsUpdate, + V0AuthenticationRequest, + V0Heartbeat, + V0MachineSpecsUpdate, ) from django.conf import settings from pydantic import BaseModel @@ -128,14 +128,14 @@ async def run_forever(self): async def handle_connection(self, ws: websockets.WebSocketClientProtocol): """handle a single websocket connection""" - await ws.send(AuthenticationRequest.from_keypair(self.keypair).model_dump_json()) + await ws.send(V0AuthenticationRequest.from_keypair(self.keypair).model_dump_json()) raw_msg = await ws.recv() try: response = Response.model_validate_json(raw_msg) except pydantic.ValidationError as exc: raise AuthenticationError( - "did not receive Response for AuthenticationRequest", [] + "did not receive Response for V0AuthenticationRequest", [] ) from exc if response.status != "success": raise AuthenticationError("auth request received failed response", response.errors) @@ -146,7 +146,7 @@ async def handle_connection(self, ws: websockets.WebSocketClientProtocol): await self.handle_message(raw_msg) async def wait_for_specs(self): - specs_queue: deque[MachineSpecsUpdate] = deque() + specs_queue: deque[V0MachineSpecsUpdate] = deque() channel_layer = get_channel_layer() while True: @@ -156,7 +156,7 @@ async def wait_for_specs(self): channel_layer.receive(MACHINE_SPEC_CHANNEL), timeout=20 * 60 ) - specs = MachineSpecsUpdate( + specs = V0MachineSpecsUpdate( specs=msg["specs"], miner_hotkey=msg["miner_hotkey"], batch_id=msg["batch_id"], @@ -190,7 +190,7 @@ async def heartbeat(self): while True: if self.ws is not None: try: - await self.send_model(Heartbeat()) + await self.send_model(V0Heartbeat()) except Exception as exc: msg = f"Error occurred while sending heartbeat: {exc}" logger.warning(msg) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 475b61488..6cb212eb6 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -5,8 +5,8 @@ from compute_horde.executor_class import ExecutorClass from compute_horde.fv_protocol.facilitator_requests import ( - V0FacilitatorJobRequest, - V1FacilitatorJobRequest, + V0JobRequest, + V1JobRequest, ) from compute_horde.miner_client.organic import ( FailureReason, @@ -93,7 +93,7 @@ async def _dummy_notify_callback(_: JobStatusUpdate) -> None: async def execute_organic_job( miner_client: MinerClient, job: OrganicJob, - job_request: V0FacilitatorJobRequest | V1FacilitatorJobRequest | AdminJobRequest, + job_request: V0JobRequest | V1JobRequest | AdminJobRequest, total_job_timeout: int = 300, wait_timeout: int = 300, notify_callback: Callable[[JobStatusUpdate], Awaitable[None]] = _dummy_notify_callback, diff --git a/validator/app/src/compute_horde_validator/validator/tests/helpers.py b/validator/app/src/compute_horde_validator/validator/tests/helpers.py index 179234d9f..ed5588fe7 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/helpers.py +++ b/validator/app/src/compute_horde_validator/validator/tests/helpers.py @@ -15,8 +15,8 @@ from bittensor import Balance from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS from compute_horde.fv_protocol.facilitator_requests import ( - V0FacilitatorJobRequest, - V1FacilitatorJobRequest, + V0JobRequest, + V1JobRequest, ) from compute_horde.mv_protocol.miner_requests import ( V0AcceptJobRequest, @@ -135,8 +135,8 @@ def __init__(self, *args, **kwargs): ) -def get_dummy_job_request_v0(uuid: str) -> V0FacilitatorJobRequest: - return V0FacilitatorJobRequest( +def get_dummy_job_request_v0(uuid: str) -> V0JobRequest: + return V0JobRequest( type="job.new", uuid=uuid, miner_hotkey="miner_hotkey", @@ -151,8 +151,8 @@ def get_dummy_job_request_v0(uuid: str) -> V0FacilitatorJobRequest: ) -def get_dummy_job_request_v1(uuid: str) -> V1FacilitatorJobRequest: - return V1FacilitatorJobRequest( +def get_dummy_job_request_v1(uuid: str) -> V1JobRequest: + return V1JobRequest( type="job.new", uuid=uuid, miner_hotkey="miner_hotkey", diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py index 4e6bfdde8..d4a1bc1c5 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_facilitator_client.py @@ -7,7 +7,10 @@ import websockets from channels.layers import get_channel_layer from compute_horde.fv_protocol.facilitator_requests import Response -from compute_horde.fv_protocol.validator_requests import AuthenticationRequest, MachineSpecsUpdate +from compute_horde.fv_protocol.validator_requests import ( + V0AuthenticationRequest, + V0MachineSpecsUpdate, +) from compute_horde_validator.validator.models import OrganicJob from compute_horde_validator.validator.organic_jobs.facilitator_client import FacilitatorClient @@ -57,7 +60,7 @@ async def serve(self, ws): # auth response = await asyncio.wait_for(ws.recv(), timeout=5) try: - AuthenticationRequest.model_validate_json(response) + V0AuthenticationRequest.model_validate_json(response) except Exception as e: self.facilitator_error = e @@ -167,7 +170,7 @@ class FacilitatorExpectMachineSpecsWs(FacilitatorWs): async def serve(self, ws, path): response = await asyncio.wait_for(ws.recv(), timeout=5) try: - AuthenticationRequest.model_validate_json(response) + V0AuthenticationRequest.model_validate_json(response) except Exception as e: self.facilitator_error = e @@ -175,7 +178,7 @@ async def serve(self, ws, path): async for message in ws: try: - MachineSpecsUpdate.model_validate_json(message) + V0MachineSpecsUpdate.model_validate_json(message) except Exception: continue else: From eff12ab29fa42610234c5244e15dd9c34c5a2201 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 22:46:56 +0300 Subject: [PATCH 84/99] copied methods from facilitator --- .../fv_protocol/validator_requests.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/compute_horde/compute_horde/fv_protocol/validator_requests.py b/compute_horde/compute_horde/fv_protocol/validator_requests.py index b5d4accda..c57f8385c 100644 --- a/compute_horde/compute_horde/fv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/validator_requests.py @@ -24,6 +24,21 @@ def from_keypair(cls, keypair: bittensor.Keypair) -> Self: signature=f"0x{keypair.sign(keypair.public_key).hex()}", ) + def verify_signature(self) -> bool: + public_key_bytes = bytes.fromhex(self.public_key) + keypair = bittensor.Keypair(public_key=public_key_bytes, ss58_format=42) + # make mypy happy + valid: bool = keypair.verify(public_key_bytes, self.signature) + return valid + + @property + def ss58_address(self) -> str: + # make mypy happy + address: str = bittensor.Keypair( + public_key=bytes.fromhex(self.public_key), ss58_format=42 + ).ss58_address + return address + class V0MachineSpecsUpdate(BaseModel, extra="forbid"): """Message sent from validator to facilitator to update miner specs""" From 8e64470163cd732b2e334de5f49b17d3bee1abe4 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Wed, 23 Oct 2024 23:09:20 +0300 Subject: [PATCH 85/99] removed obsolete defaults --- .../compute_horde/fv_protocol/facilitator_requests.py | 8 +++----- .../compute_horde_validator/validator/tests/helpers.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py index 8e800a4db..5ef911973 100644 --- a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -5,7 +5,7 @@ from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload from compute_horde.base.volume import Volume, ZipUrlVolume -from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS, ExecutorClass +from compute_horde.executor_class import ExecutorClass class Error(BaseModel, extra="allow"): @@ -30,8 +30,7 @@ class V0JobRequest(BaseModel, extra="forbid"): uuid: str miner_hotkey: str - # TODO: remove default after we add executor class support to facilitator - executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS + executor_class: ExecutorClass docker_image: str raw_script: str args: list[str] @@ -70,8 +69,7 @@ class V1JobRequest(BaseModel, extra="forbid"): message_type: Literal["V1JobRequest"] = "V1JobRequest" uuid: str miner_hotkey: str - # TODO: remove default after we add executor class support to facilitator - executor_class: ExecutorClass = DEFAULT_EXECUTOR_CLASS + executor_class: ExecutorClass docker_image: str raw_script: str args: list[str] diff --git a/validator/app/src/compute_horde_validator/validator/tests/helpers.py b/validator/app/src/compute_horde_validator/validator/tests/helpers.py index ed5588fe7..874b49f47 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/helpers.py +++ b/validator/app/src/compute_horde_validator/validator/tests/helpers.py @@ -156,6 +156,7 @@ def get_dummy_job_request_v1(uuid: str) -> V1JobRequest: type="job.new", uuid=uuid, miner_hotkey="miner_hotkey", + executor_class=DEFAULT_EXECUTOR_CLASS, docker_image="nvidia", raw_script="print('hello world')", args=[], From b7f01baf7bc0d75a0529968031225161bc1669ad Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 24 Oct 2024 10:39:28 +0300 Subject: [PATCH 86/99] added V2JobRequest --- .../fv_protocol/facilitator_requests.py | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py index 5ef911973..d4a07eca0 100644 --- a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -1,7 +1,7 @@ from typing import Annotated, Literal, Self import pydantic -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, JsonValue, model_validator from compute_horde.base.output_upload import OutputUpload, ZipAndHttpPutUpload from compute_horde.base.volume import Volume, ZipUrlVolume @@ -21,6 +21,14 @@ class Response(BaseModel, extra="forbid"): errors: list[Error] = [] +class SignedRequest(BaseModel, extra="forbid"): + signature_type: str + signatory: str + timestamp_ns: int + signature: str + signed_payload: JsonValue + + class V0JobRequest(BaseModel, extra="forbid"): """Message sent from facilitator to validator to request a job execution""" @@ -88,6 +96,34 @@ def validate_at_least_docker_image_or_raw_script(self) -> Self: return self +class V2JobRequest(BaseModel, extra="forbid"): + """Message sent from facilitator to validator to request a job execution""" + + # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) + type: Literal["job.new"] = "job.new" + message_type: Literal["V2JobRequest"] = "V2JobRequest" + uuid: str + miner_hotkey: str | None + executor_class: ExecutorClass + docker_image: str + raw_script: str + args: list[str] + env: dict[str, str] + use_gpu: bool + volume: Volume | None = None + output_upload: OutputUpload | None = None + signed_request: SignedRequest + + def get_args(self): + return self.args + + @model_validator(mode="after") + def validate_at_least_docker_image_or_raw_script(self) -> Self: + if not (bool(self.docker_image) or bool(self.raw_script)): + raise ValueError("Expected at least one of `docker_image` or `raw_script`") + return self + + JobRequest = Annotated[ V0JobRequest | V1JobRequest, pydantic.Field(discriminator="message_type"), From fb0dc141fa54c07c77077a3b2b6e869e14d0b963 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 24 Oct 2024 10:46:06 +0300 Subject: [PATCH 87/99] use pydantic.JsonValue --- compute_horde/compute_horde/signature.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/compute_horde/compute_horde/signature.py b/compute_horde/compute_horde/signature.py index f3d7134d9..0a496d9f5 100644 --- a/compute_horde/compute_horde/signature.py +++ b/compute_horde/compute_horde/signature.py @@ -12,15 +12,11 @@ from typing import ClassVar, Protocol from class_registry import ClassRegistry, RegistryKeyError +from pydantic import JsonValue if typing.TYPE_CHECKING: import bittensor -JSONValue = str | int | float | bool | None -JSONDict = dict[str, "JSONType"] -JSONArray = list["JSONType"] -JSONType = JSONValue | JSONDict | JSONArray - SIGNERS_REGISTRY: ClassRegistry[Signer] = ClassRegistry("signature_type") VERIFIERS_REGISTRY: ClassRegistry[Verifier] = ClassRegistry("signature_type") @@ -34,7 +30,7 @@ class Signature: def verify_signature( - payload: JSONType | bytes, + payload: JsonValue | bytes, signature: Signature, *, newer_than: datetime.datetime | None = None, @@ -87,7 +83,7 @@ def verify_request( method: str, url: str, headers: dict[str, str], - json: JSONType | None = None, + json: JsonValue | None = None, *, newer_than: datetime.datetime | None = None, signature_extractor: SignatureExtractor = signature_from_headers, @@ -150,7 +146,7 @@ class SignatureTimeoutException(SignatureInvalidException): pass -def hash_message_signature(payload: bytes | JSONType, signature: Signature) -> bytes: +def hash_message_signature(payload: bytes | JsonValue, signature: Signature) -> bytes: """ Hashes the message to be signed with the signature parameters @@ -171,8 +167,8 @@ def hash_message_signature(payload: bytes | JSONType, signature: Signature) -> b def signature_payload( - method: str, url: str, headers: dict[str, str], json: JSONType | None = None -) -> JSONType: + method: str, url: str, headers: dict[str, str], json: JsonValue | None = None +) -> JsonValue: reduced_url = _REMOVE_URL_SCHEME_N_HOST_RE.sub("", url) return { "action": f"{method.upper()} {reduced_url}", @@ -188,7 +184,7 @@ def payload_from_request( method: str, url: str, headers: dict[str, str], - json: JSONType | None = None, + json: JsonValue | None = None, ): return signature_payload( method=method, @@ -199,7 +195,7 @@ def payload_from_request( class Signer(SignatureScheme): - def sign(self, payload: JSONType | bytes) -> Signature: + def sign(self, payload: JsonValue | bytes) -> Signature: signature = Signature( signature_type=self.signature_type, signatory=self.get_signatory(), @@ -211,7 +207,7 @@ def sign(self, payload: JSONType | bytes) -> Signature: return signature def signature_for_request( - self, method: str, url: str, headers: dict[str, str], json: JSONType | None = None + self, method: str, url: str, headers: dict[str, str], json: JsonValue | None = None ) -> Signature: return self.sign(self.payload_from_request(method, url, headers=headers, json=json)) @@ -227,7 +223,7 @@ def get_signatory(self) -> str: class Verifier(SignatureScheme): def verify( self, - payload: JSONType | bytes, + payload: JsonValue | bytes, signature: Signature, newer_than: datetime.datetime | None = None, ): From bde23eec2566c8bef70bb3241c4daebcfc448a26 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 24 Oct 2024 10:30:21 +0300 Subject: [PATCH 88/99] updated .gitignore --- .gitignore | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b6a7e4a5c..7c2e879fc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,18 @@ *.egg-info/ .idea/ .env -venv +venv/ +.venv/ .hypothesis .envrc .nox/ -__pycache__ -build -dist +.mypy_cache/ +.pdm-build/ +.pytest_cache/ +.ruff_cache/ +__pycache__/ +build/ +dist/ .pdm-python /wallets/ /facilitator/ From f4b9ecb3bd6e07050d3dd68d711155faf93c63a4 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 16:10:56 +0600 Subject: [PATCH 89/99] Fix receipts missing fields --- miner/app/src/compute_horde_miner/miner/liveness_check.py | 1 + .../miner/miner_consumer/validator_interface.py | 3 ++- miner/app/src/compute_horde_miner/miner/tasks.py | 6 ++++++ .../miner/tests/integration/test_mocked_executor_manager.py | 1 + .../app/src/compute_horde_validator/validator/tasks.py | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/liveness_check.py b/miner/app/src/compute_horde_miner/miner/liveness_check.py index c400a76a2..f372ff305 100644 --- a/miner/app/src/compute_horde_miner/miner/liveness_check.py +++ b/miner/app/src/compute_horde_miner/miner/liveness_check.py @@ -158,6 +158,7 @@ async def drive_executor() -> float: timestamp=datetime.datetime.now(datetime.UTC), executor_class=DEFAULT_EXECUTOR_CLASS, max_timeout=JOB_TIMEOUT_SECONDS, + is_organic=False, ttl=30, ) receipt_signature = f"0x{keypair.sign(receipt_payload.blob_for_signing()).hex()}" diff --git a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py index 89aedb54b..d57d6cbe3 100644 --- a/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py +++ b/miner/app/src/compute_horde_miner/miner/miner_consumer/validator_interface.py @@ -400,7 +400,7 @@ async def handle_job_started_receipt(self, payload: JobStartedReceiptPayload, si timestamp=payload.timestamp, executor_class=payload.executor_class, max_timeout=payload.max_timeout, - is_organic=msg.payload.is_organic, + is_organic=payload.is_organic, ttl=payload.ttl, ) @@ -422,6 +422,7 @@ async def handle_job_accepted_receipt( miner_hotkey=msg.payload.miner_hotkey, validator_hotkey=msg.payload.validator_hotkey, timestamp=msg.payload.timestamp, + time_accepted=msg.payload.time_accepted, ttl=msg.payload.ttl, ) prepare_receipts.delay() diff --git a/miner/app/src/compute_horde_miner/miner/tasks.py b/miner/app/src/compute_horde_miner/miner/tasks.py index 79f1a698e..953d52fef 100644 --- a/miner/app/src/compute_horde_miner/miner/tasks.py +++ b/miner/app/src/compute_horde_miner/miner/tasks.py @@ -119,6 +119,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, executor_class=receipt.payload.executor_class, max_timeout=receipt.payload.max_timeout, @@ -146,6 +148,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, time_accepted=receipt.payload.time_accepted, ttl=receipt.payload.ttl, @@ -173,6 +177,8 @@ def get_receipts_from_old_miner(): job_uuid=receipt.payload.job_uuid, miner_hotkey=receipt.payload.miner_hotkey, validator_hotkey=receipt.payload.validator_hotkey, + validator_signature=receipt.validator_signature, + miner_signature=receipt.miner_signature, timestamp=receipt.payload.timestamp, time_started=receipt.payload.time_started, time_took_us=receipt.payload.time_took_us, diff --git a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py index 311da15ab..ff7646e8d 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py +++ b/miner/app/src/compute_horde_miner/miner/tests/integration/test_mocked_executor_manager.py @@ -100,6 +100,7 @@ async def run_regular_flow_test(validator_key: str, job_uuid: str): "timestamp": "2020-01-01T00:00Z", "executor_class": DEFAULT_EXECUTOR_CLASS, "max_timeout": 60, + "is_organic": True, "ttl": 5, }, "job_started_receipt_signature": "gibberish", diff --git a/validator/app/src/compute_horde_validator/validator/tasks.py b/validator/app/src/compute_horde_validator/validator/tasks.py index d1bc87804..3cb47b89e 100644 --- a/validator/app/src/compute_horde_validator/validator/tasks.py +++ b/validator/app/src/compute_horde_validator/validator/tasks.py @@ -1066,6 +1066,7 @@ def fetch_receipts_from_miner(hotkey: str, ip: str, port: int): miner_signature=receipt.miner_signature, validator_signature=receipt.validator_signature, timestamp=receipt.payload.timestamp, + time_accepted=receipt.payload.time_accepted, ttl=receipt.payload.ttl, ) for receipt in receipts From aa38a402b755c1918465e6e98562e781f0100f66 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Tue, 15 Oct 2024 19:36:37 +0300 Subject: [PATCH 90/99] manifest executor count per executor class --- ...manifest_unique_miner_manifest_and_more.py | 28 +++++++++++++ .../validator/models.py | 5 ++- .../validator/synthetic_jobs/batch_run.py | 39 +++++++++++-------- .../tests/test_synthetic_jobs/test_batch.py | 2 + .../validator/tests/test_utils.py | 2 + 5 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 validator/app/src/compute_horde_validator/validator/migrations/0040_remove_minermanifest_unique_miner_manifest_and_more.py diff --git a/validator/app/src/compute_horde_validator/validator/migrations/0040_remove_minermanifest_unique_miner_manifest_and_more.py b/validator/app/src/compute_horde_validator/validator/migrations/0040_remove_minermanifest_unique_miner_manifest_and_more.py new file mode 100644 index 000000000..19d9bd672 --- /dev/null +++ b/validator/app/src/compute_horde_validator/validator/migrations/0040_remove_minermanifest_unique_miner_manifest_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.15 on 2024-10-17 13:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("validator", "0039_delete_jobfinishedreceipt_delete_jobstartedreceipt"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="minermanifest", + name="unique_miner_manifest", + ), + migrations.AddField( + model_name="minermanifest", + name="executor_class", + field=models.CharField(default="spin_up-4min.gpu-24gb", max_length=255), + preserve_default=False, + ), + migrations.AddConstraint( + model_name="minermanifest", + constraint=models.UniqueConstraint( + fields=("miner", "batch", "executor_class"), name="unique_miner_manifest" + ), + ), + ] diff --git a/validator/app/src/compute_horde_validator/validator/models.py b/validator/app/src/compute_horde_validator/validator/models.py index 7ebec6261..414914d3e 100644 --- a/validator/app/src/compute_horde_validator/validator/models.py +++ b/validator/app/src/compute_horde_validator/validator/models.py @@ -162,12 +162,15 @@ class MinerManifest(models.Model): miner = models.ForeignKey(Miner, on_delete=models.CASCADE) batch = models.ForeignKey(SyntheticJobBatch, on_delete=models.CASCADE) created_at = models.DateTimeField(auto_now_add=True) + executor_class = models.CharField(max_length=255) executor_count = models.IntegerField(default=0) online_executor_count = models.IntegerField(default=0) class Meta: constraints = [ - UniqueConstraint(fields=["miner", "batch"], name="unique_miner_manifest"), + UniqueConstraint( + fields=["miner", "batch", "executor_class"], name="unique_miner_manifest" + ), ] diff --git a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py index bec9b29c8..211edac84 100644 --- a/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py +++ b/validator/app/src/compute_horde_validator/validator/synthetic_jobs/batch_run.py @@ -407,8 +407,8 @@ class BatchContext: clients: dict[str, MinerClient] executors: dict[str, defaultdict[ExecutorClass, int]] job_generators: dict[str, dict[ExecutorClass, list[BaseSyntheticJobGenerator]]] - online_executor_count: dict[str, int] - previous_online_executor_count: dict[str, int | None] + online_executor_count: dict[str, defaultdict[ExecutorClass, int]] + previous_online_executor_count: dict[str, defaultdict[ExecutorClass, int]] manifests: dict[str, ExecutorManifest | None] manifest_events: dict[str, asyncio.Event] @@ -654,8 +654,8 @@ def _init_context( ctx.clients[hotkey] = create_miner_client(ctx=ctx, miner_hotkey=hotkey) ctx.executors[hotkey] = defaultdict(int) ctx.job_generators[hotkey] = {} - ctx.online_executor_count[hotkey] = 0 - ctx.previous_online_executor_count[hotkey] = None + ctx.online_executor_count[hotkey] = defaultdict(int) + ctx.previous_online_executor_count[hotkey] = defaultdict(int) ctx.manifests[hotkey] = None ctx.manifest_events[hotkey] = asyncio.Event() @@ -665,10 +665,11 @@ def _init_context( def _get_max_spin_up_time(ctx: BatchContext) -> int: max_spin_up_time = _MIN_SPIN_UP_TIME for executors in ctx.executors.values(): - for executor_class in executors.keys(): - spin_up_time = EXECUTOR_CLASS[executor_class].spin_up_time - assert spin_up_time is not None - max_spin_up_time = max(max_spin_up_time, spin_up_time) + for executor_class, count in executors.items(): + if count > 0: + spin_up_time = EXECUTOR_CLASS[executor_class].spin_up_time + assert spin_up_time is not None + max_spin_up_time = max(max_spin_up_time, spin_up_time) return max_spin_up_time @@ -1368,7 +1369,7 @@ async def _score_jobs(ctx: BatchContext) -> None: # compute for each hotkey how many executors finished successfully for job in ctx.jobs.values(): if job.success: - ctx.online_executor_count[job.miner_hotkey] += 1 + ctx.online_executor_count[job.miner_hotkey][job.executor_class] += 1 # apply manifest bonus # do not combine with the previous loop, we use online_executor_count @@ -1376,8 +1377,10 @@ async def _score_jobs(ctx: BatchContext) -> None: if job.success: try: job.score_manifest_multiplier = await get_manifest_multiplier( - ctx.previous_online_executor_count[job.miner_hotkey], - ctx.online_executor_count[job.miner_hotkey], + ctx.previous_online_executor_count[job.miner_hotkey].get( + job.executor_class, None + ), + ctx.online_executor_count[job.miner_hotkey].get(job.executor_class, 0), ) except (Exception, asyncio.CancelledError) as exc: logger.warning("%s failed to score: %r", job.name, exc) @@ -1405,7 +1408,8 @@ def _db_get_previous_online_executor_count(ctx: BatchContext) -> None: for manifest in MinerManifest.objects.filter(batch_id=previous_batch.id): # only update if the miner is still serving if manifest.miner.hotkey in ctx.previous_online_executor_count: - ctx.previous_online_executor_count[manifest.miner.hotkey] = ( + executor_class = ExecutorClass(manifest.executor_class) + ctx.previous_online_executor_count[manifest.miner.hotkey][executor_class] = ( manifest.online_executor_count ) @@ -1474,14 +1478,15 @@ def _db_persist(ctx: BatchContext) -> None: miner_manifests: list[MinerManifest] = [] for miner in ctx.miners.values(): - manifest = ctx.manifests[miner.hotkey] - if manifest is not None: + for executor_class, count in ctx.executors[miner.hotkey].items(): + online_executor_count = ctx.online_executor_count[miner.hotkey].get(executor_class, 0) miner_manifests.append( MinerManifest( miner=miner, batch=batch, - executor_count=manifest.total_count, - online_executor_count=ctx.online_executor_count[miner.hotkey], + executor_class=executor_class, + executor_count=count, + online_executor_count=online_executor_count, ) ) MinerManifest.objects.bulk_create(miner_manifests) @@ -1573,7 +1578,7 @@ async def execute_synthetic_batch_run( await ctx.checkpoint_system_event("_get_total_executor_count") total_executor_count = _get_total_executor_count(ctx) - if total_executor_count != 0: + if total_executor_count > 0: await ctx.checkpoint_system_event("_generate_jobs") await _generate_jobs(ctx) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/test_batch.py b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/test_batch.py index 732d9499a..b79383922 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/test_batch.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_synthetic_jobs/test_batch.py @@ -87,6 +87,7 @@ async def test_manifest_dance_incentives( await MinerManifest.objects.acreate( miner=miner, batch=batch, + executor_class=DEFAULT_EXECUTOR_CLASS, executor_count=prev_online_executor_count, online_executor_count=prev_online_executor_count, ) @@ -168,6 +169,7 @@ async def test_synthetic_job_batch( await MinerManifest.objects.acreate( miner=miner, batch=batch, + executor_class=DEFAULT_EXECUTOR_CLASS, executor_count=prev_online_executor_count, online_executor_count=prev_online_executor_count, ) diff --git a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py index cacc49e40..d6a79e89d 100644 --- a/validator/app/src/compute_horde_validator/validator/tests/test_utils.py +++ b/validator/app/src/compute_horde_validator/validator/tests/test_utils.py @@ -427,6 +427,7 @@ async def test_manifest_dance_incentives( await MinerManifest.objects.acreate( miner=miner, batch=batch, + executor_class=DEFAULT_EXECUTOR_CLASS, executor_count=prev_online_executor_count, online_executor_count=prev_online_executor_count, ) @@ -511,6 +512,7 @@ def test_create_and_run_synthetic_job_batch( MinerManifest.objects.create( miner=miner, batch=batch, + executor_class=DEFAULT_EXECUTOR_CLASS, executor_count=previous_online_executors, online_executor_count=previous_online_executors, ) From fcc5f4a755e36e3c74f4c4c6273926fa2298606b Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 16:15:17 +0600 Subject: [PATCH 91/99] Fix miner receipts test --- .../miner/tests/test_receipts.py | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 6376b6fce..b6870ef54 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -9,14 +9,16 @@ JobFinishedReceiptPayload, JobStartedReceiptPayload, V0AuthenticateRequest, + V0InitialJobRequest, + V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, - V0JobStartedReceiptRequest, ) from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.schemas import JobAcceptedReceiptPayload from django.utils import timezone from pytest_mock import MockerFixture -from compute_horde_miner.miner.models import AcceptedJob, Validator +from compute_horde_miner.miner.models import Validator from compute_horde_miner.miner.tests.validator import fake_validator @@ -41,7 +43,7 @@ async def test_receipt_is_saved( settings.DEBUG_TURN_AUTHENTICATION_OFF = True settings.BITTENSOR_WALLET = lambda: miner_wallet job_uuid = str(uuid4()) - validator = await Validator.objects.acreate( + await Validator.objects.acreate( public_key=validator_wallet.hotkey.ss58_address, active=True, ) @@ -69,28 +71,44 @@ async def test_receipt_is_saved( }, } - # Skip doing the job - await AcceptedJob.objects.acreate( - job_uuid=job_uuid, - validator=validator, - initial_job_details={}, - ) - # Send the receipts job_started_receipt_payload = JobStartedReceiptPayload( job_uuid=job_uuid, is_organic=organic_job, miner_hotkey=miner_wallet.hotkey.ss58_address, validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), executor_class=ExecutorClass.spin_up_4min__gpu_24gb, + max_timeout=60, + ttl=5, + ) + job_started_receipt_signature = _sign(job_started_receipt_payload, validator_wallet.hotkey) + await fake_validator_channel.send_to( + V0InitialJobRequest( + job_uuid=job_uuid, + executor_class=ExecutorClass.spin_up_4min__gpu_24gb, + base_docker_image_name="it's teeeeests", + timeout_seconds=123, + job_started_receipt_payload=job_started_receipt_payload, + job_started_receipt_signature=job_started_receipt_signature, + ).model_dump_json() + ) + + # skip doing the job, and only send receipts + + job_accepted_receipt_payload = JobAcceptedReceiptPayload( + job_uuid=job_uuid, + miner_hotkey=miner_wallet.hotkey.ss58_address, + validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), time_accepted=timezone.now(), - max_timeout=123, + ttl=5, ) await fake_validator_channel.send_to( - V0JobStartedReceiptRequest( + V0JobAcceptedReceiptRequest( job_uuid=job_uuid, - payload=job_started_receipt_payload, - signature=_sign(job_started_receipt_payload, validator_wallet.hotkey), + payload=job_accepted_receipt_payload, + signature=_sign(job_accepted_receipt_payload, validator_wallet.hotkey), ).model_dump_json() ) @@ -98,6 +116,7 @@ async def test_receipt_is_saved( job_uuid=job_uuid, miner_hotkey=miner_wallet.hotkey.ss58_address, validator_hotkey=validator_wallet.hotkey.ss58_address, + timestamp=timezone.now(), time_started=timezone.now(), time_took_us=123, score_str="123.45", From fa596324e59f7d538a0adf5807186468d6bbcfa8 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Thu, 24 Oct 2024 17:37:44 +0600 Subject: [PATCH 92/99] Align receipts test with stub executor manager --- .../miner/tests/test_receipts.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index b6870ef54..4f63e253f 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -3,6 +3,7 @@ import bittensor import pytest +from compute_horde.base.volume import VolumeType from compute_horde.executor_class import ExecutorClass from compute_horde.mv_protocol.validator_requests import ( AuthenticationPayload, @@ -19,6 +20,7 @@ from pytest_mock import MockerFixture from compute_horde_miner.miner.models import Validator +from compute_horde_miner.miner.tests.executor_manager import fake_executor from compute_horde_miner.miner.tests.validator import fake_validator @@ -26,6 +28,14 @@ def _sign(msg, key: bittensor.Keypair): return f"0x{key.sign(msg.blob_for_signing()).hex()}" +@pytest.fixture +def job_uuid(): + _job_uuid = str(uuid4()) + fake_executor.job_uuid = _job_uuid + yield _job_uuid + fake_executor.job_uuid = None + + @pytest.mark.parametrize( "organic_job", (True, False), @@ -37,12 +47,12 @@ async def test_receipt_is_saved( miner_wallet: bittensor.wallet, mocker: MockerFixture, organic_job: bool, + job_uuid: str, settings, ) -> None: mocker.patch("compute_horde_miner.miner.miner_consumer.validator_interface.prepare_receipts") settings.DEBUG_TURN_AUTHENTICATION_OFF = True settings.BITTENSOR_WALLET = lambda: miner_wallet - job_uuid = str(uuid4()) await Validator.objects.acreate( public_key=validator_wallet.hotkey.ss58_address, active=True, @@ -88,7 +98,8 @@ async def test_receipt_is_saved( job_uuid=job_uuid, executor_class=ExecutorClass.spin_up_4min__gpu_24gb, base_docker_image_name="it's teeeeests", - timeout_seconds=123, + timeout_seconds=60, + volume_type=VolumeType.inline, job_started_receipt_payload=job_started_receipt_payload, job_started_receipt_signature=job_started_receipt_signature, ).model_dump_json() From 533c95f14905ffe30d4ae6cc49effcc8d1b148d5 Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Thu, 24 Oct 2024 21:30:20 +0300 Subject: [PATCH 93/99] fix types --- .../compute_horde/fv_protocol/facilitator_requests.py | 2 +- .../compute_horde/fv_protocol/validator_requests.py | 8 ++++---- .../validator/organic_jobs/facilitator_client.py | 1 + .../validator/organic_jobs/miner_driver.py | 7 ++----- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py index d4a07eca0..c5e2374c0 100644 --- a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -125,6 +125,6 @@ def validate_at_least_docker_image_or_raw_script(self) -> Self: JobRequest = Annotated[ - V0JobRequest | V1JobRequest, + V0JobRequest | V1JobRequest | V2JobRequest, pydantic.Field(discriminator="message_type"), ] diff --git a/compute_horde/compute_horde/fv_protocol/validator_requests.py b/compute_horde/compute_horde/fv_protocol/validator_requests.py index c57f8385c..e5258497c 100644 --- a/compute_horde/compute_horde/fv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/validator_requests.py @@ -1,4 +1,4 @@ -from typing import Any, Self +from typing import Any, Literal, Self import bittensor from pydantic import BaseModel @@ -7,13 +7,13 @@ class V0Heartbeat(BaseModel, extra="forbid"): """Message sent from validator to facilitator to keep connection alive""" - message_type: str = "V0Heartbeat" + message_type: Literal["V0Heartbeat"] = "V0Heartbeat" class V0AuthenticationRequest(BaseModel, extra="forbid"): """Message sent from validator to facilitator to authenticate itself""" - message_type: str = "V0AuthenticationRequest" + message_type: Literal["V0AuthenticationRequest"] = "V0AuthenticationRequest" public_key: str signature: str @@ -43,7 +43,7 @@ def ss58_address(self) -> str: class V0MachineSpecsUpdate(BaseModel, extra="forbid"): """Message sent from validator to facilitator to update miner specs""" - message_type: str = "V0MachineSpecsUpdate" + message_type: Literal["V0MachineSpecsUpdate"] = "V0MachineSpecsUpdate" miner_hotkey: str validator_hotkey: str specs: dict[str, Any] diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py index 9b41e11c3..605926942 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py @@ -243,6 +243,7 @@ async def get_miner_axon_info(self, hotkey: str) -> bittensor.AxonInfo: async def miner_driver(self, job_request: JobRequest): """drive a miner client from job start to completion, then close miner connection""" + assert job_request.miner_hotkey is not None miner, _ = await Miner.objects.aget_or_create(hotkey=job_request.miner_hotkey) miner_axon_info = await self.get_miner_axon_info(job_request.miner_hotkey) job = await OrganicJob.objects.acreate( diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py index 6cb212eb6..55537b20b 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/miner_driver.py @@ -4,10 +4,7 @@ from typing import Literal from compute_horde.executor_class import ExecutorClass -from compute_horde.fv_protocol.facilitator_requests import ( - V0JobRequest, - V1JobRequest, -) +from compute_horde.fv_protocol.facilitator_requests import JobRequest from compute_horde.miner_client.organic import ( FailureReason, OrganicJobDetails, @@ -93,7 +90,7 @@ async def _dummy_notify_callback(_: JobStatusUpdate) -> None: async def execute_organic_job( miner_client: MinerClient, job: OrganicJob, - job_request: V0JobRequest | V1JobRequest | AdminJobRequest, + job_request: JobRequest | AdminJobRequest, total_job_timeout: int = 300, wait_timeout: int = 300, notify_callback: Callable[[JobStatusUpdate], Awaitable[None]] = _dummy_notify_callback, From e56d38fdffdc8eeb475e290095eb8e2f18249b7a Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Fri, 25 Oct 2024 01:02:23 +0600 Subject: [PATCH 94/99] Fix integration tests --- tests/integration_tests/test_miner_on_dev_executor_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index ec8c1d852..636e574e1 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -161,6 +161,7 @@ async def test_echo_image(self): "timestamp": datetime.now(tz=UTC).isoformat(), "executor_class": DEFAULT_EXECUTOR_CLASS, "max_timeout": 60, + "is_organic": True, "ttl": 30, } blob = json.dumps(receipt_payload, sort_keys=True) From f0dcef727aba9f9adfa5fd15f5e20ecb1d8db26b Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 25 Oct 2024 20:24:05 +0300 Subject: [PATCH 95/99] fix function name to signal it's public --- compute_horde/compute_horde/mv_protocol/validator_requests.py | 4 ++-- compute_horde/compute_horde/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compute_horde/compute_horde/mv_protocol/validator_requests.py b/compute_horde/compute_horde/mv_protocol/validator_requests.py index 129da6568..a3a76d0bb 100644 --- a/compute_horde/compute_horde/mv_protocol/validator_requests.py +++ b/compute_horde/compute_horde/mv_protocol/validator_requests.py @@ -12,7 +12,7 @@ from ..base.volume import Volume, VolumeType from ..base_requests import BaseRequest, JobMixin from ..executor_class import ExecutorClass -from ..utils import MachineSpecs, _json_dumps_default +from ..utils import MachineSpecs, json_dumps_default SAFE_DOMAIN_REGEX = re.compile(r".*") @@ -99,7 +99,7 @@ class ReceiptPayload(pydantic.BaseModel): def blob_for_signing(self): # pydantic v2 does not support sort_keys anymore. - return json.dumps(self.model_dump(), sort_keys=True, default=_json_dumps_default) + return json.dumps(self.model_dump(), sort_keys=True, default=json_dumps_default) class JobFinishedReceiptPayload(ReceiptPayload): diff --git a/compute_horde/compute_horde/utils.py b/compute_horde/compute_horde/utils.py index 1b95e343e..70dfc281b 100644 --- a/compute_horde/compute_horde/utils.py +++ b/compute_horde/compute_horde/utils.py @@ -52,7 +52,7 @@ def get_validators(netuid=12, network="finney", block: int | None = None) -> lis return neurons[:VALIDATORS_LIMIT] -def _json_dumps_default(obj): +def json_dumps_default(obj): if isinstance(obj, datetime.datetime): return obj.isoformat() From 49c6f18dcb61f566a477832ed415d8b25be3bc2b Mon Sep 17 00:00:00 2001 From: Adal Chiriliuc Date: Fri, 25 Oct 2024 20:33:46 +0300 Subject: [PATCH 96/99] removed separate payload from signed request --- .../fv_protocol/facilitator_requests.py | 25 ++++++--- compute_horde/tests/test_job_request.py | 53 +++++++++++++++++++ .../organic_jobs/facilitator_client.py | 4 +- 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 compute_horde/tests/test_job_request.py diff --git a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py index c5e2374c0..3c8a0a81b 100644 --- a/compute_horde/compute_horde/fv_protocol/facilitator_requests.py +++ b/compute_horde/compute_horde/fv_protocol/facilitator_requests.py @@ -21,12 +21,12 @@ class Response(BaseModel, extra="forbid"): errors: list[Error] = [] -class SignedRequest(BaseModel, extra="forbid"): - signature_type: str - signatory: str - timestamp_ns: int - signature: str - signed_payload: JsonValue +class Signature(BaseModel, extra="forbid"): + # has defaults to allow easy instantiation + signature_type: str = "" + signatory: str = "" + timestamp_ns: int = 0 + signature: str = "" class V0JobRequest(BaseModel, extra="forbid"): @@ -102,8 +102,10 @@ class V2JobRequest(BaseModel, extra="forbid"): # this points to a `ValidatorConsumer.job_new` handler (fuck you django-channels!) type: Literal["job.new"] = "job.new" message_type: Literal["V2JobRequest"] = "V2JobRequest" + signature: Signature | None = None + + # !!! all fields below are included in the signed json payload uuid: str - miner_hotkey: str | None executor_class: ExecutorClass docker_image: str raw_script: str @@ -112,11 +114,18 @@ class V2JobRequest(BaseModel, extra="forbid"): use_gpu: bool volume: Volume | None = None output_upload: OutputUpload | None = None - signed_request: SignedRequest + # !!! all fields above are included in the signed json payload def get_args(self): return self.args + def json_for_signing(self) -> JsonValue: + payload = self.model_dump(mode="json") + del payload["type"] + del payload["message_type"] + del payload["signature"] + return payload + @model_validator(mode="after") def validate_at_least_docker_image_or_raw_script(self) -> Self: if not (bool(self.docker_image) or bool(self.raw_script)): diff --git a/compute_horde/tests/test_job_request.py b/compute_horde/tests/test_job_request.py new file mode 100644 index 000000000..190084191 --- /dev/null +++ b/compute_horde/tests/test_job_request.py @@ -0,0 +1,53 @@ +import base64 +import uuid + +from compute_horde.base.volume import VolumeType, ZipUrlVolume +from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS +from compute_horde.fv_protocol.facilitator_requests import Signature, V2JobRequest +from compute_horde.signature import BittensorWalletSigner, BittensorWalletVerifier +from compute_horde.signature import Signature as RawSignature + + +def test_signed_job_roundtrip(signature_wallet): + volume = ZipUrlVolume( + volume_type=VolumeType.zip_url, + contents="https://example.com/input.zip", + relative_path="input", + ) + job = V2JobRequest( + uuid=str(uuid.uuid4()), + executor_class=DEFAULT_EXECUTOR_CLASS, + docker_image="hello-world", + raw_script="bash", + args=["--verbose", "--dry-run"], + env={"CUDA": "1"}, + use_gpu=False, + volume=volume, + output_upload=None, + ) + + signer = BittensorWalletSigner(signature_wallet) + payload = job.json_for_signing() + raw_signature = signer.sign(payload) + + job.signature = Signature( + signature_type=raw_signature.signature_type, + signatory=raw_signature.signatory, + timestamp_ns=raw_signature.timestamp_ns, + signature=base64.b64encode(raw_signature.signature).decode("utf8"), + ) + + job_json = job.model_dump_json() + deserialized_job = V2JobRequest.model_validate_json(job_json) + + assert deserialized_job.signature is not None + deserialized_raw_signature = RawSignature( + signature_type=deserialized_job.signature.signature_type, + signatory=deserialized_job.signature.signatory, + timestamp_ns=deserialized_job.signature.timestamp_ns, + signature=base64.b64decode(deserialized_job.signature.signature), + ) + + deserialized_payload = deserialized_job.json_for_signing() + verifier = BittensorWalletVerifier() + verifier.verify(deserialized_payload, deserialized_raw_signature) diff --git a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py index 605926942..5868beda4 100644 --- a/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py +++ b/validator/app/src/compute_horde_validator/validator/organic_jobs/facilitator_client.py @@ -8,7 +8,7 @@ import tenacity import websockets from channels.layers import get_channel_layer -from compute_horde.fv_protocol.facilitator_requests import Error, JobRequest, Response +from compute_horde.fv_protocol.facilitator_requests import Error, JobRequest, Response, V2JobRequest from compute_horde.fv_protocol.validator_requests import ( V0AuthenticationRequest, V0Heartbeat, @@ -243,7 +243,7 @@ async def get_miner_axon_info(self, hotkey: str) -> bittensor.AxonInfo: async def miner_driver(self, job_request: JobRequest): """drive a miner client from job start to completion, then close miner connection""" - assert job_request.miner_hotkey is not None + assert not isinstance(job_request, V2JobRequest) miner, _ = await Miner.objects.aget_or_create(hotkey=job_request.miner_hotkey) miner_axon_info = await self.get_miner_axon_info(job_request.miner_hotkey) job = await OrganicJob.objects.acreate( From f749d4bd5f23515139a6b029b99c53207821f37f Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:39:48 +0600 Subject: [PATCH 97/99] Add missing assert for miner receipts test --- miner/app/src/compute_horde_miner/miner/tests/test_receipts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py index 4f63e253f..b5b3bac63 100644 --- a/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py +++ b/miner/app/src/compute_horde_miner/miner/tests/test_receipts.py @@ -14,7 +14,7 @@ V0JobAcceptedReceiptRequest, V0JobFinishedReceiptRequest, ) -from compute_horde.receipts.models import JobFinishedReceipt, JobStartedReceipt +from compute_horde.receipts.models import JobAcceptedReceipt, JobFinishedReceipt, JobStartedReceipt from compute_horde.receipts.schemas import JobAcceptedReceiptPayload from django.utils import timezone from pytest_mock import MockerFixture @@ -143,4 +143,5 @@ async def test_receipt_is_saved( assert await JobStartedReceipt.objects.filter( job_uuid=job_uuid, is_organic=organic_job ).aexists() + assert await JobAcceptedReceipt.objects.filter(job_uuid=job_uuid).aexists() assert await JobFinishedReceipt.objects.filter(job_uuid=job_uuid).aexists() From 65f144b4892fc1e516730cda5a882785041050e8 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:45:55 +0600 Subject: [PATCH 98/99] Use different seed for miner/validator wallets in tests --- tests/integration_tests/test_miner_on_dev_executor_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_miner_on_dev_executor_manager.py b/tests/integration_tests/test_miner_on_dev_executor_manager.py index 636e574e1..2610858e2 100644 --- a/tests/integration_tests/test_miner_on_dev_executor_manager.py +++ b/tests/integration_tests/test_miner_on_dev_executor_manager.py @@ -42,8 +42,8 @@ def get_validator_wallet(): wallet = bittensor.wallet(name="test_validator") try: # workaround the overwrite flag - wallet.regenerate_coldkey(seed="0" * 64, use_password=False, overwrite=True) - wallet.regenerate_hotkey(seed="1" * 64, use_password=False, overwrite=True) + wallet.regenerate_coldkey(seed="2" * 64, use_password=False, overwrite=True) + wallet.regenerate_hotkey(seed="3" * 64, use_password=False, overwrite=True) except Exception as e: logger.error(f"Failed to create wallet: {e}") return wallet From be1f0b888d89a13a656f11616865ae47da2be483 Mon Sep 17 00:00:00 2001 From: Enam Mijbah Noor Date: Mon, 28 Oct 2024 18:56:24 +0600 Subject: [PATCH 99/99] Add indices for `timestamp` fields of receipt models --- ...py => 0003_jobacceptedreceipt_and_more.py} | 67 +++++++++++-------- .../compute_horde/receipts/models.py | 3 + 2 files changed, 43 insertions(+), 27 deletions(-) rename compute_horde/compute_horde/receipts/migrations/{0003_remove_jobstartedreceipt_time_accepted_and_more.py => 0003_jobacceptedreceipt_and_more.py} (71%) diff --git a/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py b/compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py similarity index 71% rename from compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py rename to compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py index 903d6ebbf..1ec5d3df6 100644 --- a/compute_horde/compute_horde/receipts/migrations/0003_remove_jobstartedreceipt_time_accepted_and_more.py +++ b/compute_horde/compute_horde/receipts/migrations/0003_jobacceptedreceipt_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.1 on 2024-10-23 14:11 +# Generated by Django 5.1.1 on 2024-10-28 12:53 import datetime @@ -11,6 +11,28 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name="JobAcceptedReceipt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("job_uuid", models.UUIDField()), + ("validator_hotkey", models.CharField(max_length=256)), + ("miner_hotkey", models.CharField(max_length=256)), + ("validator_signature", models.CharField(max_length=256)), + ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), + ("timestamp", models.DateTimeField()), + ("time_accepted", models.DateTimeField()), + ("ttl", models.IntegerField()), + ], + options={ + "abstract": False, + }, + ), migrations.RemoveField( model_name="jobstartedreceipt", name="time_accepted", @@ -37,31 +59,22 @@ class Migration(migrations.Migration): field=models.IntegerField(default=0), preserve_default=False, ), - migrations.CreateModel( - name="JobAcceptedReceipt", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ("job_uuid", models.UUIDField()), - ("validator_hotkey", models.CharField(max_length=256)), - ("miner_hotkey", models.CharField(max_length=256)), - ("validator_signature", models.CharField(max_length=256)), - ("miner_signature", models.CharField(blank=True, max_length=256, null=True)), - ("timestamp", models.DateTimeField()), - ("time_accepted", models.DateTimeField()), - ("ttl", models.IntegerField()), - ], - options={ - "abstract": False, - "constraints": [ - models.UniqueConstraint( - fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" - ) - ], - }, + migrations.AddIndex( + model_name="jobfinishedreceipt", + index=models.Index(fields=["timestamp"], name="jobfinishedreceipt_ts_idx"), + ), + migrations.AddIndex( + model_name="jobstartedreceipt", + index=models.Index(fields=["timestamp"], name="jobstartedreceipt_ts_idx"), + ), + migrations.AddIndex( + model_name="jobacceptedreceipt", + index=models.Index(fields=["timestamp"], name="jobacceptedreceipt_ts_idx"), + ), + migrations.AddConstraint( + model_name="jobacceptedreceipt", + constraint=models.UniqueConstraint( + fields=("job_uuid",), name="receipts_unique_jobacceptedreceipt_job_uuid" + ), ), ] diff --git a/compute_horde/compute_horde/receipts/models.py b/compute_horde/compute_horde/receipts/models.py index 56a3a27e5..04c45f662 100644 --- a/compute_horde/compute_horde/receipts/models.py +++ b/compute_horde/compute_horde/receipts/models.py @@ -28,6 +28,9 @@ class Meta: constraints = [ models.UniqueConstraint(fields=["job_uuid"], name="receipts_unique_%(class)s_job_uuid"), ] + indexes = [ + models.Index(fields=["timestamp"], name="%(class)s_ts_idx"), + ] def __str__(self): return f"job_uuid: {self.job_uuid}"